diff --git a/.github/scripts/package-client-cpp-manylinux228.sh b/.github/scripts/package-client-cpp-manylinux228.sh new file mode 100755 index 0000000000000..d8e073c459979 --- /dev/null +++ b/.github/scripts/package-client-cpp-manylinux228.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Build client-cpp in manylinux_2_28 and verify max required GLIBC symbol <= 2.28. +set -euxo pipefail + +MACHINE=$(uname -m) +case "${MACHINE}" in + x86_64) + CMAKE_PKG_ARCH=linux-x86_64 + JDK_API_ARCH=linux/x64 + DEFAULT_CLASSIFIER=linux-x86_64-glibc2.28 + ;; + aarch64) + CMAKE_PKG_ARCH=linux-aarch64 + JDK_API_ARCH=linux/aarch64 + DEFAULT_CLASSIFIER=linux-aarch64-glibc2.28 + ;; + *) + echo "Unsupported architecture: ${MACHINE}" >&2 + exit 1 + ;; +esac + +PACKAGE_CLASSIFIER="${PACKAGE_CLASSIFIER:-${DEFAULT_CLASSIFIER}}" + +CMAKE_VERSION=3.28.4 +CMAKE_DIR="/opt/cmake-${CMAKE_VERSION}" +if [[ ! -x "${CMAKE_DIR}/bin/cmake" ]]; then + curl -fsSL -o /tmp/cmake.tar.gz "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-${CMAKE_PKG_ARCH}.tar.gz" + rm -rf "${CMAKE_DIR}" + mkdir -p /opt + tar xf /tmp/cmake.tar.gz -C /opt + mv "/opt/cmake-${CMAKE_VERSION}-${CMAKE_PKG_ARCH}" "${CMAKE_DIR}" +fi + +JAVA_HOME=/opt/jdk-17 +if [[ ! -x "${JAVA_HOME}/bin/java" ]]; then + curl -fsSL -o /tmp/jdk17.tar.gz "https://api.adoptium.net/v3/binary/latest/17/ga/${JDK_API_ARCH}/jdk/hotspot/normal/eclipse?project=jdk" + rm -rf /opt/jdk-17* + mkdir -p /opt + tar xf /tmp/jdk17.tar.gz -C /opt + JAVA_HOME=$(find /opt -maxdepth 1 -type d -name 'jdk-17*' -print -quit) + ln -sfn "${JAVA_HOME}" /opt/jdk-17 + JAVA_HOME=/opt/jdk-17 +fi + +export PATH="${CMAKE_DIR}/bin:${JAVA_HOME}/bin:${PATH}" +export JAVA_HOME + +gcc --version +c++ --version +gcc_major=$(gcc -dumpversion | cut -d. -f1) +if (( gcc_major < 14 )); then + echo "ERROR: GCC >= 14 is required; got $(gcc -dumpversion)" + exit 1 +fi +cmake --version +java -version + +cd "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE is not set}" +./mvnw clean package -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests \ + -Dspotless.skip=true \ + -Dclient.cpp.package.classifier="${PACKAGE_CLASSIFIER}" + +SO="iotdb-client/client-cpp/target/install/lib/libiotdb_session.so" +test -f "${SO}" + +echo "=== Build host glibc ===" +ldd --version 2>&1 | sed -n '1p' + +echo "=== Highest GLIBC_* symbols in libiotdb_session.so ===" +objdump -T "${SO}" | grep GLIBC_ | sed "s/.*GLIBC_/GLIBC_/" | sort -Vu | tail -10 + +max_glibc=$(objdump -T "${SO}" | grep -oE "GLIBC_[0-9.]+" | sed "s/GLIBC_//" | sort -t. -k1,1n -k2,2n -k3,3n | tail -1) +echo "max_glibc=${max_glibc}" + +if [[ -z "${max_glibc}" ]]; then + echo "ERROR: could not determine max GLIBC version from ${SO}" + exit 1 +fi + +if awk -v max="${max_glibc}" "BEGIN { exit !(max > 2.28) }"; then + echo "ERROR: libiotdb_session.so requires glibc > 2.28 (max=${max_glibc})" + exit 1 +fi + +echo "glibc compatibility check passed (max=${max_glibc} <= 2.28)" diff --git a/.github/workflows/client-cpp-package.yml b/.github/workflows/client-cpp-package.yml new file mode 100644 index 0000000000000..db2d754be8eaa --- /dev/null +++ b/.github/workflows/client-cpp-package.yml @@ -0,0 +1,454 @@ +# Publish-oriented packaging workflow: +# - manual runs, release:published, v* tags +# - rc/** pushes only when C++-related paths change (job should-package) +# release events always use the workflow file from the default branch. +name: C++ Client package + +on: + workflow_dispatch: + inputs: + variants: + description: "Which packages to build" + required: false + default: all + type: choice + options: + - all + - linux + - macos + - windows + - windows-vs2017 + - windows-vs2019 + - windows-vs2022 + - windows-vs2026 + release: + types: [published] + push: + branches: + - "rc/**" + tags: + - "v*" + +concurrency: + group: client-cpp-package-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_ARGS: --batch-mode --no-transfer-progress + +jobs: + should-package: + # Keep rc branch cost low: skip full matrix when unrelated files change. + runs-on: ubuntu-latest + outputs: + run: ${{ steps.result.outputs.run }} + steps: + - uses: actions/checkout@v5 + if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/rc/') + - uses: dorny/paths-filter@v3 + id: filter + if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/rc/') + with: + filters: | + cpp: + - 'iotdb-client/client-cpp/**' + - 'iotdb-client/pom.xml' + - 'iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift' + - 'iotdb-protocol/thrift-commons/src/main/thrift/common.thrift' + - '.github/workflows/client-cpp-package.yml' + - '.github/scripts/package-client-cpp-*.sh' + - id: result + shell: bash + run: | + set -euo pipefail + E="${{ github.event_name }}" + R="${{ github.ref }}" + if [[ "$E" == "workflow_dispatch" ]] || [[ "$E" == "release" ]]; then + echo "run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + if [[ "$E" == "push" ]] && [[ "$R" == refs/tags/v* ]]; then + echo "run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + if [[ "$E" == "push" ]] && [[ "$R" =~ ^refs/heads/rc/ ]]; then + if [[ "${{ steps.filter.outputs.cpp }}" == "true" ]]; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "run=false" >> "$GITHUB_OUTPUT" + fi + exit 0 + fi + echo "run=false" >> "$GITHUB_OUTPUT" + + resolve-matrix: + name: Resolve package matrix + needs: should-package + if: needs.should-package.outputs.run == 'true' + runs-on: ubuntu-latest + outputs: + run_linux: ${{ steps.filter.outputs.run_linux }} + run_macos: ${{ steps.filter.outputs.run_macos }} + run_windows: ${{ steps.filter.outputs.run_windows }} + windows_matrix: ${{ steps.filter.outputs.windows_matrix }} + steps: + - id: filter + # Resolve workflow_dispatch variants to a compact matrix payload for fromJSON(). + shell: bash + run: | + set -euo pipefail + VARIANT="${{ github.event.inputs.variants || 'all' }}" + + run_linux=false + run_macos=false + run_windows=false + case "$VARIANT" in + all) + run_linux=true + run_macos=true + run_windows=true + ;; + linux) run_linux=true ;; + macos) run_macos=true ;; + windows) run_windows=true ;; + windows-vs2017|windows-vs2019|windows-vs2022|windows-vs2026) + run_windows=true + ;; + *) + echo "Unknown variant: $VARIANT" >&2 + exit 1 + ;; + esac + + echo "run_linux=${run_linux}" >> "$GITHUB_OUTPUT" + echo "run_macos=${run_macos}" >> "$GITHUB_OUTPUT" + echo "run_windows=${run_windows}" >> "$GITHUB_OUTPUT" + + # Compact JSON (no leading whitespace); required for GITHUB_OUTPUT + fromJSON(). + WINDOWS_MATRIX='[{"name":"windows-vs2026","runs-on":"windows-2025-vs2026","boost_choco":"","boost_choco_version":"","cmake_generator":"Visual Studio 18 2026","package_classifier":"windows-x86_64-msvc14.4","vs_choco":"","vs_choco_params":""},{"name":"windows-vs2022","runs-on":"windows-2022","boost_choco":"","boost_choco_version":"","cmake_generator":"","package_classifier":"windows-x86_64-msvc14.3","vs_choco":"","vs_choco_params":""},{"name":"windows-vs2019","runs-on":"windows-2022","boost_choco":"","boost_choco_version":"","cmake_generator":"Visual Studio 16 2019","package_classifier":"windows-x86_64-msvc14.2","vs_choco":"visualstudio2019buildtools","vs_choco_params":"--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended"},{"name":"windows-vs2017","runs-on":"windows-2022","boost_choco":"","boost_choco_version":"","cmake_generator":"Visual Studio 15 2017","package_classifier":"windows-x86_64-msvc14.1","vs_choco":"visualstudio2017buildtools","vs_choco_params":"--add Microsoft.VisualStudio.Workload.VCTools --includeRecommended"}]' + + write_windows_matrix_output() { + local matrix_json="$1" + { + echo 'windows_matrix<<__MATRIX_EOF__' + echo "${matrix_json}" + echo '__MATRIX_EOF__' + } >> "$GITHUB_OUTPUT" + } + + if [[ "$run_windows" == true ]]; then + if [[ "$VARIANT" == all || "$VARIANT" == windows ]]; then + FILTERED="$WINDOWS_MATRIX" + elif [[ "$VARIANT" =~ ^windows-vs ]]; then + FILTERED=$(echo "$WINDOWS_MATRIX" | jq -c --arg v "$VARIANT" '[.[] | select(.name == $v)]') + else + FILTERED='[]' + fi + if [[ $(echo "$FILTERED" | jq 'length') -eq 0 ]]; then + echo "No Windows matrix rows for variant=${VARIANT}" >&2 + exit 1 + fi + FILTERED=$(echo "$FILTERED" | jq -c '.') + write_windows_matrix_output "${FILTERED}" + else + write_windows_matrix_output '[]' + fi + + package-linux-glibc228: + name: Package (linux-x86_64-glibc2.28) + needs: [should-package, resolve-matrix] + if: needs.should-package.outputs.run == 'true' && needs.resolve-matrix.outputs.run_linux == 'true' + # Checkout/cache on host (Node actions need modern glibc); build in manylinux_2_28 via docker run. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Cache Maven packages + uses: actions/cache@v5 + with: + path: ~/.m2 + key: linux-glibc228-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + linux-glibc228-m2- + - name: Package client-cpp (glibc 2.28 baseline) + shell: bash + run: | + set -euxo pipefail + chmod +x .github/scripts/package-client-cpp-manylinux228.sh + docker run --rm \ + -v "${{ github.workspace }}:/workspace" \ + -v "${HOME}/.m2:/root/.m2" \ + -w /workspace \ + -e GITHUB_WORKSPACE=/workspace \ + quay.io/pypa/manylinux_2_28_x86_64 \ + bash .github/scripts/package-client-cpp-manylinux228.sh + - name: Restore workspace ownership after container build + if: always() + run: sudo chown -R "$(id -u):$(id -g)" "${{ github.workspace }}" + - name: Resolve package zip + id: pkg + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + packages=(iotdb-client/client-cpp/target/iotdb-session-cpp-*-linux-x86_64-glibc2.28.zip) + if [ "${#packages[@]}" -ne 1 ]; then + echo "Expected exactly one package zip, got: ${packages[*]:-(none)}" + exit 1 + fi + echo "path=${packages[0]}" >> "$GITHUB_OUTPUT" + echo "name=$(basename "${packages[0]}" .zip)" >> "$GITHUB_OUTPUT" + sha512="${packages[0]}.sha512" + if [ ! -f "${sha512}" ]; then + echo "Expected checksum file: ${sha512}" + exit 1 + fi + echo "sha512_path=${sha512}" >> "$GITHUB_OUTPUT" + - name: Upload zip artifact + uses: actions/upload-artifact@v6 + with: + name: ${{ steps.pkg.outputs.name }} + path: | + ${{ steps.pkg.outputs.path }} + ${{ steps.pkg.outputs.sha512_path }} + if-no-files-found: error + + package-linux-aarch64-glibc228: + name: Package (linux-aarch64-glibc2.28) + needs: [should-package, resolve-matrix] + if: needs.should-package.outputs.run == 'true' && needs.resolve-matrix.outputs.run_linux == 'true' + # Checkout/cache on host; build in manylinux_2_28 aarch64 via docker run (glibc 2.28 baseline). + runs-on: ubuntu-22.04-arm + steps: + - uses: actions/checkout@v5 + - name: Cache Maven packages + uses: actions/cache@v5 + with: + path: ~/.m2 + key: linux-aarch64-glibc228-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + linux-aarch64-glibc228-m2- + - name: Package client-cpp (glibc 2.28 baseline) + shell: bash + run: | + set -euxo pipefail + chmod +x .github/scripts/package-client-cpp-manylinux228.sh + docker run --rm \ + -v "${{ github.workspace }}:/workspace" \ + -v "${HOME}/.m2:/root/.m2" \ + -w /workspace \ + -e GITHUB_WORKSPACE=/workspace \ + quay.io/pypa/manylinux_2_28_aarch64 \ + bash .github/scripts/package-client-cpp-manylinux228.sh + - name: Restore workspace ownership after container build + if: always() + run: sudo chown -R "$(id -u):$(id -g)" "${{ github.workspace }}" + - name: Resolve package zip + id: pkg + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + packages=(iotdb-client/client-cpp/target/iotdb-session-cpp-*-linux-aarch64-glibc2.28.zip) + if [ "${#packages[@]}" -ne 1 ]; then + echo "Expected exactly one package zip, got: ${packages[*]:-(none)}" + exit 1 + fi + echo "path=${packages[0]}" >> "$GITHUB_OUTPUT" + echo "name=$(basename "${packages[0]}" .zip)" >> "$GITHUB_OUTPUT" + sha512="${packages[0]}.sha512" + if [ ! -f "${sha512}" ]; then + echo "Expected checksum file: ${sha512}" + exit 1 + fi + echo "sha512_path=${sha512}" >> "$GITHUB_OUTPUT" + - name: Upload zip artifact + uses: actions/upload-artifact@v6 + with: + name: ${{ steps.pkg.outputs.name }} + path: | + ${{ steps.pkg.outputs.path }} + ${{ steps.pkg.outputs.sha512_path }} + if-no-files-found: error + + package-macos: + name: Package (${{ matrix.name }}) + needs: [should-package, resolve-matrix] + if: needs.should-package.outputs.run == 'true' && needs.resolve-matrix.outputs.run_macos == 'true' + strategy: + fail-fast: false + matrix: + include: + - name: macos-x86_64 + runs-on: macos-15-intel + - name: macos-arm64 + runs-on: macos-latest + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@v5 + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "17" + - name: Install C++ dependencies (macOS) + shell: bash + run: | + set -euxo pipefail + brew install boost openssl llvm@17 bison + ln -sf "$(brew --prefix llvm@17)/bin/clang-format" "$(brew --prefix)/bin/clang-format" + echo "$(brew --prefix bison)/bin" >> "$GITHUB_PATH" + echo "$(brew --prefix llvm@17)/bin" >> "$GITHUB_PATH" + clang-format --version + bison --version + - name: Cache Maven packages + uses: actions/cache@v5 + with: + path: ~/.m2 + key: macos-${{ matrix.name }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + macos-${{ matrix.name }}-m2- + - name: Package client-cpp + shell: bash + env: + MACOSX_DEPLOYMENT_TARGET: "12.0" + run: | + set -euxo pipefail + ./mvnw clean package -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests \ + -Dspotless.skip=true + - name: Resolve package zip + id: pkg + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + packages=(iotdb-client/client-cpp/target/iotdb-session-cpp-*.zip) + if [ "${#packages[@]}" -ne 1 ]; then + echo "Expected exactly one package zip, got: ${packages[*]:-(none)}" + exit 1 + fi + echo "path=${packages[0]}" >> "$GITHUB_OUTPUT" + echo "name=$(basename "${packages[0]}" .zip)" >> "$GITHUB_OUTPUT" + sha512="${packages[0]}.sha512" + if [ ! -f "${sha512}" ]; then + echo "Expected checksum file: ${sha512}" + exit 1 + fi + echo "sha512_path=${sha512}" >> "$GITHUB_OUTPUT" + - name: Upload zip artifact + uses: actions/upload-artifact@v6 + with: + name: ${{ steps.pkg.outputs.name }} + path: | + ${{ steps.pkg.outputs.path }} + ${{ steps.pkg.outputs.sha512_path }} + if-no-files-found: error + + package-windows: + name: Package (${{ matrix.name }}) + needs: [should-package, resolve-matrix] + if: needs.should-package.outputs.run == 'true' && needs.resolve-matrix.outputs.run_windows == 'true' + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.resolve-matrix.outputs.windows_matrix) }} + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@v5 + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "17" + - name: Install Visual Studio Build Tools (${{ matrix.name }}) + if: matrix.vs_choco != '' + shell: pwsh + run: | + $params = '${{ matrix.vs_choco_params }}' + if ($params) { + choco install ${{ matrix.vs_choco }} -y --package-parameters="$params" --no-progress + } else { + choco install ${{ matrix.vs_choco }} -y --no-progress + } + - name: Install C++ dependencies (Windows) + shell: pwsh + run: | + choco install winflexbison3 -y --no-progress + if ('${{ matrix.boost_choco }}' -ne '') { + $boostArgs = @('install', '${{ matrix.boost_choco }}', '-y', '--no-progress') + if ('${{ matrix.boost_choco_version }}' -ne '') { + $boostArgs += @("--version=${{ matrix.boost_choco_version }}") + } + & choco @boostArgs + $boostDir = Get-ChildItem -Path 'C:\local\' -Filter 'boost_*' -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not $boostDir) { + throw "Boost not found under C:\local after installing ${{ matrix.boost_choco }}" + } + echo "BOOST_INCLUDEDIR=$($boostDir.FullName)" >> $env:GITHUB_ENV + } + - name: Cache Maven packages + uses: actions/cache@v5 + with: + path: ~/.m2 + key: windows-${{ matrix.name }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + windows-${{ matrix.name }}-m2- + - name: Package client-cpp + shell: bash + env: + CMAKE_GENERATOR: ${{ matrix.cmake_generator }} + PACKAGE_CLASSIFIER: ${{ matrix.package_classifier }} + run: | + set -euxo pipefail + MVN_ARGS=(./mvnw clean package -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests \ + -Dspotless.skip=true \ + "-Dclient.cpp.package.classifier=${PACKAGE_CLASSIFIER}") + if [ -n "${BOOST_INCLUDEDIR:-}" ]; then + MVN_ARGS+=("-Dboost.include.dir=${BOOST_INCLUDEDIR}") + fi + if [ -n "${CMAKE_GENERATOR:-}" ]; then + MVN_ARGS+=("-Dcmake.generator=${CMAKE_GENERATOR}") + fi + "${MVN_ARGS[@]}" + - name: Verify Windows SDK is x64 + # Guard against accidental Win32 generator defaults on legacy VS matrices. + shell: pwsh + run: | + $dll = "iotdb-client/client-cpp/target/install/lib/iotdb_session.dll" + if (-not (Test-Path $dll)) { + throw "Missing $dll" + } + $bytes = [System.IO.File]::ReadAllBytes($dll) + $peOffset = [BitConverter]::ToInt32($bytes, 0x3C) + $machine = [BitConverter]::ToUInt16($bytes, $peOffset + 4) + if ($machine -ne 0x8664) { + throw "Expected PE32+ x64 (machine=0x8664), got 0x$($machine.ToString('X4'))" + } + Write-Host "Verified x64 DLL: $dll" + - name: Resolve package zip + id: pkg + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + packages=(iotdb-client/client-cpp/target/iotdb-session-cpp-*-${{ matrix.package_classifier }}.zip) + if [ "${#packages[@]}" -ne 1 ]; then + echo "Expected exactly one package zip, got: ${packages[*]:-(none)}" + exit 1 + fi + echo "path=${packages[0]}" >> "$GITHUB_OUTPUT" + echo "name=$(basename "${packages[0]}" .zip)" >> "$GITHUB_OUTPUT" + sha512="${packages[0]}.sha512" + if [ ! -f "${sha512}" ]; then + echo "Expected checksum file: ${sha512}" + exit 1 + fi + echo "sha512_path=${sha512}" >> "$GITHUB_OUTPUT" + - name: Upload zip artifact + uses: actions/upload-artifact@v6 + with: + name: ${{ steps.pkg.outputs.name }} + path: | + ${{ steps.pkg.outputs.path }} + ${{ steps.pkg.outputs.sha512_path }} + if-no-files-found: error diff --git a/.github/workflows/multi-language-client.yml b/.github/workflows/multi-language-client.yml index a1fc44af61c95..65629ce12d050 100644 --- a/.github/workflows/multi-language-client.yml +++ b/.github/workflows/multi-language-client.yml @@ -46,41 +46,64 @@ jobs: fail-fast: false max-parallel: 15 matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-22.04, ubuntu-24.04, windows-2022, windows-2025-vs2026, macos-latest] runs-on: ${{ matrix.os}} steps: - uses: actions/checkout@v4 - name: Install CPP Dependencies (Ubuntu) - if: matrix.os == 'ubuntu-latest' + if: runner.os == 'Linux' shell: bash run: | + set -euxo pipefail sudo apt-get update - sudo apt-get install libboost-all-dev + sudo apt-get install -y libboost-all-dev openssl libssl-dev wget + # jammy (22.04): no clang-format-17 in default repos — use apt.llvm.org + . /etc/os-release + if [[ "${VERSION_CODENAME}" == "jammy" ]]; then + wget -qO /tmp/llvm.sh https://apt.llvm.org/llvm.sh + chmod +x /tmp/llvm.sh + sudo DEBIAN_FRONTEND=noninteractive /tmp/llvm.sh 17 + sudo apt-get install -y clang-format-17 + else + sudo apt-get install -y clang-format-17 + fi + sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100 + sudo update-alternatives --set clang-format /usr/bin/clang-format-17 + clang-format --version - name: Install CPP Dependencies (Mac) # remove some xcode to release disk space - if: matrix.os == 'macos-latest' + if: runner.os == 'macOS' shell: bash run: | - brew install boost + brew install boost openssl llvm@17 bison + ln -sf "$(brew --prefix llvm@17)/bin/clang-format" "$(brew --prefix)/bin/clang-format" + echo "$(brew --prefix bison)/bin" >> "$GITHUB_PATH" + echo "$(brew --prefix llvm@17)/bin" >> "$GITHUB_PATH" + clang-format --version + bison --version sudo rm -rf /Applications/Xcode_14.3.1.app sudo rm -rf /Applications/Xcode_15.0.1.app sudo rm -rf /Applications/Xcode_15.1.app sudo rm -rf /Applications/Xcode_15.2.app sudo rm -rf /Applications/Xcode_15.3.app - name: Install CPP Dependencies (Windows) - if: matrix.os == 'windows-latest' + if: runner.os == 'Windows' run: | - choco install winflexbison3 - choco install boost-msvc-14.3 - $boost_path = (Get-ChildItem -Path 'C:\local\' -Filter 'boost_*').FullName - echo $boost_path >> $env:GITHUB_PATH + choco install winflexbison3 -y + choco install llvm --version=17.0.6 --force -y + clang-format --version - name: Cache Maven packages uses: actions/cache@v4 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2- + - name: Check C++ format (Spotless) + shell: bash + run: | + ./mvnw -P with-cpp -pl iotdb-client/client-cpp spotless:check + ./mvnw -P with-cpp -pl example/client-cpp-example spotless:check - name: Build IoTDB server shell: bash run: ./mvnw clean install -pl distribution -am -DskipTests @@ -89,7 +112,14 @@ jobs: # Explicitly using mvnw here as the build requires maven 3.9 and the default installation is older # Explicitly using "install" instead of package in order to be sure we're using libs built on this machine # (was causing problems on windows, but could cause problem on linux, when updating the thrift module) - run: ./mvnw clean verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am + run: | + if [[ "${{ matrix.os }}" == "windows-2025-vs2026" ]]; then + ./mvnw verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am -Dcmake.generator="Visual Studio 18 2026" -Dbuild.tests=OFF -Dmaven.test.skip=true + elif [[ "${{ runner.os }}" == "Windows" ]]; then + ./mvnw verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am -Dbuild.tests=OFF -Dmaven.test.skip=true + else + ./mvnw verify -P with-cpp -pl iotdb-client/client-cpp,example/client-cpp-example -am + fi - name: Upload Artifact if: failure() uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index 71e0fc85edb50..21e32c3fcb61e 100644 --- a/.gitignore +++ b/.gitignore @@ -40,8 +40,8 @@ tsfile-jdbc/src/main/resources/output/queryRes.csv *.txt !iotdb-client/client-py/requirements.txt !iotdb-client/client-py/requirements_dev.txt -!iotdb-client/client-cpp/src/main/CMakeLists.txt -!iotdb-client/client-cpp/src/test/CMakeLists.txt +!iotdb-client/client-cpp/CMakeLists.txt +!iotdb-client/client-cpp/test/CMakeLists.txt !example/rest-client-c-example/CMakeLists.txt !example/client-cpp-example/src/CMakeLists.txt @@ -88,6 +88,11 @@ tsfile-jdbc/src/main/resources/output/queryRes.csv *.tsfile tsfile/src/test/resources/*.ts +### clion project +**/cmake-build-debug/ +**/cmake-build-release/ +iotdb-client/client-cpp/build-*/ + ### Apache release ### local-snapshots-dir/ venv/ @@ -107,6 +112,7 @@ classes/ ### Cmake files ### *.cmake +!iotdb-client/client-cpp/cmake/*.cmake Makefile **/CMakeFiles/ diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index 38b4fe1781a88..ac504fa7b3a22 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -15,5 +15,5 @@ # specific language governing permissions and limitations # under the License. -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.6/apache-maven-3.9.6-bin.zip +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.12/apache-maven-3.9.12-bin.zip wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.3.2/maven-wrapper-3.3.2.jar diff --git a/LICENSE b/LICENSE index 3f3de45e81a7c..0a1e5e42a9ec1 100644 --- a/LICENSE +++ b/LICENSE @@ -303,3 +303,15 @@ The following files include code modified from Dropwizard Metrics project. Copyright (c) 2010-2013 Coda Hale, Yammer.com, 2014-2021 Dropwizard Team Project page: https://github.com/dropwizard/metrics License: https://github.com/dropwizard/metrics/blob/release/4.2.x/LICENSE + +-------------------------------------------------------------------------------- + +The following files include code modified from LMax Disruptor project. + +./iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/* + +LMax Disruptor is open source software licensed under the Apache License 2.0 and supported by the Apache Software Foundation. +Project page: https://github.com/LMAX-Exchange/disruptor +License: https://github.com/LMAX-Exchange/disruptor/blob/master/LICENCE.txt + +-------------------------------------------------------------------------------- diff --git a/LICENSE-binary b/LICENSE-binary index f3fadab3af857..e64118ab4394f 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -241,17 +241,17 @@ io.dropwizard.metrics:metrics-core:4.2.19 io.dropwizard.metrics:metrics-jvm:3.2.2 com.librato.metrics:metrics-librato:5.1.0 com.github.moquette-io.moquette:moquette-broker:0.18 -io.netty:netty-buffer:4.1.126.Final -io.netty:netty-codec:4.1.126.Final -io.netty:netty-codec-http:4.1.126.Final -io.netty:netty-codec-mqtt:4.1.126.Final -io.netty:netty-common:4.1.126.Final -io.netty:netty-handler:4.1.126.Final -io.netty:netty-resolver:4.1.126.Final -io.netty:netty-transport:4.1.126.Final -io.netty:netty-transport-native-epoll:4.1.126.Final:linux-aarch_64 -io.netty:netty-transport-native-epoll:4.1.126.Final:linux-x86_64 -io.netty:netty-transport-native-unix-common:4.1.126.Final +io.netty:netty-buffer:4.1.134.Final +io.netty:netty-codec:4.1.134.Final +io.netty:netty-codec-http:4.1.134.Final +io.netty:netty-codec-mqtt:4.1.134.Final +io.netty:netty-common:4.1.134.Final +io.netty:netty-handler:4.1.134.Final +io.netty:netty-resolver:4.1.134.Final +io.netty:netty-transport:4.1.134.Final +io.netty:netty-transport-native-epoll:4.1.134.Final:linux-aarch_64 +io.netty:netty-transport-native-epoll:4.1.134.Final:linux-x86_64 +io.netty:netty-transport-native-unix-common:4.1.134.Final org.osgi:org.osgi.core:7.0.0 org.osgi:osgi.cmpn:7.0.0 org.ops4j.pax.jdbc:pax-jdbc-common:1.5.6 diff --git a/distribution/pom.xml b/distribution/pom.xml index 138cea976408c..1aa47d8122077 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -29,6 +29,9 @@ iotdb-distribution pom IoTDB: Distribution + + ${os.classifier} + org.apache.iotdb @@ -117,6 +120,67 @@ + + .os-unix + + + Linux + unix + !aarch64 + + + + linux-x86_64-glibc2.28 + + + + .os-unix-arm + + + Linux + unix + aarch64 + + + + linux-aarch64-glibc2.28 + + + + .os-mac + + + mac + !aarch64 + + + + macos-x86_64 + + + + .os-mac-arm + + + mac + aarch64 + + + + macos-aarch64 + + + + .os-windows + + + windows + + + + windows-x86_64-msvc14.3 + + with-cpp @@ -157,7 +221,7 @@ - apache-iotdb-${project.version}-client-cpp-${os.classifier}-bin.zip + apache-iotdb-${project.version}-client-cpp-${client.cpp.package.classifier}-bin.zip diff --git a/example/client-cpp-example/README.md b/example/client-cpp-example/README.md index f36a084f888e5..5ff964595fda7 100644 --- a/example/client-cpp-example/README.md +++ b/example/client-cpp-example/README.md @@ -19,30 +19,240 @@ --> -# How to get a complete CPP client demo project +# IoTDB C++ client examples -## Get a project +[中文说明](README_zh.md) -Using maven to build this example project: +Sample programs that link against the pre-built **IoTDB C++ Session SDK** +(`iotdb_session`). Thrift and Boost are **not** required at application compile +time; they are embedded inside the SDK shared library. -* cd the root path of the whole project -* run `mvn clean package -DskipTests -P with-cpp -pl example/client-cpp-example -am` -* cd example/client-cpp-example/target +All examples connect to a running IoTDB instance (default `127.0.0.1:6667`, +user `root` / `root`). + +| Example | Description | +|---------|-------------| +| `SessionExample` | Tree model: DDL, insert, query, delete | +| `AlignedTimeseriesSessionExample` | Aligned time series and templates | +| `MultiSvrNodeClient` | Multi-node insert/query loop | +| `tree_example` | C Session API (tree model) | + +## Which SDK zip to use + +Release CI ([client-cpp-package.yml](../../.github/workflows/client-cpp-package.yml)) +publishes one zip per platform/toolchain: +`client-cpp--.zip` (zip root contains `include/` and `lib/`). + +| Deployment target | Classifier suffix | +|-------------------|-------------------| +| Linux x86_64, glibc ≥ 2.24, CXX11 ABI (**recommended**) | `linux-x86_64-glibc224` | +| Linux aarch64, glibc ≥ 2.24, CXX11 ABI (**recommended**) | `linux-aarch64-glibc224` | +| Linux x86_64, glibc ≥ 2.17, legacy ABI | `linux-x86_64-glibc217` | +| Linux aarch64, glibc ≥ 2.17, legacy ABI | `linux-aarch64-glibc217` | +| macOS x86_64 | `mac-x86_64` | +| macOS arm64 | `mac-aarch64` | +| Windows (match your Visual Studio version) | `windows-x86_64-vs2017` … `vs2026` | + +The current build compiles Thrift 0.21 from source at CMake configure time. +Legacy `-Diotdb-tools-thrift.version=...` flags applied to the **old** +pre-built Thrift workflow only. On Linux, prefer **`glibc224`** when your host has +glibc ≥ 2.24 and you use the default `g++`. Use **`glibc217`** only for glibc 2.17 +systems or legacy ABI; on Ubuntu 22/24 you may need `-D_GLIBCXX_USE_CXX11_ABI=0` +when linking against `glibc217`. See [client-cpp README](../../iotdb-client/client-cpp/README.md). + +## SDK layout (after unpack) + +The SDK zip produced by `client-cpp` contains **public headers only** and one +shared library: + +``` +client/ +├── include/ +│ ├── Session.h +│ ├── Export.h +│ └── ... (17 public headers; no thrift/ or boost/) +└── lib/ + ├── iotdb_session.dll + iotdb_session.lib (Windows) + ├── libiotdb_session.so (Linux) + └── libiotdb_session.dylib (macOS) +``` + +## Build the examples + +### Option A – Maven (recommended in this repo) + +From the repository root: + +```bash +mvn clean package -DskipTests -P with-cpp -pl example/client-cpp-example -am +``` + +Maven unpacks the SDK zip into `example/client-cpp-example/target/client/` and +runs CMake in `target/`. Binaries are under `target/` (exact path depends on +the generator; on Windows with Visual Studio: `target/Release/`). + +### Option B – CMake only (manual SDK) + +1. Build or download the SDK and unpack it so `client/include` and + `client/lib` exist (see layout above). +2. Copy `src/*.{cpp,c}` and `src/CMakeLists.txt` into one directory (or use + `src/` as the source tree and place `client/` beside it). +3. Configure and build: + +```bash +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build +``` + +Windows (Visual Studio generator): + +```powershell +cmake -S . -B build -A x64 +cmake --build build --config Release +``` + +Each executable is built with the IoTDB runtime library copied **next to the +`.exe` / binary** (POST_BUILD step). Linux/macOS binaries use `$ORIGIN` rpath +so they resolve the `.so` / `.dylib` in the same directory. + +Optional staging folder for deployment: + +```bash +cmake --build build --target example-dist +# -> build/dist/ contains all example binaries + libiotdb_session.{so,dll,dylib} +``` + +## Run on a clean machine (no compiler, no IoTDB SDK headers) + +You only need: + +1. A running IoTDB server reachable from the machine. +2. The **example executable(s)** and the **IoTDB runtime library** in the + **same directory** (or on the system library path). + +Copy either from `build/.../Release/` (Windows) / `build/` (Ninja/Make) or from +`build/dist/` after `example-dist`. + +### Windows + +**Files to copy** + +``` +SessionExample.exe +iotdb_session.dll +``` + +(Repeat for the other example names if needed.) + +**Prerequisites on the target PC** + +- **64-bit Windows** (examples are built x64). +- **[Microsoft Visual C++ Redistributable for Visual Studio 2015–2022](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist)** + (x64). The SDK and examples are built with **`/MD`**; the redistributable + supplies `vcruntime140.dll`, `msvcp140.dll`, etc. + Installing this package is enough—you do **not** need Visual Studio or the + IoTDB SDK on the target machine. + +**Run** + +```powershell +.\SessionExample.exe +``` + +If you see “The code execution cannot proceed because VCRUNRuntime140.dll was +missing”, install the VC++ redistributable above. + +You do **not** need a separate Thrift or Boost runtime; they are inside +`iotdb_session.dll`. + +### Linux + +**Files to copy** -You can find some files to form a complete project: ``` -+-- client -| +-- include -| +-- Session.h -| +-- IClientRPCService.h -| +-- rpc_types.h -| +-- rpc_constants.h -| +-- thrift -| +-- thrift_headers... -| +-- lib -| +-- libiotdb_session.dylib -+-- CMakeLists.txt -+-- SessionExample.cpp +SessionExample +libiotdb_session.so +chmod +x SessionExample ``` +**Prerequisites on the target machine** + +- **glibc** on the target must be **≥ the glibc version on the machine that + built the SDK** (backward compatible only in that direction). + +Check **build machine** (record in release notes): + +```bash +ldd --version | head -1 +# e.g. ldd (Ubuntu GLIBC 2.35-0ubuntu3) 2.35 +``` + +Check **target machine**: + +```bash +ldd --version | head -1 +# must be >= build glibc (same major.minor or newer) +``` + +See which `GLIBC_` symbols the binary needs: + +```bash +objdump -T SessionExample | grep GLIBC_ | sed 's/.*GLIBC_/GLIBC_/' | sort -Vu | tail -5 +objdump -T libiotdb_session.so | grep GLIBC_ | sed 's/.*GLIBC_/GLIBC_/' | sort -Vu | tail -5 +``` + +If the target glibc is too old, you'll get errors like +`version 'GLIBC_2.34' not found` at runtime. Rebuild the SDK on an older distro +(or in an older container) to widen compatibility. + +**Run** (with `.so` beside the binary): + +```bash +./SessionExample +``` + +If the shared library is not found: + +```bash +export LD_LIBRARY_PATH=. +./SessionExample +``` + +No separate Thrift install is required. + +### macOS + +Copy the example binary and `libiotdb_session.dylib` together. The target macOS +version should be **≥ the deployment target used to build the SDK**. Check with: + +```bash +otool -L SessionExample +``` + +## Development notes + +- **Windows**: Application and SDK both use **`/MD`** (dynamic CRT). This + matches a default Visual Studio project; link `iotdb_session.lib`, ship + `iotdb_session.dll`. +- **Linux**: SDK is `libiotdb_session.so`; link it directly. Prefer shipping + the `.so` next to your binary or setting `RPATH` to `$ORIGIN`. +- Examples assume IoTDB is listening on `127.0.0.1:6667`; change host/port in + the source if needed. + +## Project layout in this module + +``` +client-cpp-example/ +├── pom.xml # Maven: unpack SDK + invoke CMake +├── README.md +├── README_zh.md +└── src/ + ├── CMakeLists.txt + ├── SessionExample.cpp + ├── AlignedTimeseriesSessionExample.cpp + ├── MultiSvrNodeClient.cpp + └── tree_example.c +``` +After `mvn package`, the runnable tree is under `target/` (sources, `client/`, +and CMake build output). diff --git a/example/client-cpp-example/README_zh.md b/example/client-cpp-example/README_zh.md new file mode 100644 index 0000000000000..84df19792a118 --- /dev/null +++ b/example/client-cpp-example/README_zh.md @@ -0,0 +1,252 @@ + + +# IoTDB C++ 客户端示例 + +[English](README.md) + +本目录提供链接 **IoTDB C++ Session SDK**（`iotdb_session`）的示例程序。 +应用侧编译 **不需要** 单独安装 Thrift 或 Boost 头文件/库，它们已封装在 SDK +共享库内部。 + +所有示例默认连接本机 IoTDB（`127.0.0.1:6667`，用户 `root` / 密码 `root`）。 + +| 示例 | 说明 | +|------|------| +| `SessionExample` | 树模型：建库建序列、写入、查询、删除 | +| `AlignedTimeseriesSessionExample` | 对齐时间序列与模板 | +| `MultiSvrNodeClient` | 多节点写入/查询循环 | +| `tree_example` | C Session API（树模型） | + +## 选择哪个 SDK 压缩包 + +CI 发版（[client-cpp-package.yml](../../.github/workflows/client-cpp-package.yml)） +会按平台/工具链打出多份 zip，文件名形如 +`client-cpp--.zip`（解压后根目录即为 `include/` 与 `lib/`）。请按目标环境选择： + +| 目标环境 | classifier 后缀 | +|----------|-----------------| +| Linux x86_64，glibc ≥ 2.24，CXX11 ABI（**推荐**） | `linux-x86_64-glibc224` | +| Linux aarch64，glibc ≥ 2.24，CXX11 ABI（**推荐**） | `linux-aarch64-glibc224` | +| Linux x86_64，glibc ≥ 2.17，旧 libstdc++ ABI | `linux-x86_64-glibc217` | +| Linux aarch64，glibc ≥ 2.17，旧 libstdc++ ABI | `linux-aarch64-glibc217` | +| macOS x86_64 | `mac-x86_64` | +| macOS arm64 | `mac-aarch64` | +| Windows + 与工程相同的 VS 版本 | `windows-x86_64-vs2017` … `vs2026` | + +当前 CMake 构建在配置阶段从源码编译 Thrift 0.21，**不再**通过 +`-Diotdb-tools-thrift.version=0.14.1.1-gcc4-SNAPSHOT` 等旧参数控制 glibc； +Linux 上若部署机 glibc ≥ 2.24 且使用系统默认 `g++`，请优先选用 **`glibc224`** +包。仅当目标机停留在 glibc 2.17（如 CentOS 7）或必须与旧 libstdc++ ABI 一致时， +选用 **`glibc217`** 包；在 Ubuntu 22/24 上链 `glibc217` 时常需 +`-D_GLIBCXX_USE_CXX11_ABI=0`。详见 [client-cpp README](../../iotdb-client/client-cpp/README.md)。 + +## SDK 目录结构（解压后） + +`client-cpp` 打出的 SDK 压缩包只包含 **公开头文件** 和 **一个共享库**： + +``` +client/ +├── include/ +│ ├── Session.h +│ ├── Export.h +│ └── ... （17 个公开头；无 thrift/、boost/） +└── lib/ + ├── iotdb_session.dll + iotdb_session.lib （Windows） + ├── libiotdb_session.so （Linux） + └── libiotdb_session.dylib （macOS） +``` + +## 编译示例 + +### 方式 A：Maven（本仓库推荐） + +在仓库根目录执行： + +```bash +mvn clean package -DskipTests -P with-cpp -pl example/client-cpp-example -am +``` + +Maven 会将 SDK 解压到 `example/client-cpp-example/target/client/`，并在 +`target/` 下调用 CMake。可执行文件位于 `target/`（具体路径取决于生成器； +Windows + Visual Studio 一般为 `target/Release/`）。 + +### 方式 B：仅 CMake（手动准备 SDK） + +1. 自行编译或下载 SDK，解压后保证存在 `client/include` 与 `client/lib`（见 + 上文目录结构）。 +2. 将 `src/*.{cpp,c}` 与 `src/CMakeLists.txt` 放在同一目录（或保留 `src/` + 结构，并在同级放置 `client/`）。 +3. 配置并编译： + +```bash +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build +``` + +Windows（Visual Studio 生成器）： + +```powershell +cmake -S . -B build -A x64 +cmake --build build --config Release +``` + +编译完成后，IoTDB 运行时库会通过 POST_BUILD 自动复制到 **与可执行文件相同 +的目录**。Linux/macOS 可执行文件还设置了 `$ORIGIN` rpath，可在同目录加载 +`.so` / `.dylib`。 + +可选：打部署包目录 + +```bash +cmake --build build --target example-dist +# 生成 build/dist/，内含全部示例二进制 + libiotdb_session.{so,dll,dylib} +``` + +## 在「干净机器」上运行（无需编译器、无需 SDK 头文件） + +目标机器只需： + +1. 可访问的 **IoTDB 服务**（已启动）。 +2. **示例可执行文件** 与 **IoTDB 运行时库** 放在 **同一目录**（或位于系统 + 库搜索路径中）。 + +可从 `build/.../Release/`（Windows）或 `build/`（Ninja/Make）复制，也可在 +执行 `example-dist` 后直接使用 `build/dist/`。 + +### Windows + +**需要拷贝的文件** + +``` +SessionExample.exe +iotdb_session.dll +``` + +（其他示例同理，可执行文件与 `iotdb_session.dll` 成对拷贝。） + +**目标机器前置条件** + +- **64 位 Windows**（示例为 x64 构建）。 +- 安装 **[Microsoft Visual C++ 2015–2022 可再发行组件包（x64）](https://learn.microsoft.com/zh-cn/cpp/windows/latest-supported-vc-redist)**。 + SDK 与示例均使用 **`/MD`**（动态 CRT），该安装包提供 `vcruntime140.dll`、 + `msvcp140.dll` 等运行时。 + **仅安装此 Redistributable 即可**，目标机 **不需要** Visual Studio，也 + **不需要** IoTDB SDK 头文件或 Thrift/Boost。 + +**运行** + +```powershell +.\SessionExample.exe +``` + +若提示缺少 `VCRUNTIME140.dll`，请安装上述 VC++ 可再发行包。 + +Thrift、Boost 已包含在 `iotdb_session.dll` 内，无需单独部署。 + +### Linux + +**需要拷贝的文件** + +``` +SessionExample +libiotdb_session.so +chmod +x SessionExample +``` + +**目标机器前置条件** + +- 目标机的 **glibc 版本必须 ≥ 编译 SDK 时的 glibc 版本**（仅向后兼容： + 新系统可跑旧库要求，旧系统不能跑需要更高 glibc 的二进制）。 + +在 **编译机** 上记录版本（建议写入发布说明）： + +```bash +ldd --version | head -1 +# 例如：ldd (Ubuntu GLIBC 2.35-0ubuntu3) 2.35 +``` + +在 **目标机** 上检查： + +```bash +ldd --version | head -1 +# 版本号应 >= 编译机（同主版本次版本或更新） +``` + +查看二进制依赖的最高 `GLIBC_` 符号： + +```bash +objdump -T SessionExample | grep GLIBC_ | sed 's/.*GLIBC_/GLIBC_/' | sort -Vu | tail -5 +objdump -T libiotdb_session.so | grep GLIBC_ | sed 's/.*GLIBC_/GLIBC_/' | sort -Vu | tail -5 +``` + +若目标 glibc 过旧，运行时会报错，例如 +`version 'GLIBC_2.34' not found`。可在更旧的发行版（或旧版容器）上重新编译 +SDK，以扩大兼容范围。 + +**运行**（`.so` 与可执行文件同目录）： + +```bash +./SessionExample +``` + +若找不到共享库： + +```bash +export LD_LIBRARY_PATH=. +./SessionExample +``` + +无需单独安装 Thrift。 + +### macOS + +将示例可执行文件与 `libiotdb_session.dylib` 放在同一目录。目标 macOS 版本 +应 **≥ 编译 SDK 时设置的 deployment target**。可用以下命令检查依赖： + +```bash +otool -L SessionExample +``` + +## 开发说明 + +- **Windows**：应用与 SDK 均使用 **`/MD`**，与 Visual Studio 默认工程一致； + 链接 `iotdb_session.lib`，部署时携带 `iotdb_session.dll`。 +- **Linux**：直接链接 `libiotdb_session.so`；建议与可执行文件同目录发布，或 + 设置 `RPATH=$ORIGIN`。 +- 示例默认连接 `127.0.0.1:6667`；如需修改地址/端口，请编辑对应源码。 + +## 本模块目录结构 + +``` +client-cpp-example/ +├── pom.xml # Maven：解压 SDK + 调用 CMake +├── README.md # 英文说明 +├── README_zh.md # 中文说明（本文件） +└── src/ + ├── CMakeLists.txt + ├── SessionExample.cpp + ├── AlignedTimeseriesSessionExample.cpp + ├── MultiSvrNodeClient.cpp + └── tree_example.c +``` + +执行 `mvn package` 后，可在 `target/` 下找到源码、`client/` SDK 与 CMake +构建产物。 diff --git a/example/client-cpp-example/pom.xml b/example/client-cpp-example/pom.xml index cf6f65775df73..c6ac16dbdf09f 100644 --- a/example/client-cpp-example/pom.xml +++ b/example/client-cpp-example/pom.xml @@ -28,6 +28,9 @@ client-cpp-example IoTDB: Example: CPP Client + + ${os.classifier} + org.apache.iotdb @@ -60,6 +63,14 @@ ${project.basedir}/src/AlignedTimeseriesSessionExample.cpp ${project.build.directory}/AlignedTimeseriesSessionExample.cpp + + ${project.basedir}/src/MultiSvrNodeClient.cpp + ${project.build.directory}/MultiSvrNodeClient.cpp + + + ${project.basedir}/src/tree_example.c + ${project.build.directory}/tree_example.c + ${project.basedir}/src/CMakeLists.txt ${project.build.directory}/CMakeLists.txt @@ -90,37 +101,17 @@ client-cpp ${project.version} zip - cpp-${os.classifier} + ${client.cpp.package.classifier} true ${project.build.directory}/client - - unpack-thrift - - unpack - - generate-sources - - - - org.apache.iotdb.tools - iotdb-tools-thrift - ${iotdb-tools-thrift.version} - ${os.classifier} - zip - true - ${project.build.directory}/thrift - - - - - com.googlecode.cmake-maven-project + io.github.cmake-maven-plugin cmake-maven-plugin @@ -132,10 +123,7 @@ ${cmake.generator} ${project.build.directory} - ${project.build.directory} - - - + ${project.build.directory} @@ -154,4 +142,100 @@ + + + .os-unix + + + Linux + unix + !aarch64 + + + + linux-x86_64-glibc2.28 + + + + .os-unix-arm + + + Linux + unix + aarch64 + + + + linux-aarch64-glibc2.28 + + + + .os-mac + + + mac + !aarch64 + + + + macos-x86_64 + + + + .os-mac-arm + + + mac + aarch64 + + + + macos-aarch64 + + + + .os-windows + + + windows + + + + windows-x86_64-msvc14.3 + + + + spotless-cpp + + [11,) + + + + + com.diffplug.spotless + spotless-maven-plugin + 2.44.5 + + + + src/**/*.h + src/**/*.hpp + src/**/*.c + src/**/*.cc + src/**/*.cpp + + + src/test/catch2/** + + + ${clang.format.version} + + + + + + + + + diff --git a/example/client-cpp-example/src/AlignedTimeseriesSessionExample.cpp b/example/client-cpp-example/src/AlignedTimeseriesSessionExample.cpp index 8f175cb7650c6..c8e00efe68e8b 100644 --- a/example/client-cpp-example/src/AlignedTimeseriesSessionExample.cpp +++ b/example/client-cpp-example/src/AlignedTimeseriesSessionExample.cpp @@ -26,388 +26,403 @@ Session *session; #define DEFAULT_ROW_NUMBER 1000000 void createAlignedTimeseries() { - string alignedDeviceId = "root.sg1.d1"; - vector measurements = {"s1", "s2", "s3"}; - vector alignedTimeseries = {"root.sg1.d1.s1", "root.sg1.d1.s2", "root.sg1.d1.s3"}; - vector dataTypes = {TSDataType::INT32, TSDataType::DOUBLE, TSDataType::BOOLEAN}; - vector encodings = {TSEncoding::PLAIN, TSEncoding::GORILLA, TSEncoding::RLE}; - vector compressors = { - CompressionType::SNAPPY, CompressionType::UNCOMPRESSED, CompressionType::SNAPPY}; - for (const string ×eries: alignedTimeseries) { - if (session->checkTimeseriesExists(timeseries)) { - session->deleteTimeseries(timeseries); - } + string alignedDeviceId = "root.sg1.d1"; + vector measurements = {"s1", "s2", "s3"}; + vector alignedTimeseries = {"root.sg1.d1.s1", "root.sg1.d1.s2", + "root.sg1.d1.s3"}; + vector dataTypes = { + TSDataType::INT32, TSDataType::DOUBLE, TSDataType::BOOLEAN}; + vector encodings = { + TSEncoding::PLAIN, TSEncoding::GORILLA, TSEncoding::RLE}; + vector compressors = { + CompressionType::SNAPPY, CompressionType::UNCOMPRESSED, + CompressionType::SNAPPY}; + for (const string ×eries : alignedTimeseries) { + if (session->checkTimeseriesExists(timeseries)) { + session->deleteTimeseries(timeseries); } - session->createAlignedTimeseries(alignedDeviceId, measurements, dataTypes, encodings, compressors); + } + session->createAlignedTimeseries(alignedDeviceId, measurements, dataTypes, + encodings, compressors); } void createSchemaTemplate() { - if (!session->checkTemplateExists("template1")) { - Template temp("template1", false); - - InternalNode iNodeD99("d99", true); - - MeasurementNode mNodeS1("s1", TSDataType::INT32, TSEncoding::RLE, CompressionType::SNAPPY); - MeasurementNode mNodeS2("s2", TSDataType::INT64, TSEncoding::RLE, CompressionType::SNAPPY); - MeasurementNode mNodeD99S1("s1", TSDataType::DOUBLE, TSEncoding::RLE, CompressionType::SNAPPY); - MeasurementNode mNodeD99S2("s2", TSDataType::BOOLEAN, TSEncoding::RLE, CompressionType::SNAPPY); - - iNodeD99.addChild(mNodeD99S1); - iNodeD99.addChild(mNodeD99S2); - - temp.addToTemplate(iNodeD99); - temp.addToTemplate(mNodeS1); - temp.addToTemplate(mNodeS2); - - session->createSchemaTemplate(temp); - session->setSchemaTemplate("template1", "root.sg3.d1"); - } + if (!session->checkTemplateExists("template1")) { + Template temp("template1", false); + + InternalNode iNodeD99("d99", true); + + MeasurementNode mNodeS1("s1", TSDataType::INT32, TSEncoding::RLE, + CompressionType::SNAPPY); + MeasurementNode mNodeS2("s2", TSDataType::INT64, TSEncoding::RLE, + CompressionType::SNAPPY); + MeasurementNode mNodeD99S1("s1", TSDataType::DOUBLE, TSEncoding::RLE, + CompressionType::SNAPPY); + MeasurementNode mNodeD99S2("s2", TSDataType::BOOLEAN, TSEncoding::RLE, + CompressionType::SNAPPY); + + iNodeD99.addChild(mNodeD99S1); + iNodeD99.addChild(mNodeD99S2); + + temp.addToTemplate(iNodeD99); + temp.addToTemplate(mNodeS1); + temp.addToTemplate(mNodeS2); + + session->createSchemaTemplate(temp); + session->setSchemaTemplate("template1", "root.sg3.d1"); + } } void ActivateTemplate() { - session->executeNonQueryStatement("insert into root.sg3.d1(timestamp,s1, s2) values(200, 1, 1);"); + session->executeNonQueryStatement( + "insert into root.sg3.d1(timestamp,s1, s2) values(200, 1, 1);"); } void showDevices() { - unique_ptr dataSet = session->executeQueryStatement("show devices with database"); - for (const string &name: dataSet->getColumnNames()) { - cout << name << " "; - } - cout << endl; - - dataSet->setFetchSize(1024); - while (dataSet->hasNext()) { - cout << dataSet->next()->toString(); - } - cout << endl; - - dataSet->closeOperationHandle(); + unique_ptr dataSet = + session->executeQueryStatement("show devices with database"); + for (const string &name : dataSet->getColumnNames()) { + cout << name << " "; + } + cout << endl; + + dataSet->setFetchSize(1024); + while (dataSet->hasNext()) { + cout << dataSet->next()->toString(); + } + cout << endl; + + dataSet->closeOperationHandle(); } void showTimeseries() { - unique_ptr dataSet = session->executeQueryStatement("show timeseries"); - for (const string &name: dataSet->getColumnNames()) { - cout << name << " "; - } - cout << endl; - - dataSet->setFetchSize(1024); - while (dataSet->hasNext()) { - cout << dataSet->next()->toString(); - } - cout << endl; - - dataSet->closeOperationHandle(); + unique_ptr dataSet = + session->executeQueryStatement("show timeseries"); + for (const string &name : dataSet->getColumnNames()) { + cout << name << " "; + } + cout << endl; + + dataSet->setFetchSize(1024); + while (dataSet->hasNext()) { + cout << dataSet->next()->toString(); + } + cout << endl; + + dataSet->closeOperationHandle(); } void insertAlignedRecord() { - string deviceId = "root.sg1.d1"; - vector measurements; - measurements.emplace_back("s1"); - measurements.emplace_back("s2"); - measurements.emplace_back("s3"); - - for (int64_t time = 0; time < 10; time++) { - vector values; - values.emplace_back("1"); - values.emplace_back("1.0"); - values.emplace_back("true"); - session->insertAlignedRecord(deviceId, time, measurements, values); - } + string deviceId = "root.sg1.d1"; + vector measurements; + measurements.emplace_back("s1"); + measurements.emplace_back("s2"); + measurements.emplace_back("s3"); + + for (int64_t time = 0; time < 10; time++) { + vector values; + values.emplace_back("1"); + values.emplace_back("1.0"); + values.emplace_back("true"); + session->insertAlignedRecord(deviceId, time, measurements, values); + } } void insertAlignedRecords() { - string deviceId = "root.sg1.d1"; - vector measurements; - measurements.emplace_back("s1"); - measurements.emplace_back("s2"); - measurements.emplace_back("s3"); - - vector deviceIds; - vector> measurementsList; - vector> valuesList; - vector timestamps; - - for (int64_t time = 10; time < 20; time++) { - vector values; - values.emplace_back("1"); - values.emplace_back("1.0"); - values.emplace_back("true"); - - deviceIds.push_back(deviceId); - measurementsList.push_back(measurements); - valuesList.push_back(values); - timestamps.push_back(time); - if (time != 10 && time % 10 == 0) { - session->insertAlignedRecords(deviceIds, timestamps, measurementsList, valuesList); - deviceIds.clear(); - measurementsList.clear(); - valuesList.clear(); - timestamps.clear(); - } + string deviceId = "root.sg1.d1"; + vector measurements; + measurements.emplace_back("s1"); + measurements.emplace_back("s2"); + measurements.emplace_back("s3"); + + vector deviceIds; + vector> measurementsList; + vector> valuesList; + vector timestamps; + + for (int64_t time = 10; time < 20; time++) { + vector values; + values.emplace_back("1"); + values.emplace_back("1.0"); + values.emplace_back("true"); + + deviceIds.push_back(deviceId); + measurementsList.push_back(measurements); + valuesList.push_back(values); + timestamps.push_back(time); + if (time != 10 && time % 10 == 0) { + session->insertAlignedRecords(deviceIds, timestamps, measurementsList, + valuesList); + deviceIds.clear(); + measurementsList.clear(); + valuesList.clear(); + timestamps.clear(); } + } - session->insertAlignedRecords(deviceIds, timestamps, measurementsList, valuesList); + session->insertAlignedRecords(deviceIds, timestamps, measurementsList, + valuesList); } void insertAlignedTablet() { - pair pairA("s1", TSDataType::INT32); - pair pairB("s2", TSDataType::DOUBLE); - pair pairC("s3", TSDataType::DOUBLE); - vector> schemas; - schemas.push_back(pairA); - schemas.push_back(pairB); - schemas.push_back(pairC); - - Tablet tablet("root.sg2.d2", schemas, 100000); - tablet.setAligned(true); - - for (int64_t time = 0; time < DEFAULT_ROW_NUMBER; time++) { - size_t row = tablet.rowSize++; - tablet.timestamps[row] = time; - int randVal1 = 123456; - double randVal2 = 123456.1234; - double randVal3 = 123456.1234; - tablet.addValue(0, row, &randVal1); - tablet.addValue(1, row, &randVal2); - tablet.addValue(2, row, &randVal3); - if (tablet.rowSize == tablet.maxRowNumber) { - session->insertTablet(tablet, true); - tablet.reset(); - } + pair pairA("s1", TSDataType::INT32); + pair pairB("s2", TSDataType::DOUBLE); + pair pairC("s3", TSDataType::DOUBLE); + vector> schemas; + schemas.push_back(pairA); + schemas.push_back(pairB); + schemas.push_back(pairC); + + Tablet tablet("root.sg2.d2", schemas, 100000); + tablet.setAligned(true); + + for (int64_t time = 0; time < DEFAULT_ROW_NUMBER; time++) { + size_t row = tablet.rowSize++; + tablet.timestamps[row] = time; + int randVal1 = 123456; + double randVal2 = 123456.1234; + double randVal3 = 123456.1234; + tablet.addValue(0, row, randVal1); + tablet.addValue(1, row, randVal2); + tablet.addValue(2, row, randVal3); + if (tablet.rowSize == tablet.maxRowNumber) { + session->insertTablet(tablet, true); + tablet.reset(); } + } - if (tablet.rowSize != 0) { - session->insertTablet(tablet); - tablet.reset(); - } + if (tablet.rowSize != 0) { + session->insertTablet(tablet); + tablet.reset(); + } } void insertAlignedTablets() { - pair pairA("s1", TSDataType::INT32); - pair pairB("s2", TSDataType::DOUBLE); - pair pairC("s3", TSDataType::BOOLEAN); - vector> schemas; - schemas.push_back(pairA); - schemas.push_back(pairB); - schemas.push_back(pairC); - - Tablet tablet1("root.sg1.d1", schemas, 100); - Tablet tablet2("root.sg1.d2", schemas, 100); - Tablet tablet3("root.sg1.d3", schemas, 100); - - unordered_map tabletMap; - tabletMap["root.sg1.d1"] = &tablet1; - tabletMap["root.sg1.d2"] = &tablet2; - tabletMap["root.sg1.d3"] = &tablet3; - - for (int64_t time = 0; time < 20; time++) { - size_t row1 = tablet1.rowSize++; - size_t row2 = tablet2.rowSize++; - size_t row3 = tablet3.rowSize++; - tablet1.timestamps[row1] = time; - tablet2.timestamps[row2] = time; - tablet3.timestamps[row3] = time; - - int randVal11 = rand(); - int randVal12 = rand(); - int randVal13 = rand(); - tablet1.addValue(0, row1, &randVal11); - tablet2.addValue(0, row2, &randVal12); - tablet3.addValue(0, row3, &randVal13); - - double randVal21 = rand() / 99.9; - double randVal22 = rand() / 99.9; - double randVal23 = rand() / 99.9; - tablet1.addValue(1, row1, &randVal21); - tablet2.addValue(1, row2, &randVal22); - tablet3.addValue(1, row3, &randVal23); - - bool randVal31 = (bool)(rand() % 2); - bool randVal32 = (bool)(rand() % 2); - bool randVal33 = (bool)(rand() % 2); - tablet1.addValue(2, row1, &randVal31); - tablet2.addValue(2, row2, &randVal32); - tablet3.addValue(2, row3, &randVal33); - - if (tablet1.rowSize == tablet1.maxRowNumber) { - session->insertAlignedTablets(tabletMap, true); - - tablet1.reset(); - tablet2.reset(); - tablet3.reset(); - } - } - - if (tablet1.rowSize != 0) { - session->insertAlignedTablets(tabletMap, true); - tablet1.reset(); - tablet2.reset(); - tablet3.reset(); + pair pairA("s1", TSDataType::INT32); + pair pairB("s2", TSDataType::DOUBLE); + pair pairC("s3", TSDataType::BOOLEAN); + vector> schemas; + schemas.push_back(pairA); + schemas.push_back(pairB); + schemas.push_back(pairC); + + Tablet tablet1("root.sg1.d1", schemas, 100); + Tablet tablet2("root.sg1.d2", schemas, 100); + Tablet tablet3("root.sg1.d3", schemas, 100); + + unordered_map tabletMap; + tabletMap["root.sg1.d1"] = &tablet1; + tabletMap["root.sg1.d2"] = &tablet2; + tabletMap["root.sg1.d3"] = &tablet3; + + for (int64_t time = 0; time < 20; time++) { + size_t row1 = tablet1.rowSize++; + size_t row2 = tablet2.rowSize++; + size_t row3 = tablet3.rowSize++; + tablet1.timestamps[row1] = time; + tablet2.timestamps[row2] = time; + tablet3.timestamps[row3] = time; + + int randVal11 = rand(); + int randVal12 = rand(); + int randVal13 = rand(); + tablet1.addValue(0, row1, randVal11); + tablet2.addValue(0, row2, randVal12); + tablet3.addValue(0, row3, randVal13); + + double randVal21 = rand() / 99.9; + double randVal22 = rand() / 99.9; + double randVal23 = rand() / 99.9; + tablet1.addValue(1, row1, randVal21); + tablet2.addValue(1, row2, randVal22); + tablet3.addValue(1, row3, randVal23); + + bool randVal31 = (bool)(rand() % 2); + bool randVal32 = (bool)(rand() % 2); + bool randVal33 = (bool)(rand() % 2); + tablet1.addValue(2, row1, randVal31); + tablet2.addValue(2, row2, randVal32); + tablet3.addValue(2, row3, randVal33); + + if (tablet1.rowSize == tablet1.maxRowNumber) { + session->insertAlignedTablets(tabletMap, true); + + tablet1.reset(); + tablet2.reset(); + tablet3.reset(); } + } + + if (tablet1.rowSize != 0) { + session->insertAlignedTablets(tabletMap, true); + tablet1.reset(); + tablet2.reset(); + tablet3.reset(); + } } void insertNullableTabletWithAlignedTimeseries() { - pair pairA("s1", TSDataType::INT32); - pair pairB("s2", TSDataType::INT64); - pair pairC("s3", TSDataType::BOOLEAN); - vector> schemas; - schemas.push_back(pairA); - schemas.push_back(pairB); - schemas.push_back(pairC); - - Tablet tablet("root.sg1.d4", schemas, 20); - tablet.setAligned(true); - - for (int64_t time = 0; time < 20; time++) { - size_t row = tablet.rowSize++; - tablet.timestamps[row] = time; - for (int i = 0; i < 3; i++) { - int randVal1 = rand(); - int64_t randVal2 = rand(); - bool randVal3 = (bool)(rand() % 2); - if (i == 0) { - tablet.addValue(i, row, &randVal1); - } else if (i == 1) { - tablet.addValue(i, row, &randVal2); - } else { - tablet.addValue(i, row, &randVal3); - } - // mark null value - if ((row % 3) == (unsigned int) i) { - tablet.bitMaps[i].mark(row); - } - } - if (tablet.rowSize == tablet.maxRowNumber) { - session->insertTablet(tablet, true); - tablet.reset(); - } + pair pairA("s1", TSDataType::INT32); + pair pairB("s2", TSDataType::INT64); + pair pairC("s3", TSDataType::BOOLEAN); + vector> schemas; + schemas.push_back(pairA); + schemas.push_back(pairB); + schemas.push_back(pairC); + + Tablet tablet("root.sg1.d4", schemas, 20); + tablet.setAligned(true); + + for (int64_t time = 0; time < 20; time++) { + size_t row = tablet.rowSize++; + tablet.timestamps[row] = time; + for (int i = 0; i < 3; i++) { + int randVal1 = rand(); + int64_t randVal2 = rand(); + bool randVal3 = (bool)(rand() % 2); + if (i == 0) { + tablet.addValue(i, row, randVal1); + } else if (i == 1) { + tablet.addValue(i, row, randVal2); + } else { + tablet.addValue(i, row, randVal3); + } + // mark null value + if ((row % 3) == (unsigned int)i) { + tablet.bitMaps[i].mark(row); + } } - - if (tablet.rowSize != 0) { - session->insertTablet(tablet); - tablet.reset(); + if (tablet.rowSize == tablet.maxRowNumber) { + session->insertTablet(tablet, true); + tablet.reset(); } + } + + if (tablet.rowSize != 0) { + session->insertTablet(tablet); + tablet.reset(); + } } void query() { - unique_ptr dataSet = session->executeQueryStatement("select * from root.sg1.**"); - cout << "timestamp" << " "; - for (const string &name: dataSet->getColumnNames()) { - cout << name << " "; - } - cout << endl; - - dataSet->setFetchSize(1024); - while (dataSet->hasNext()) { - cout << dataSet->next()->toString(); - } - cout << endl; - - dataSet->closeOperationHandle(); + unique_ptr dataSet = + session->executeQueryStatement("select * from root.sg1.**"); + cout << "timestamp" + << " "; + for (const string &name : dataSet->getColumnNames()) { + cout << name << " "; + } + cout << endl; + + dataSet->setFetchSize(1024); + while (dataSet->hasNext()) { + cout << dataSet->next()->toString(); + } + cout << endl; + + dataSet->closeOperationHandle(); } void deleteData() { - string path = "root.**"; - int64_t deleteTime = 49; - session->deleteData(path, deleteTime); + string path = "root.**"; + int64_t deleteTime = 49; + session->deleteData(path, deleteTime); } void deleteTimeseries() { - vector paths; - vector alignedTimeseries = {"root.sg1.d1.s1", "root.sg1.d1.s2", "root.sg1.d1.s3", "root.sg1.d1.s4", - "root.sg1.d2.s1", "root.sg1.d2.s2", "root.sg1.d2.s3", - "root.sg1.d3.s1", "root.sg1.d3.s2", "root.sg1.d3.s3", - "root.sg1.d4.s1", "root.sg1.d4.s2", "root.sg1.d4.s3", - "root.sg2.d2.s1", "root.sg2.d2.s2", "root.sg2.d2.s3", }; - for (const string ×eries: alignedTimeseries) { - if (session->checkTimeseriesExists(timeseries)) { - paths.push_back(timeseries); - } + vector paths; + vector alignedTimeseries = { + "root.sg1.d1.s1", "root.sg1.d1.s2", "root.sg1.d1.s3", "root.sg1.d1.s4", + "root.sg1.d2.s1", "root.sg1.d2.s2", "root.sg1.d2.s3", "root.sg1.d3.s1", + "root.sg1.d3.s2", "root.sg1.d3.s3", "root.sg1.d4.s1", "root.sg1.d4.s2", + "root.sg1.d4.s3", "root.sg2.d2.s1", "root.sg2.d2.s2", "root.sg2.d2.s3", + }; + for (const string ×eries : alignedTimeseries) { + if (session->checkTimeseriesExists(timeseries)) { + paths.push_back(timeseries); } - session->deleteTimeseries(paths); + } + session->deleteTimeseries(paths); } void deleteStorageGroups() { - vector storageGroups; - storageGroups.emplace_back("root.sg1"); - storageGroups.emplace_back("root.sg2"); - session->deleteStorageGroups(storageGroups); + vector storageGroups; + storageGroups.emplace_back("root.sg1"); + storageGroups.emplace_back("root.sg2"); + session->deleteStorageGroups(storageGroups); } - int main() { - LOG_LEVEL = LEVEL_DEBUG; + LOG_LEVEL = LEVEL_DEBUG; - session = new Session("127.0.0.1", 6667, "root", "root"); + session = new Session("127.0.0.1", 6667, "root", "root"); - cout << "session open\n" << endl; - session->open(false); + cout << "session open\n" << endl; + session->open(false); - cout << "setStorageGroup\n" << endl; - try { - session->setStorageGroup("root.sg1"); - } - catch (IoTDBException &e) { - string errorMessage(e.what()); - if (errorMessage.find("StorageGroupAlreadySetException") == string::npos) { - cout << errorMessage << endl; - //throw e; - } + cout << "setStorageGroup\n" << endl; + try { + session->setStorageGroup("root.sg1"); + } catch (IoTDBException &e) { + string errorMessage(e.what()); + if (errorMessage.find("StorageGroupAlreadySetException") == string::npos) { + cout << errorMessage << endl; + // throw e; } + } - cout << "createAlignedTimeseries\n" << endl; - createAlignedTimeseries(); + cout << "createAlignedTimeseries\n" << endl; + createAlignedTimeseries(); - cout << "createSchemaTemplate\n" << endl; - createSchemaTemplate(); + cout << "createSchemaTemplate\n" << endl; + createSchemaTemplate(); - cout << "ActivateTemplate\n" << endl; - ActivateTemplate(); + cout << "ActivateTemplate\n" << endl; + ActivateTemplate(); - cout << "showDevices\n" << endl; - showDevices(); + cout << "showDevices\n" << endl; + showDevices(); - cout << "showTimeseries\n" << endl; - showTimeseries(); + cout << "showTimeseries\n" << endl; + showTimeseries(); - cout << "insertAlignedRecord\n" << endl; - insertAlignedRecord(); + cout << "insertAlignedRecord\n" << endl; + insertAlignedRecord(); - cout << "insertAlignedRecords\n" << endl; - insertAlignedRecords(); + cout << "insertAlignedRecords\n" << endl; + insertAlignedRecords(); - cout << "insertAlignedTablet" << endl; - cout << "Insert " << DEFAULT_ROW_NUMBER << " records." << endl; - time_t now1 = time(0); - insertAlignedTablet(); - time_t now2 = time(0); - time_t useTime = now2 - now1; - cout << "Use time: " << useTime << "s.\n" << endl; + cout << "insertAlignedTablet" << endl; + cout << "Insert " << DEFAULT_ROW_NUMBER << " records." << endl; + time_t now1 = time(0); + insertAlignedTablet(); + time_t now2 = time(0); + time_t useTime = now2 - now1; + cout << "Use time: " << useTime << "s.\n" << endl; - cout << "insertAlignedTablets\n" << endl; - insertAlignedTablets(); + cout << "insertAlignedTablets\n" << endl; + insertAlignedTablets(); - cout << "insertNullableTabletWithAlignedTimeseries\n" << endl; - insertNullableTabletWithAlignedTimeseries(); + cout << "insertNullableTabletWithAlignedTimeseries\n" << endl; + insertNullableTabletWithAlignedTimeseries(); - cout << "query\n" << endl; - query(); + cout << "query\n" << endl; + query(); - cout << "deleteData\n" << endl; - deleteData(); + cout << "deleteData\n" << endl; + deleteData(); - cout << "deleteTimeseries\n" << endl; - deleteTimeseries(); + cout << "deleteTimeseries\n" << endl; + deleteTimeseries(); - cout << "deleteStorageGroups\n" << endl; - deleteStorageGroups(); + cout << "deleteStorageGroups\n" << endl; + deleteStorageGroups(); - cout << "session close\n" << endl; - session->close(); + cout << "session close\n" << endl; + session->close(); - delete session; + delete session; - cout << "finished\n" << endl; - return 0; + cout << "finished\n" << endl; + return 0; } diff --git a/example/client-cpp-example/src/CMakeLists.txt b/example/client-cpp-example/src/CMakeLists.txt index 91d93193e38b9..e4bb9eac354c4 100644 --- a/example/client-cpp-example/src/CMakeLists.txt +++ b/example/client-cpp-example/src/CMakeLists.txt @@ -16,33 +16,134 @@ # under the License. # -PROJECT(SessionExample) -CMAKE_MINIMUM_REQUIRED(VERSION 3.7) +CMAKE_MINIMUM_REQUIRED(VERSION 3.15) +CMAKE_POLICY(SET CMP0091 NEW) + +PROJECT(iotdb_cpp_client_examples) SET(CMAKE_CXX_STANDARD 11) SET(CMAKE_CXX_STANDARD_REQUIRED ON) SET(CMAKE_POSITION_INDEPENDENT_CODE ON) -# Add Thrift include directory -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/thrift/include) -# Add cpp-client include directory -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/client/include) +IF(MSVC) + # Match the IoTDB C++ SDK (/MD); same as a default Visual Studio application. + SET(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>DLL") +ENDIF() + +set(IOTDB_SDK_ROOT "${CMAKE_SOURCE_DIR}/client" + CACHE PATH "Unpacked IoTDB C++ SDK directory (contains include/ and lib/)") -FIND_PACKAGE(Boost REQUIRED) -IF (DEFINED BOOST_INCLUDEDIR) - INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIR}) +if(NOT EXISTS "${IOTDB_SDK_ROOT}/include/Session.h" + OR NOT EXISTS "${IOTDB_SDK_ROOT}/include/SessionC.h") + file(GLOB _iotdb_sdk_dirs LIST_DIRECTORIES true "${IOTDB_SDK_ROOT}/iotdb-session-cpp-*") + foreach(_iotdb_sdk_dir IN LISTS _iotdb_sdk_dirs) + if(EXISTS "${_iotdb_sdk_dir}/include/Session.h" + AND EXISTS "${_iotdb_sdk_dir}/include/SessionC.h") + set(IOTDB_SDK_ROOT "${_iotdb_sdk_dir}" + CACHE PATH "Unpacked IoTDB C++ SDK directory (contains include/ and lib/)" FORCE) + break() + endif() + endforeach() +endif() + +INCLUDE_DIRECTORIES("${IOTDB_SDK_ROOT}/include") + +option(WITH_SSL "Build with SSL support" OFF) + +IF(WITH_SSL) + FIND_PACKAGE(OpenSSL REQUIRED) + IF(OpenSSL_FOUND) + MESSAGE(STATUS "OpenSSL found: ${OPENSSL_VERSION}") + INCLUDE_DIRECTORIES(${OPENSSL_INCLUDE_DIR}) + ADD_DEFINITIONS(-DWITH_SSL=1) + ELSE() + MESSAGE(FATAL_ERROR "OpenSSL not found, but WITH_SSL is enabled") + ENDIF() +ELSE() + MESSAGE(STATUS "Building without SSL support") + ADD_DEFINITIONS(-DWITH_SSL=0) ENDIF() -# Add the libs for the cpp-client -LINK_DIRECTORIES(${CMAKE_SOURCE_DIR}/client/lib) +if(WIN32) + set(_iotdb_link_lib "${IOTDB_SDK_ROOT}/lib/iotdb_session.lib") + set(_iotdb_runtime "${IOTDB_SDK_ROOT}/lib/iotdb_session.dll") + if(NOT EXISTS "${_iotdb_link_lib}" AND CMAKE_BUILD_TYPE) + set(_iotdb_link_lib "${IOTDB_SDK_ROOT}/lib/${CMAKE_BUILD_TYPE}/iotdb_session.lib") + set(_iotdb_runtime "${IOTDB_SDK_ROOT}/lib/${CMAKE_BUILD_TYPE}/iotdb_session.dll") + endif() + if(NOT EXISTS "${_iotdb_link_lib}") + set(_iotdb_link_lib "${IOTDB_SDK_ROOT}/lib/Release/iotdb_session.lib") + set(_iotdb_runtime "${IOTDB_SDK_ROOT}/lib/Release/iotdb_session.dll") + endif() + if(NOT EXISTS "${_iotdb_link_lib}") + set(_iotdb_link_lib "${IOTDB_SDK_ROOT}/lib/Debug/iotdb_session.lib") + set(_iotdb_runtime "${IOTDB_SDK_ROOT}/lib/Debug/iotdb_session.dll") + endif() +elseif(APPLE) + set(_iotdb_link_lib "${IOTDB_SDK_ROOT}/lib/libiotdb_session.dylib") + set(_iotdb_runtime "${_iotdb_link_lib}") +else() + set(_iotdb_link_lib "${IOTDB_SDK_ROOT}/lib/libiotdb_session.so") + set(_iotdb_runtime "${_iotdb_link_lib}") +endif() + +if(NOT EXISTS "${_iotdb_link_lib}") + message(FATAL_ERROR + "IoTDB SDK not found at ${IOTDB_SDK_ROOT}. " + "Unpack client-cpp--.zip so that ${_iotdb_link_lib} exists.") +endif() ADD_EXECUTABLE(SessionExample SessionExample.cpp) ADD_EXECUTABLE(AlignedTimeseriesSessionExample AlignedTimeseriesSessionExample.cpp) +ADD_EXECUTABLE(MultiSvrNodeClient MultiSvrNodeClient.cpp) +ADD_EXECUTABLE(tree_example tree_example.c) -IF(MSVC) - TARGET_LINK_LIBRARIES(SessionExample iotdb_session "${CMAKE_SOURCE_DIR}/thrift/lib/Release/thriftmd.lib") - TARGET_LINK_LIBRARIES(AlignedTimeseriesSessionExample iotdb_session "${CMAKE_SOURCE_DIR}/thrift/lib/Release/thriftmd.lib") -ELSE() - TARGET_LINK_LIBRARIES(SessionExample iotdb_session pthread) - TARGET_LINK_LIBRARIES(AlignedTimeseriesSessionExample iotdb_session pthread) -ENDIF() +set(_example_targets + SessionExample + AlignedTimeseriesSessionExample + MultiSvrNodeClient + tree_example) + +foreach(_t IN LISTS _example_targets) + IF(WITH_SSL) + TARGET_LINK_LIBRARIES(${_t} PRIVATE "${_iotdb_link_lib}" OpenSSL::SSL OpenSSL::Crypto) + ELSE() + TARGET_LINK_LIBRARIES(${_t} PRIVATE "${_iotdb_link_lib}") + ENDIF() + IF(UNIX) + TARGET_LINK_LIBRARIES(${_t} PRIVATE pthread) + ENDIF() + + # Run from the build output directory without setting LD_LIBRARY_PATH / PATH. + if(UNIX) + set_target_properties(${_t} PROPERTIES + BUILD_RPATH "\$ORIGIN" + INSTALL_RPATH "\$ORIGIN") + endif() + + if(EXISTS "${_iotdb_runtime}") + add_custom_command(TARGET ${_t} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${_iotdb_runtime}" $ + COMMENT "Copy IoTDB runtime library next to ${_t}") + elseif(WIN32) + message(WARNING "Missing ${_iotdb_runtime}; copy iotdb_session.dll manually before running ${_t}.") + endif() +endforeach() + +# Optional: stage a self-contained folder for copying to another machine (see README). +set(_example_dist_dir "${CMAKE_BINARY_DIR}/dist") +add_custom_target(example-dist DEPENDS ${_example_targets} + COMMENT "Collect example binaries and IoTDB runtime into ${_example_dist_dir}") +foreach(_t IN LISTS _example_targets) + add_custom_command(TARGET example-dist POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory "${_example_dist_dir}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ "${_example_dist_dir}/" + COMMENT "Stage ${_t}") +endforeach() +if(EXISTS "${_iotdb_runtime}") + add_custom_command(TARGET example-dist POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${_iotdb_runtime}" "${_example_dist_dir}/") +endif() diff --git a/example/client-cpp-example/src/MultiSvrNodeClient.cpp b/example/client-cpp-example/src/MultiSvrNodeClient.cpp new file mode 100644 index 0000000000000..c4b1ca46a8366 --- /dev/null +++ b/example/client-cpp-example/src/MultiSvrNodeClient.cpp @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "Session.h" +#include "SessionBuilder.h" +#include "SessionDataSet.h" + +namespace { + +void RunTreeExample() { + try { + std::vector node_urls = {"127.0.0.1:6667", "127.0.0.1:6668", + "127.0.0.1:6669"}; + + auto builder = std::make_shared(); + auto session = std::shared_ptr(builder->username("root") + ->password("root") + ->nodeUrls(node_urls) + ->build()); + + session->open(); + if (!session->checkTimeseriesExists("root.test.d1.s1")) { + session->createTimeseries("root.test.d1.s1", TSDataType::INT64, + TSEncoding::RLE, CompressionType::SNAPPY); + } + session->deleteTimeseries("root.test.d1.s1"); + session->close(); + } catch (const std::exception &e) { + std::cout << "Caught exception: " << e.what() << std::endl; + } +} + +// Example: continuously write/query data so you can manually stop a node +// to test client failover behavior. +void RunResilienceExample() { + try { + std::vector node_urls = {"127.0.0.1:6667", "127.0.0.1:6668", + "127.0.0.1:6669"}; + + auto builder = std::make_shared(); + auto session = std::shared_ptr(builder->username("root") + ->password("root") + ->nodeUrls(node_urls) + ->build()); + + session->open(); + + if (!session->checkTimeseriesExists("root.resilience.d1.s1")) { + session->createTimeseries("root.resilience.d1.s1", TSDataType::INT64, + TSEncoding::RLE, CompressionType::SNAPPY); + } + + std::cout << "Starting resilience test. " + "Stop one node manually to see failover handling..." + << std::endl; + + for (int i = 0; i < 60; ++i) { // run ~60 seconds + int64_t timestamp = std::chrono::system_clock::now().time_since_epoch() / + std::chrono::milliseconds(1); + std::string value = std::to_string(i); + + try { + session->insertRecord("root.resilience.d1", timestamp, {"s1"}, {value}); + std::cout << "[Insert] ts=" << timestamp << ", value=" << value + << std::endl; + + auto dataset = session->executeQueryStatement( + "SELECT s1 FROM root.resilience.d1 LIMIT 1"); + std::cout << "[Query] Got dataset: " << (dataset ? "Success" : "Null") + << std::endl; + + } catch (const std::exception &e) { + std::cout << "Caught exception during resilience loop: " << e.what() + << std::endl; + } + + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + + session->close(); + } catch (const std::exception &e) { + std::cout << "Caught exception in RunResilienceExample: " << e.what() + << std::endl; + } +} + +} // namespace + +int main() { + // RunTreeExample(); + RunResilienceExample(); + return 0; +} diff --git a/example/client-cpp-example/src/SessionExample.cpp b/example/client-cpp-example/src/SessionExample.cpp index c3f5602cb08a8..bcc243c44f22f 100644 --- a/example/client-cpp-example/src/SessionExample.cpp +++ b/example/client-cpp-example/src/SessionExample.cpp @@ -24,438 +24,449 @@ using namespace std; Session *session; void createTimeseries() { - if (!session->checkTimeseriesExists("root.sg1.d1.s1")) { - session->createTimeseries("root.sg1.d1.s1", TSDataType::BOOLEAN, TSEncoding::RLE, - CompressionType::SNAPPY); - } - if (!session->checkTimeseriesExists("root.sg1.d1.s2")) { - session->createTimeseries("root.sg1.d1.s2", TSDataType::INT32, TSEncoding::RLE, - CompressionType::SNAPPY); - } - if (!session->checkTimeseriesExists("root.sg1.d1.s3")) { - session->createTimeseries("root.sg1.d1.s3", TSDataType::FLOAT, TSEncoding::RLE, - CompressionType::SNAPPY); - } - - // create timeseries with tags and attributes - if (!session->checkTimeseriesExists("root.sg1.d1.s4")) { - map tags; - tags["tag1"] = "v1"; - map attributes; - attributes["description"] = "v1"; - session->createTimeseries("root.sg1.d1.s4", TSDataType::INT64, TSEncoding::RLE, - CompressionType::SNAPPY, nullptr, &tags, &attributes, "temperature"); - } + if (!session->checkTimeseriesExists("root.sg1.d1.s1")) { + session->createTimeseries("root.sg1.d1.s1", TSDataType::BOOLEAN, + TSEncoding::RLE, CompressionType::SNAPPY); + } + if (!session->checkTimeseriesExists("root.sg1.d1.s2")) { + session->createTimeseries("root.sg1.d1.s2", TSDataType::INT32, + TSEncoding::RLE, CompressionType::SNAPPY); + } + if (!session->checkTimeseriesExists("root.sg1.d1.s3")) { + session->createTimeseries("root.sg1.d1.s3", TSDataType::FLOAT, + TSEncoding::RLE, CompressionType::SNAPPY); + } + + // create timeseries with tags and attributes + if (!session->checkTimeseriesExists("root.sg1.d1.s4")) { + map tags; + tags["tag1"] = "v1"; + map attributes; + attributes["description"] = "v1"; + session->createTimeseries("root.sg1.d1.s4", TSDataType::INT64, + TSEncoding::RLE, CompressionType::SNAPPY, nullptr, + &tags, &attributes, "temperature"); + } } void createMultiTimeseries() { - if (!session->checkTimeseriesExists("root.sg1.d2.s1") && !session->checkTimeseriesExists("root.sg1.d2.s2")) { - vector paths; - paths.emplace_back("root.sg1.d2.s1"); - paths.emplace_back("root.sg1.d2.s2"); - vector tsDataTypes; - tsDataTypes.push_back(TSDataType::INT64); - tsDataTypes.push_back(TSDataType::DOUBLE); - vector tsEncodings; - tsEncodings.push_back(TSEncoding::RLE); - tsEncodings.push_back(TSEncoding::RLE); - vector compressionTypes; - compressionTypes.push_back(CompressionType::SNAPPY); - compressionTypes.push_back(CompressionType::SNAPPY); - - vector> tagsList; - map tags; - tags["unit"] = "kg"; - tagsList.push_back(tags); - tagsList.push_back(tags); - - vector> attributesList; - map attributes; - attributes["minValue"] = "1"; - attributes["maxValue"] = "100"; - attributesList.push_back(attributes); - attributesList.push_back(attributes); - - vector alias; - alias.emplace_back("weight1"); - alias.emplace_back("weight2"); - - session->createMultiTimeseries(paths, tsDataTypes, tsEncodings, compressionTypes, nullptr, &tagsList, - &attributesList, &alias); - } + if (!session->checkTimeseriesExists("root.sg1.d2.s1") && + !session->checkTimeseriesExists("root.sg1.d2.s2")) { + vector paths; + paths.emplace_back("root.sg1.d2.s1"); + paths.emplace_back("root.sg1.d2.s2"); + vector tsDataTypes; + tsDataTypes.push_back(TSDataType::INT64); + tsDataTypes.push_back(TSDataType::DOUBLE); + vector tsEncodings; + tsEncodings.push_back(TSEncoding::RLE); + tsEncodings.push_back(TSEncoding::RLE); + vector compressionTypes; + compressionTypes.push_back(CompressionType::SNAPPY); + compressionTypes.push_back(CompressionType::SNAPPY); + + vector> tagsList; + map tags; + tags["unit"] = "kg"; + tagsList.push_back(tags); + tagsList.push_back(tags); + + vector> attributesList; + map attributes; + attributes["minValue"] = "1"; + attributes["maxValue"] = "100"; + attributesList.push_back(attributes); + attributesList.push_back(attributes); + + vector alias; + alias.emplace_back("weight1"); + alias.emplace_back("weight2"); + + session->createMultiTimeseries(paths, tsDataTypes, tsEncodings, + compressionTypes, nullptr, &tagsList, + &attributesList, &alias); + } } void createSchemaTemplate() { - if (!session->checkTemplateExists("template1")) { - Template temp("template1", false); - - InternalNode iNodeD99("d99", false); - - MeasurementNode mNodeS1("s1", TSDataType::INT32, TSEncoding::RLE, CompressionType::SNAPPY); - MeasurementNode mNodeS2("s2", TSDataType::INT64, TSEncoding::RLE, CompressionType::SNAPPY); - MeasurementNode mNodeD99S1("s1", TSDataType::DOUBLE, TSEncoding::RLE, CompressionType::SNAPPY); - MeasurementNode mNodeD99S2("s2", TSDataType::BOOLEAN, TSEncoding::RLE, CompressionType::SNAPPY); - - iNodeD99.addChild(mNodeD99S1); - iNodeD99.addChild(mNodeD99S2); - - temp.addToTemplate(iNodeD99); - temp.addToTemplate(mNodeS1); - temp.addToTemplate(mNodeS2); - - session->createSchemaTemplate(temp); - session->setSchemaTemplate("template1", "root.sg3.d1"); - } + if (!session->checkTemplateExists("template1")) { + Template temp("template1", false); + + InternalNode iNodeD99("d99", false); + + MeasurementNode mNodeS1("s1", TSDataType::INT32, TSEncoding::RLE, + CompressionType::SNAPPY); + MeasurementNode mNodeS2("s2", TSDataType::INT64, TSEncoding::RLE, + CompressionType::SNAPPY); + MeasurementNode mNodeD99S1("s1", TSDataType::DOUBLE, TSEncoding::RLE, + CompressionType::SNAPPY); + MeasurementNode mNodeD99S2("s2", TSDataType::BOOLEAN, TSEncoding::RLE, + CompressionType::SNAPPY); + + iNodeD99.addChild(mNodeD99S1); + iNodeD99.addChild(mNodeD99S2); + + temp.addToTemplate(iNodeD99); + temp.addToTemplate(mNodeS1); + temp.addToTemplate(mNodeS2); + + session->createSchemaTemplate(temp); + session->setSchemaTemplate("template1", "root.sg3.d1"); + } } void ActivateTemplate() { - session->executeNonQueryStatement("insert into root.sg3.d1(timestamp,s1, s2) values(200, 1, 1);"); + session->executeNonQueryStatement( + "insert into root.sg3.d1(timestamp,s1, s2) values(200, 1, 1);"); } void showTimeseries() { - unique_ptr dataSet = session->executeQueryStatement("show timeseries"); - for (const string &name: dataSet->getColumnNames()) { - cout << name << " "; - } - cout << endl; - - dataSet->setFetchSize(1024); - while (dataSet->hasNext()) { - cout << dataSet->next()->toString(); - } - cout << endl; - - dataSet->closeOperationHandle(); + unique_ptr dataSet = + session->executeQueryStatement("show timeseries"); + for (const string &name : dataSet->getColumnNames()) { + cout << name << " "; + } + cout << endl; + + dataSet->setFetchSize(1024); + while (dataSet->hasNext()) { + cout << dataSet->next()->toString(); + } + cout << endl; + + dataSet->closeOperationHandle(); } void insertRecord() { - string deviceId = "root.sg2.d1"; - vector measurements; - measurements.emplace_back("s1"); - measurements.emplace_back("s2"); - measurements.emplace_back("s3"); - for (int64_t time = 0; time < 10; time++) { - vector values; - values.emplace_back("1"); - values.emplace_back("2"); - values.emplace_back("3"); - session->insertRecord(deviceId, time, measurements, values); - } + string deviceId = "root.sg2.d1"; + vector measurements; + measurements.emplace_back("s1"); + measurements.emplace_back("s2"); + measurements.emplace_back("s3"); + for (int64_t time = 0; time < 10; time++) { + vector values; + values.emplace_back("1"); + values.emplace_back("2"); + values.emplace_back("3"); + session->insertRecord(deviceId, time, measurements, values); + } } void insertTablet() { - pair pairA("s1", TSDataType::BOOLEAN); - pair pairB("s2", TSDataType::INT32); - pair pairC("s3", TSDataType::FLOAT); - vector> schemas; - schemas.push_back(pairA); - schemas.push_back(pairB); - schemas.push_back(pairC); + pair pairA("s1", TSDataType::BOOLEAN); + pair pairB("s2", TSDataType::INT32); + pair pairC("s3", TSDataType::FLOAT); + vector> schemas; + schemas.push_back(pairA); + schemas.push_back(pairB); + schemas.push_back(pairC); - Tablet tablet("root.sg1.d1", schemas, 100); + Tablet tablet("root.sg1.d1", schemas, 100); - for (int64_t time = 0; time < 30; time++) { - size_t row = tablet.rowSize++; - tablet.timestamps[row] = time; + for (int64_t time = 0; time < 30; time++) { + size_t row = tablet.rowSize++; + tablet.timestamps[row] = time; - bool randVal1 = rand() % 2; - tablet.addValue(0, row, &randVal1); + bool randVal1 = rand() % 2; + tablet.addValue(0, row, randVal1); - int randVal2 = rand(); - tablet.addValue(1, row, &randVal2); + int randVal2 = rand(); + tablet.addValue(1, row, randVal2); - float randVal3 = (float)(rand() / 99.9); - tablet.addValue(2, row, &randVal3); + float randVal3 = (float)(rand() / 99.9); + tablet.addValue(2, row, randVal3); - if (tablet.rowSize == tablet.maxRowNumber) { - session->insertTablet(tablet, true); - tablet.reset(); - } + if (tablet.rowSize == tablet.maxRowNumber) { + session->insertTablet(tablet, true); + tablet.reset(); } + } - if (tablet.rowSize != 0) { - session->insertTablet(tablet); - tablet.reset(); - } + if (tablet.rowSize != 0) { + session->insertTablet(tablet); + tablet.reset(); + } } void insertRecords() { - string deviceId = "root.sg2.d1"; - vector measurements; - measurements.emplace_back("s1"); - measurements.emplace_back("s2"); - measurements.emplace_back("s3"); - - vector deviceIds; - vector> measurementsList; - vector> valuesList; - vector timestamps; - - for (int64_t time = 10; time < 30; time++) { - vector values; - values.emplace_back("1"); - values.emplace_back("2"); - values.emplace_back("3"); - - deviceIds.push_back(deviceId); - measurementsList.push_back(measurements); - valuesList.push_back(values); - timestamps.push_back(time); - if (time != 20 && time % 10 == 0) { - session->insertRecords(deviceIds, timestamps, measurementsList, valuesList); - deviceIds.clear(); - measurementsList.clear(); - valuesList.clear(); - timestamps.clear(); - } + string deviceId = "root.sg2.d1"; + vector measurements; + measurements.emplace_back("s1"); + measurements.emplace_back("s2"); + measurements.emplace_back("s3"); + + vector deviceIds; + vector> measurementsList; + vector> valuesList; + vector timestamps; + + for (int64_t time = 10; time < 30; time++) { + vector values; + values.emplace_back("1"); + values.emplace_back("2"); + values.emplace_back("3"); + + deviceIds.push_back(deviceId); + measurementsList.push_back(measurements); + valuesList.push_back(values); + timestamps.push_back(time); + if (time != 20 && time % 10 == 0) { + session->insertRecords(deviceIds, timestamps, measurementsList, + valuesList); + deviceIds.clear(); + measurementsList.clear(); + valuesList.clear(); + timestamps.clear(); } + } - session->insertRecords(deviceIds, timestamps, measurementsList, valuesList); + session->insertRecords(deviceIds, timestamps, measurementsList, valuesList); } void insertTablets() { - pair pairA("s1", TSDataType::INT64); - pair pairB("s2", TSDataType::DOUBLE); - pair pairC("s3", TSDataType::TEXT); - vector> schemas; - schemas.push_back(pairA); - schemas.push_back(pairB); - schemas.push_back(pairC); + pair pairA("s1", TSDataType::INT64); + pair pairB("s2", TSDataType::DOUBLE); + pair pairC("s3", TSDataType::TEXT); + vector> schemas; + schemas.push_back(pairA); + schemas.push_back(pairB); + schemas.push_back(pairC); - Tablet tablet1("root.sg1.d2", schemas, 100); - Tablet tablet2("root.sg1.d3", schemas, 100); + Tablet tablet1("root.sg1.d2", schemas, 100); + Tablet tablet2("root.sg1.d3", schemas, 100); - unordered_map tabletMap; - tabletMap["root.sg1.d2"] = &tablet1; - tabletMap["root.sg1.d3"] = &tablet2; + unordered_map tabletMap; + tabletMap["root.sg1.d2"] = &tablet1; + tabletMap["root.sg1.d3"] = &tablet2; - for (int64_t time = 0; time < 30; time++) { - size_t row1 = tablet1.rowSize++; - size_t row2 = tablet2.rowSize++; - tablet1.timestamps[row1] = time; - tablet2.timestamps[row2] = time; + for (int64_t time = 0; time < 30; time++) { + size_t row1 = tablet1.rowSize++; + size_t row2 = tablet2.rowSize++; + tablet1.timestamps[row1] = time; + tablet2.timestamps[row2] = time; - int64_t randVal11 = rand(); - tablet1.addValue(0, row1, &randVal11); + int64_t randVal11 = rand(); + tablet1.addValue(0, row1, randVal11); - double randVal12 = rand() / 99.9; - tablet1.addValue(1, row1, &randVal12); + double randVal12 = rand() / 99.9; + tablet1.addValue(1, row1, randVal12); - string randVal13 = "string" + to_string(rand()); - tablet1.addValue(2, row1, &randVal13); + string randVal13 = "string" + to_string(rand()); + tablet1.addValue(2, row1, randVal13); - int64_t randVal21 = rand(); - tablet2.addValue(0, row2, &randVal21); + int64_t randVal21 = rand(); + tablet2.addValue(0, row2, randVal21); - double randVal22 = rand() / 99.9; - tablet2.addValue(1, row2, &randVal22); + double randVal22 = rand() / 99.9; + tablet2.addValue(1, row2, randVal22); - string randVal23 = "string" + to_string(rand()); - tablet2.addValue(2, row2, &randVal23); + string randVal23 = "string" + to_string(rand()); + tablet2.addValue(2, row2, randVal23); - if (tablet1.rowSize == tablet1.maxRowNumber) { - session->insertTablets(tabletMap, true); + if (tablet1.rowSize == tablet1.maxRowNumber) { + session->insertTablets(tabletMap, true); - tablet1.reset(); - tablet2.reset(); - } + tablet1.reset(); + tablet2.reset(); } + } - if (tablet1.rowSize != 0) { - session->insertTablets(tabletMap, true); - tablet1.reset(); - tablet2.reset(); - } + if (tablet1.rowSize != 0) { + session->insertTablets(tabletMap, true); + tablet1.reset(); + tablet2.reset(); + } } void insertTabletWithNullValues() { - /* - * A Tablet example: - * device1 - * time s1, s2, s3 - * 1, null, 1, 1 - * 2, 2, null, 2 - * 3, 3, 3, null - */ - pair pairA("s1", TSDataType::INT64); - pair pairB("s2", TSDataType::INT64); - pair pairC("s3", TSDataType::INT64); - vector> schemas; - schemas.push_back(pairA); - schemas.push_back(pairB); - schemas.push_back(pairC); - - Tablet tablet("root.sg1.d4", schemas, 30); - - for (int64_t time = 0; time < 30; time++) { - size_t row = tablet.rowSize++; - tablet.timestamps[row] = time; - for (int i = 0; i < 3; i++) { - int64_t randVal = rand(); - tablet.addValue(i, row, &randVal); - // mark null value - if (row % 3 == (unsigned int) i) { - tablet.bitMaps[i].mark(row); - } - } - if (tablet.rowSize == tablet.maxRowNumber) { - session->insertTablet(tablet, true); - tablet.reset(); - } + /* + * A Tablet example: + * device1 + * time s1, s2, s3 + * 1, null, 1, 1 + * 2, 2, null, 2 + * 3, 3, 3, null + */ + pair pairA("s1", TSDataType::INT64); + pair pairB("s2", TSDataType::INT64); + pair pairC("s3", TSDataType::INT64); + vector> schemas; + schemas.push_back(pairA); + schemas.push_back(pairB); + schemas.push_back(pairC); + + Tablet tablet("root.sg1.d4", schemas, 30); + + for (int64_t time = 0; time < 30; time++) { + size_t row = tablet.rowSize++; + tablet.timestamps[row] = time; + for (int i = 0; i < 3; i++) { + int64_t randVal = rand(); + tablet.addValue(i, row, randVal); + // mark null value + if (row % 3 == (unsigned int)i) { + tablet.bitMaps[i].mark(row); + } } - - if (tablet.rowSize != 0) { - session->insertTablet(tablet); - tablet.reset(); + if (tablet.rowSize == tablet.maxRowNumber) { + session->insertTablet(tablet, true); + tablet.reset(); } + } + + if (tablet.rowSize != 0) { + session->insertTablet(tablet); + tablet.reset(); + } } void nonQuery() { - session->executeNonQueryStatement("insert into root.sg1.d1(timestamp,s1) values(100, 1);"); + session->executeNonQueryStatement( + "insert into root.sg1.d1(timestamp,s1) values(100, 1);"); } void query() { - unique_ptr dataSet = session->executeQueryStatement("select s1, s2, s3 from root.**"); - for (const string &name: dataSet->getColumnNames()) { - cout << name << " "; - } - cout << endl; - - dataSet->setFetchSize(1024); - while (dataSet->hasNext()) { - cout << dataSet->next()->toString(); - } - cout << endl; - - dataSet->closeOperationHandle(); + unique_ptr dataSet = + session->executeQueryStatement("select s1, s2, s3 from root.**"); + for (const string &name : dataSet->getColumnNames()) { + cout << name << " "; + } + cout << endl; + + dataSet->setFetchSize(1024); + while (dataSet->hasNext()) { + cout << dataSet->next()->toString(); + } + cout << endl; + + dataSet->closeOperationHandle(); } void deleteData() { - string path = "root.sg1.d1.s1"; - int64_t deleteTime = 99; - session->deleteData(path, deleteTime); + string path = "root.sg1.d1.s1"; + int64_t deleteTime = 99; + session->deleteData(path, deleteTime); } void deleteTimeseries() { - vector paths; - vector timeseriesGrp = { "root.sg1.d1.s1", "root.sg1.d1.s2", "root.sg1.d1.s3", - "root.sg1.d2.s1", "root.sg1.d2.s2", "root.sg1.d2.s3", - "root.sg1.d3.s1", "root.sg1.d3.s2", "root.sg1.d3.s3", - "root.sg1.d4.s1", "root.sg1.d4.s2", "root.sg1.d4.s3", - "root.sg2.d1.s1", "root.sg2.d1.s2", "root.sg2.d1.s3" }; - for (const string& timeseries : timeseriesGrp) { - if (session->checkTimeseriesExists(timeseries)) { - paths.push_back(timeseries); - } + vector paths; + vector timeseriesGrp = { + "root.sg1.d1.s1", "root.sg1.d1.s2", "root.sg1.d1.s3", "root.sg1.d2.s1", + "root.sg1.d2.s2", "root.sg1.d2.s3", "root.sg1.d3.s1", "root.sg1.d3.s2", + "root.sg1.d3.s3", "root.sg1.d4.s1", "root.sg1.d4.s2", "root.sg1.d4.s3", + "root.sg2.d1.s1", "root.sg2.d1.s2", "root.sg2.d1.s3"}; + for (const string ×eries : timeseriesGrp) { + if (session->checkTimeseriesExists(timeseries)) { + paths.push_back(timeseries); } - session->deleteTimeseries(paths); + } + session->deleteTimeseries(paths); } void deleteStorageGroups() { - vector storageGroups; - storageGroups.emplace_back("root.sg1"); - storageGroups.emplace_back("root.sg2"); - session->deleteStorageGroups(storageGroups); + vector storageGroups; + storageGroups.emplace_back("root.sg1"); + storageGroups.emplace_back("root.sg2"); + session->deleteStorageGroups(storageGroups); } void queryLast() { - unique_ptr dataSet = session->executeQueryStatement("select last s1,s2,s3 from root.**"); - for (const string &name: dataSet->getColumnNames()) { - cout << name << " "; - } - cout << endl; - - while (dataSet->hasNext()) { - cout << dataSet->next()->toString(); - } - cout << endl; - - dataSet->closeOperationHandle(); + unique_ptr dataSet = + session->executeQueryStatement("select last s1,s2,s3 from root.**"); + for (const string &name : dataSet->getColumnNames()) { + cout << name << " "; + } + cout << endl; + + while (dataSet->hasNext()) { + cout << dataSet->next()->toString(); + } + cout << endl; + + dataSet->closeOperationHandle(); } int main() { - LOG_LEVEL = LEVEL_DEBUG; - - session = new Session("127.0.0.1", 6667, "root", "root"); - session->open(false); - - cout << "setStorageGroup: root.sg1\n" << endl; - try { - session->setStorageGroup("root.sg1"); - } - catch (IoTDBException &e) { - string errorMessage(e.what()); - if (errorMessage.find("StorageGroupAlreadySetException") == string::npos) { - cout << errorMessage << endl; - } - //throw e; - } - - cout << "setStorageGroup: root.sg2\n" << endl; - try { - session->setStorageGroup("root.sg2"); + LOG_LEVEL = LEVEL_DEBUG; + + session = new Session("127.0.0.1", 6667, "root", "root"); + session->open(false); + + cout << "setStorageGroup: root.sg1\n" << endl; + try { + session->setStorageGroup("root.sg1"); + } catch (IoTDBException &e) { + string errorMessage(e.what()); + if (errorMessage.find("StorageGroupAlreadySetException") == string::npos) { + cout << errorMessage << endl; } - catch (IoTDBException &e) { - string errorMessage(e.what()); - if (errorMessage.find("StorageGroupAlreadySetException") == string::npos) { - cout << errorMessage << endl; - } - //throw e; + // throw e; + } + + cout << "setStorageGroup: root.sg2\n" << endl; + try { + session->setStorageGroup("root.sg2"); + } catch (IoTDBException &e) { + string errorMessage(e.what()); + if (errorMessage.find("StorageGroupAlreadySetException") == string::npos) { + cout << errorMessage << endl; } + // throw e; + } - cout << "createTimeseries\n" << endl; - createTimeseries(); + cout << "createTimeseries\n" << endl; + createTimeseries(); - cout << "createMultiTimeseries\n" << endl; - createMultiTimeseries(); + cout << "createMultiTimeseries\n" << endl; + createMultiTimeseries(); - cout << "createSchemaTemplate\n" << endl; - createSchemaTemplate(); + cout << "createSchemaTemplate\n" << endl; + createSchemaTemplate(); - cout << "ActivateTemplate\n" << endl; - ActivateTemplate(); + cout << "ActivateTemplate\n" << endl; + ActivateTemplate(); - cout << "showTimeseries\n" << endl; - showTimeseries(); + cout << "showTimeseries\n" << endl; + showTimeseries(); - cout << "insertRecord\n" << endl; - insertRecord(); + cout << "insertRecord\n" << endl; + insertRecord(); - cout << "insertTablet\n" << endl; - insertTablet(); + cout << "insertTablet\n" << endl; + insertTablet(); - cout << "insertRecords\n" << endl; - insertRecords(); + cout << "insertRecords\n" << endl; + insertRecords(); - cout << "insertTablets\n" << endl; - insertTablets(); + cout << "insertTablets\n" << endl; + insertTablets(); - cout << "insertTabletWithNullValues\n" << endl; - insertTabletWithNullValues(); + cout << "insertTabletWithNullValues\n" << endl; + insertTabletWithNullValues(); - cout << "nonQuery\n" << endl; - nonQuery(); + cout << "nonQuery\n" << endl; + nonQuery(); - cout << "queryLast\n" << endl; - queryLast(); + cout << "queryLast\n" << endl; + queryLast(); - cout << "query\n" << endl; - query(); + cout << "query\n" << endl; + query(); - cout << "deleteData\n" << endl; - deleteData(); + cout << "deleteData\n" << endl; + deleteData(); - cout << "deleteTimeseries\n" << endl; - deleteTimeseries(); + cout << "deleteTimeseries\n" << endl; + deleteTimeseries(); - cout << "deleteStorageGroups\n" << endl; - deleteStorageGroups(); + cout << "deleteStorageGroups\n" << endl; + deleteStorageGroups(); - cout << "session close\n" << endl; - session->close(); + cout << "session close\n" << endl; + session->close(); - delete session; + delete session; - cout << "finished!\n" << endl; - return 0; + cout << "finished!\n" << endl; + return 0; } diff --git a/example/client-cpp-example/src/tree_example.c b/example/client-cpp-example/src/tree_example.c new file mode 100644 index 0000000000000..1d1045a8c4eee --- /dev/null +++ b/example/client-cpp-example/src/tree_example.c @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Tree model: create one timeseries, insert one row via string values, SELECT, + * cleanup. Edit HOST / PORT / credentials below to match your IoTDB. + */ + +#include +#include +#include +#include + +#include "SessionC.h" + +#define HOST "127.0.0.1" +#define PORT 6667 +#define USER "root" +#define PASS "root" + +#define TS_PATH "root.cdemo.d0.s0" +#define DEVICE "root.cdemo.d0" + +static void fail(const char *ctx, CSession *s) { + fprintf(stderr, "[tree_example] %s failed: %s\n", ctx, ts_get_last_error()); + if (s) { + ts_session_close(s); + ts_session_destroy(s); + } + exit(1); +} + +int main(void) { + const char *path = TS_PATH; + CSession *session = ts_session_new(HOST, PORT, USER, PASS); + if (!session) { + fprintf(stderr, "[tree_example] ts_session_new returned NULL: %s\n", + ts_get_last_error()); + return 1; + } + if (ts_session_open(session) != TS_OK) { + fail("ts_session_open", session); + } + + bool exists = false; + if (ts_session_check_timeseries_exists(session, path, &exists) != TS_OK) { + fail("ts_session_check_timeseries_exists", session); + } + if (exists) { + if (ts_session_delete_timeseries(session, path) != TS_OK) { + fail("ts_session_delete_timeseries (cleanup old)", session); + } + } + if (ts_session_create_timeseries(session, path, TS_TYPE_INT64, + TS_ENCODING_RLE, + TS_COMPRESSION_SNAPPY) != TS_OK) { + fail("ts_session_create_timeseries", session); + } + + const char *measurements[] = {"s0"}; + const char *values[] = {"100"}; + if (ts_session_insert_record_str(session, DEVICE, 1LL, 1, measurements, + values) != TS_OK) { + fail("ts_session_insert_record_str", session); + } + + CSessionDataSet *dataSet = NULL; + if (ts_session_execute_query(session, "select s0 from root.cdemo.d0", + &dataSet) != TS_OK) { + fail("ts_session_execute_query", session); + } + if (!dataSet) { + fprintf(stderr, "[tree_example] dataSet is NULL\n"); + ts_session_close(session); + ts_session_destroy(session); + return 1; + } + ts_dataset_set_fetch_size(dataSet, 1024); + + int rows = 0; + while (ts_dataset_has_next(dataSet)) { + CRowRecord *record = ts_dataset_next(dataSet); + if (!record) { + break; + } + int64_t v = ts_row_record_get_int64(record, 0); + printf("[tree_example] row %d: s0 = %lld\n", rows, (long long)v); + ts_row_record_destroy(record); + rows++; + } + ts_dataset_destroy(dataSet); + + printf("[tree_example] done, read %d row(s).\n", rows); + + if (ts_session_delete_timeseries(session, path) != TS_OK) { + fail("ts_session_delete_timeseries", session); + } + + ts_session_close(session); + ts_session_destroy(session); + return 0; +} diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java index 9663fa371e95a..136399a3778b0 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java @@ -117,7 +117,7 @@ private static String getWindowsSearchPortCmd(final List ports) { } private static String getUnixSearchPortCmd(final List ports) { - return "lsof -iTCP -sTCP:LISTEN -P -n | awk '{print $9}' | grep -E " + return "lsof -iTCP -sTCP:LISTEN,TIME_WAIT -P -n | awk '{print $9}' | grep -E " + ports.stream().map(String::valueOf).collect(Collectors.joining("|")) + "\""; } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java index 473d2f1b64c7f..975ae1a5682df 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppBaseConfig.java @@ -40,6 +40,8 @@ public abstract class MppBaseConfig { /** Create an empty MppPersistentConfig. */ protected MppBaseConfig() { this.properties = new Properties(); + this.properties.setProperty("cn_selector_thread_nums_of_client_manager", "1"); + this.properties.setProperty("dn_selector_thread_nums_of_client_manager", "1"); } /** diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java index d9017df36e767..8109bf02c7ea4 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java @@ -508,7 +508,7 @@ public CommonConfig setPipeMetaSyncerSyncIntervalMinutes(long pipeMetaSyncerSync public CommonConfig setPipeConnectorRequestSliceThresholdBytes( int pipeConnectorRequestSliceThresholdBytes) { setProperty( - "pipe_connector_request_slice_threshold_bytes", + "pipe_sink_request_slice_threshold_bytes", String.valueOf(pipeConnectorRequestSliceThresholdBytes)); return this; diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java index 60ff77aa03c52..07b7364d0042d 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java @@ -56,7 +56,10 @@ import org.apache.iotdb.session.Session; import org.apache.iotdb.session.pool.SessionPool; +import org.apache.thrift.TConfiguration; import org.apache.thrift.TException; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransportException; import org.slf4j.Logger; import java.io.File; @@ -1119,7 +1122,7 @@ public void shutdownForciblyAllDataNodes() { @Override public void ensureNodeStatus( - final List nodes, final List targetStatus) + final List nodes, final List targetStatusList) throws IllegalStateException { Throwable lastException = null; for (int i = 0; i < retryCount; i++) { @@ -1147,7 +1150,9 @@ public void ensureNodeStatus( + node.getClientRpcEndPoint().getPort(), node.getDataNodeId())); for (int j = 0; j < nodes.size(); j++) { - final String endpoint = nodes.get(j).getIpAndPortString(); + BaseNodeWrapper nodeWrapper = nodes.get(j); + String ipAndPortString = nodeWrapper.getIpAndPortString(); + final String endpoint = ipAndPortString; if (!nodeIds.containsKey(endpoint)) { // Node not exist // Notice: Never modify this line, since the NodeLocation might be modified in IT @@ -1155,12 +1160,27 @@ public void ensureNodeStatus( continue; } final String status = showClusterResp.getNodeStatus().get(nodeIds.get(endpoint)); - if (!targetStatus.get(j).getStatus().equals(status)) { + final NodeStatus targetStatus = targetStatusList.get(j); + if (!targetStatus.getStatus().equals(status)) { // Error status errorMessages.add( String.format( "Node %s is in status %s, but expected %s", - endpoint, status, targetStatus.get(j))); + endpoint, status, targetStatusList.get(j))); + continue; + } + if (nodeWrapper instanceof DataNodeWrapper && targetStatus.equals(NodeStatus.Running)) { + final String[] ipPort = nodeWrapper.getIpAndPortString().split(":"); + final String ip = ipPort[0]; + final int port = Integer.parseInt(ipPort[1]); + try (TSocket socket = new TSocket(new TConfiguration(), ip, port, 1000)) { + socket.open(); + } catch (final TTransportException e) { + errorMessages.add( + String.format( + "DataNode %s is not reachable: %s", + nodeWrapper.getIpAndPortString(), e.getMessage())); + } } } if (errorMessages.isEmpty()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileWithModIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileWithModIT.java new file mode 100644 index 0000000000000..4e8e4c8a629a7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileWithModIT.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.db.exception.DataRegionException; +import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.wal.recover.file.SealedTsFileRecoverPerformer; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.itbase.exception.InconsistentDataException; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collections; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBLoadTsFileWithModIT { + private static File tmpDir; + + @BeforeClass + public static void setUp() throws Exception { + tmpDir = new File(Files.createTempDirectory("load").toUri()); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + try { + if (tmpDir != null && tmpDir.exists()) { + File[] files = tmpDir.listFiles(); + if (files != null) { + for (File file : files) { + try { + Files.delete(file.toPath()); + } catch (IOException ignored) { + // ignore + } + } + } + try { + Files.delete(tmpDir.toPath()); + } catch (IOException ignored) { + // ignore + } + } + } finally { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + } + + private static void clearTmpDir() { + if (tmpDir == null || !tmpDir.exists()) { + return; + } + File[] files = tmpDir.listFiles(); + if (files == null) { + return; + } + for (File file : files) { + try { + Files.delete(file.toPath()); + } catch (IOException ignored) { + // ignore + } + } + } + + private static void executeQuietly(Statement statement, String sql) { + try { + statement.execute(sql); + } catch (SQLException ignored) { + // ignore + } + } + + private static void cleanupTestDb() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + executeQuietly(statement, "DELETE DATABASE root.test.**"); + executeQuietly(statement, "DELETE DATABASE root.test"); + } catch (SQLException ignored) { + // ignore + } + } + + private void generateFileWithModFile() + throws IOException, WriteProcessException, IllegalPathException, DataRegionException { + TsFileResource resource = generateFile(); + try (ModificationFile modFile = ModificationFile.getNormalMods(resource)) { + modFile.write(new Deletion(new MeasurementPath("root.test.d1.de.s1"), Long.MAX_VALUE, 1, 2)); + } + } + + private TsFileResource generateFile() + throws WriteProcessException, IOException, DataRegionException { + File tsfile = new File(tmpDir, "1-1-0-0.tsfile"); + try (TsFileWriter writer = new TsFileWriter(tsfile)) { + writer.registerAlignedTimeseries( + new Path("root.test.d1.de"), + Collections.singletonList(new MeasurementSchema("s1", TSDataType.BOOLEAN))); + Tablet tablet = + new Tablet( + "root.test.d1.de", + Collections.singletonList(new MeasurementSchema("s1", TSDataType.BOOLEAN))); + for (int i = 0; i < 5; i++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, i); + tablet.addValue("s1", rowIndex, true); + } + writer.writeAligned(tablet); + } + // generate resource file + TsFileResource resource = new TsFileResource(tsfile); + try (SealedTsFileRecoverPerformer performer = new SealedTsFileRecoverPerformer(resource)) { + performer.recover(); + } + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + resource.deserialize(); + return resource; + } + + @Test + public void testWithNewModFile() + throws SQLException, + IOException, + DataRegionException, + WriteProcessException, + IllegalPathException { + try { + generateFileWithModFile(); + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + + statement.execute(String.format("load \'%s\'", tmpDir.getAbsolutePath())); + + try (final ResultSet resultSet = + statement.executeQuery("select count(s1) as c from root.test.d1.de")) { + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(3, resultSet.getLong("c")); + } + } + } finally { + clearTmpDir(); + cleanupTestDb(); + } + } + + @Test + public void testWithNewModFileAndLoadAttributes() + throws SQLException, + IOException, + DataRegionException, + WriteProcessException, + IllegalPathException { + try { + generateFileWithModFile(); + final String databaseName = "root.test.d1"; + + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + + statement.execute( + String.format( + "load \'%s\' with (" + + "'database-level'='2'," + + "'verify'='true'," + + "'on-success'='none'," + + "'async'='true')", + tmpDir.getAbsolutePath())); + + boolean databaseFound = false; + for (int i = 0; i < 10; i++) { + try (final ResultSet resultSet = statement.executeQuery("show databases")) { + while (resultSet.next()) { + final String currentDatabase = resultSet.getString(1); + if (databaseName.equalsIgnoreCase(currentDatabase)) { + databaseFound = true; + break; + } + } + } catch (InconsistentDataException ignored) { + // Async load propagates the new database metadata to different DataNodes at + // slightly different times, so "show databases" may be inconsistent transiently. + } + + if (databaseFound) { + break; + } + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + Assert.assertTrue( + "The `database-level` parameter is not working; the generated database does not contain 'root.test.d1'.", + databaseFound); + } + } finally { + clearTmpDir(); + cleanupTestDb(); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAlignByDeviceWildcardIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAlignByDeviceWildcardIT.java new file mode 100644 index 0000000000000..e2bb07f23bee8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAlignByDeviceWildcardIT.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.aggregation; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareData; +import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class}) +public class IoTDBAlignByDeviceWildcardIT { + + private static final String[] SQL_LIST = + new String[] { + "CREATE DATABASE root.min", + "CREATE TIMESERIES root.min.d1.s1 WITH DATATYPE=INT32, ENCODING=PLAIN", + "CREATE TIMESERIES root.min.d1.s2 WITH DATATYPE=INT32, ENCODING=PLAIN", + "CREATE TIMESERIES root.min.d2.s1 WITH DATATYPE=INT32, ENCODING=PLAIN", + "CREATE TIMESERIES root.min.d2.s2 WITH DATATYPE=INT32, ENCODING=PLAIN", + "INSERT INTO root.min.d1(time, s1, s2) VALUES(1, 1, 1)", + "INSERT INTO root.min.d1(time, s1, s2) VALUES(2, 1, 1)", + "FLUSH", + "INSERT INTO root.min.d2(time, s1, s2) VALUES(5, 1, 1)", + "FLUSH" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setDefaultDataRegionGroupNumPerDatabase(1) + .setTimePartitionInterval(1) + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(SQL_LIST); + } + + @AfterClass + public static void tearDown() throws Exception { + // EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testWildcardAlignByDeviceWithTimePartitionSplit() { + String sql = + "SELECT count(s1) FROM root.min.** " + + "WHERE s2 is not null and s1 is not null " + + "GROUP BY([1, 6), 1ms) ALIGN BY DEVICE"; + String[] expectedHeader = new String[] {"Time", "Device", "count(s1)"}; + String[] expectedRows = + new String[] { + "1,root.min.d1,1,", + "2,root.min.d1,1,", + "3,root.min.d1,0,", + "4,root.min.d1,0,", + "5,root.min.d1,0,", + "1,root.min.d2,0,", + "2,root.min.d2,0,", + "3,root.min.d2,0,", + "4,root.min.d2,0,", + "5,root.min.d2,1,", + }; + resultSetEqualTest(sql, expectedHeader, expectedRows); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedLastQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedLastQueryIT.java index 214a2c31806f4..c66de9e73d523 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedLastQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedLastQueryIT.java @@ -124,7 +124,8 @@ public void selectAllAlignedAndNonAlignedLastTest() { "40,root.sg1.d2.s2,40,INT32", "30,root.sg1.d2.s3,30,INT64", "30,root.sg1.d2.s4,false,BOOLEAN", - "40,root.sg1.d2.s5,non_aligned_test40,TEXT")); + "40,root.sg1.d2.s5,non_aligned_test40,TEXT", + "-40,root.sg1.d2.s6,40.0,DOUBLE")); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQueryIT.java index 016e2d59daab7..a361083106896 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBAlignedSeriesQueryIT.java @@ -165,46 +165,47 @@ public void selectAllAlignedAndNonAlignedTest() { String[] retArray = new String[] { - "1,1.0,1,1,true,aligned_test1,1.0,1,1,true,non_aligned_test1", - "2,2.0,2,2,null,aligned_test2,2.0,2,2,null,non_aligned_test2", - "3,30000.0,null,30000,true,aligned_unseq_test3,3.0,null,3,false,non_aligned_test3", - "4,4.0,4,null,true,aligned_test4,4.0,4,null,true,non_aligned_test4", - "5,5.0,5,null,true,aligned_test5,5.0,5,null,true,non_aligned_test5", - "6,6.0,6,6,true,null,6.0,6,6,true,null", - "7,7.0,7,7,false,aligned_test7,7.0,7,7,false,non_aligned_test7", - "8,8.0,8,8,null,aligned_test8,8.0,8,8,null,non_aligned_test8", - "9,9.0,9,9,false,aligned_test9,9.0,9,9,false,non_aligned_test9", - "10,null,10,10,true,aligned_test10,null,10,10,true,non_aligned_test10", - "11,11.0,11,11,null,null,11.0,11,11,null,null", - "12,12.0,12,12,null,null,12.0,12,12,null,null", - "13,130000.0,130000,130000,true,aligned_unseq_test13,13.0,13,13,null,null", - "14,14.0,14,14,null,null,14.0,14,14,null,null", - "15,15.0,15,15,null,null,15.0,15,15,null,null", - "16,16.0,16,16,null,null,16.0,16,16,null,null", - "17,17.0,17,17,null,null,17.0,17,17,null,null", - "18,18.0,18,18,null,null,18.0,18,18,null,null", - "19,19.0,19,19,null,null,19.0,19,19,null,null", - "20,20.0,20,20,null,null,20.0,20,20,null,null", - "21,null,null,21,true,null,null,null,21,true,null", - "22,null,null,22,true,null,null,null,22,true,null", - "23,230000.0,null,230000,false,null,null,null,23,true,null", - "24,null,null,24,true,null,null,null,24,true,null", - "25,null,null,25,true,null,null,null,25,true,null", - "26,null,null,26,false,null,null,null,26,false,null", - "27,null,null,27,false,null,null,null,27,false,null", - "28,null,null,28,false,null,null,null,28,false,null", - "29,null,null,29,false,null,null,null,29,false,null", - "30,null,null,30,false,null,null,null,30,false,null", - "31,null,31,null,null,aligned_test31,null,31,null,null,non_aligned_test31", - "32,null,32,null,null,aligned_test32,null,32,null,null,non_aligned_test32", - "33,null,33,null,null,aligned_test33,null,33,null,null,non_aligned_test33", - "34,null,34,null,null,aligned_test34,null,34,null,null,non_aligned_test34", - "35,null,35,null,null,aligned_test35,null,35,null,null,non_aligned_test35", - "36,null,36,null,null,aligned_test36,null,36,null,null,non_aligned_test36", - "37,null,37,null,null,aligned_test37,null,37,null,null,non_aligned_test37", - "38,null,38,null,null,aligned_test38,null,38,null,null,non_aligned_test38", - "39,null,39,null,null,aligned_test39,null,39,null,null,non_aligned_test39", - "40,null,40,null,null,aligned_test40,null,40,null,null,non_aligned_test40", + "-40,null,null,null,null,null,null,null,null,null,null,40.0", + "1,1.0,1,1,true,aligned_test1,1.0,1,1,true,non_aligned_test1,null", + "2,2.0,2,2,null,aligned_test2,2.0,2,2,null,non_aligned_test2,null", + "3,30000.0,null,30000,true,aligned_unseq_test3,3.0,null,3,false,non_aligned_test3,null", + "4,4.0,4,null,true,aligned_test4,4.0,4,null,true,non_aligned_test4,null", + "5,5.0,5,null,true,aligned_test5,5.0,5,null,true,non_aligned_test5,null", + "6,6.0,6,6,true,null,6.0,6,6,true,null,null", + "7,7.0,7,7,false,aligned_test7,7.0,7,7,false,non_aligned_test7,null", + "8,8.0,8,8,null,aligned_test8,8.0,8,8,null,non_aligned_test8,null", + "9,9.0,9,9,false,aligned_test9,9.0,9,9,false,non_aligned_test9,null", + "10,null,10,10,true,aligned_test10,null,10,10,true,non_aligned_test10,null", + "11,11.0,11,11,null,null,11.0,11,11,null,null,null", + "12,12.0,12,12,null,null,12.0,12,12,null,null,null", + "13,130000.0,130000,130000,true,aligned_unseq_test13,13.0,13,13,null,null,null", + "14,14.0,14,14,null,null,14.0,14,14,null,null,null", + "15,15.0,15,15,null,null,15.0,15,15,null,null,null", + "16,16.0,16,16,null,null,16.0,16,16,null,null,null", + "17,17.0,17,17,null,null,17.0,17,17,null,null,null", + "18,18.0,18,18,null,null,18.0,18,18,null,null,null", + "19,19.0,19,19,null,null,19.0,19,19,null,null,null", + "20,20.0,20,20,null,null,20.0,20,20,null,null,null", + "21,null,null,21,true,null,null,null,21,true,null,null", + "22,null,null,22,true,null,null,null,22,true,null,null", + "23,230000.0,null,230000,false,null,null,null,23,true,null,null", + "24,null,null,24,true,null,null,null,24,true,null,null", + "25,null,null,25,true,null,null,null,25,true,null,null", + "26,null,null,26,false,null,null,null,26,false,null,null", + "27,null,null,27,false,null,null,null,27,false,null,null", + "28,null,null,28,false,null,null,null,28,false,null,null", + "29,null,null,29,false,null,null,null,29,false,null,null", + "30,null,null,30,false,null,null,null,30,false,null,null", + "31,null,31,null,null,aligned_test31,null,31,null,null,non_aligned_test31,null", + "32,null,32,null,null,aligned_test32,null,32,null,null,non_aligned_test32,null", + "33,null,33,null,null,aligned_test33,null,33,null,null,non_aligned_test33,null", + "34,null,34,null,null,aligned_test34,null,34,null,null,non_aligned_test34,null", + "35,null,35,null,null,aligned_test35,null,35,null,null,non_aligned_test35,null", + "36,null,36,null,null,aligned_test36,null,36,null,null,non_aligned_test36,null", + "37,null,37,null,null,aligned_test37,null,37,null,null,non_aligned_test37,null", + "38,null,38,null,null,aligned_test38,null,38,null,null,non_aligned_test38,null", + "39,null,39,null,null,aligned_test39,null,39,null,null,non_aligned_test39,null", + "40,null,40,null,null,aligned_test40,null,40,null,null,non_aligned_test40,null", }; String[] columnNames = { @@ -217,7 +218,8 @@ public void selectAllAlignedAndNonAlignedTest() { "root.sg1.d2.s2", "root.sg1.d2.s3", "root.sg1.d2.s4", - "root.sg1.d2.s5" + "root.sg1.d2.s5", + "root.sg1.d2.s6", }; try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -766,11 +768,11 @@ public void selectAllAlignedAndNonAlignedTest1() { String[] retArray = new String[] { - "13,130000.0,130000,130000,true,aligned_unseq_test13,13.0,13,13,null,null", - "17,17.0,17,17,null,null,17.0,17,17,null,null", - "18,18.0,18,18,null,null,18.0,18,18,null,null", - "19,19.0,19,19,null,null,19.0,19,19,null,null", - "20,20.0,20,20,null,null,20.0,20,20,null,null", + "13,130000.0,130000,130000,true,aligned_unseq_test13,13.0,13,13,null,null,null", + "17,17.0,17,17,null,null,17.0,17,17,null,null,null", + "18,18.0,18,18,null,null,18.0,18,18,null,null,null", + "19,19.0,19,19,null,null,19.0,19,19,null,null,null", + "20,20.0,20,20,null,null,20.0,20,20,null,null,null", }; String[] columnNames = { @@ -783,7 +785,8 @@ public void selectAllAlignedAndNonAlignedTest1() { "root.sg1.d2.s2", "root.sg1.d2.s3", "root.sg1.d2.s4", - "root.sg1.d2.s5" + "root.sg1.d2.s5", + "root.sg1.d2.s6" }; try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -823,29 +826,29 @@ public void selectAllAlignedAndNonAlignedTest2() { String[] retArray = new String[] { - "3,30000.0,null,30000,true,aligned_unseq_test3,3.0,null,3,false,non_aligned_test3", - "7,7.0,7,7,false,aligned_test7,7.0,7,7,false,non_aligned_test7", - "9,9.0,9,9,false,aligned_test9,9.0,9,9,false,non_aligned_test9", - "13,130000.0,130000,130000,true,aligned_unseq_test13,13.0,13,13,null,null", - "17,17.0,17,17,null,null,17.0,17,17,null,null", - "18,18.0,18,18,null,null,18.0,18,18,null,null", - "19,19.0,19,19,null,null,19.0,19,19,null,null", - "20,20.0,20,20,null,null,20.0,20,20,null,null", - "26,null,null,26,false,null,null,null,26,false,null", - "27,null,null,27,false,null,null,null,27,false,null", - "28,null,null,28,false,null,null,null,28,false,null", - "29,null,null,29,false,null,null,null,29,false,null", - "30,null,null,30,false,null,null,null,30,false,null", - "31,null,31,null,null,aligned_test31,null,31,null,null,non_aligned_test31", - "32,null,32,null,null,aligned_test32,null,32,null,null,non_aligned_test32", - "33,null,33,null,null,aligned_test33,null,33,null,null,non_aligned_test33", - "34,null,34,null,null,aligned_test34,null,34,null,null,non_aligned_test34", - "35,null,35,null,null,aligned_test35,null,35,null,null,non_aligned_test35", - "36,null,36,null,null,aligned_test36,null,36,null,null,non_aligned_test36", - "37,null,37,null,null,aligned_test37,null,37,null,null,non_aligned_test37", - "38,null,38,null,null,aligned_test38,null,38,null,null,non_aligned_test38", - "39,null,39,null,null,aligned_test39,null,39,null,null,non_aligned_test39", - "40,null,40,null,null,aligned_test40,null,40,null,null,non_aligned_test40", + "3,30000.0,null,30000,true,aligned_unseq_test3,3.0,null,3,false,non_aligned_test3,null", + "7,7.0,7,7,false,aligned_test7,7.0,7,7,false,non_aligned_test7,null", + "9,9.0,9,9,false,aligned_test9,9.0,9,9,false,non_aligned_test9,null", + "13,130000.0,130000,130000,true,aligned_unseq_test13,13.0,13,13,null,null,null", + "17,17.0,17,17,null,null,17.0,17,17,null,null,null", + "18,18.0,18,18,null,null,18.0,18,18,null,null,null", + "19,19.0,19,19,null,null,19.0,19,19,null,null,null", + "20,20.0,20,20,null,null,20.0,20,20,null,null,null", + "26,null,null,26,false,null,null,null,26,false,null,null", + "27,null,null,27,false,null,null,null,27,false,null,null", + "28,null,null,28,false,null,null,null,28,false,null,null", + "29,null,null,29,false,null,null,null,29,false,null,null", + "30,null,null,30,false,null,null,null,30,false,null,null", + "31,null,31,null,null,aligned_test31,null,31,null,null,non_aligned_test31,null", + "32,null,32,null,null,aligned_test32,null,32,null,null,non_aligned_test32,null", + "33,null,33,null,null,aligned_test33,null,33,null,null,non_aligned_test33,null", + "34,null,34,null,null,aligned_test34,null,34,null,null,non_aligned_test34,null", + "35,null,35,null,null,aligned_test35,null,35,null,null,non_aligned_test35,null", + "36,null,36,null,null,aligned_test36,null,36,null,null,non_aligned_test36,null", + "37,null,37,null,null,aligned_test37,null,37,null,null,non_aligned_test37,null", + "38,null,38,null,null,aligned_test38,null,38,null,null,non_aligned_test38,null", + "39,null,39,null,null,aligned_test39,null,39,null,null,non_aligned_test39,null", + "40,null,40,null,null,aligned_test40,null,40,null,null,non_aligned_test40,null", }; String[] columnNames = { @@ -858,7 +861,8 @@ public void selectAllAlignedAndNonAlignedTest2() { "root.sg1.d2.s2", "root.sg1.d2.s3", "root.sg1.d2.s4", - "root.sg1.d2.s5" + "root.sg1.d2.s5", + "root.sg1.d2.s6" }; try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -1304,7 +1308,8 @@ public void countAllAlignedWithoutTimeFilterTest() { @Test public void countAllAlignedAndNonAlignedWithoutTimeFilterTest() { - String[] retArray = new String[] {"20", "29", "28", "19", "20", "19", "29", "28", "18", "19"}; + String[] retArray = + new String[] {"20", "29", "28", "19", "20", "19", "29", "28", "18", "19", "1"}; String[] columnNames = { "count(root.sg1.d1.s1)", "count(root.sg1.d1.s2)", @@ -1315,7 +1320,8 @@ public void countAllAlignedAndNonAlignedWithoutTimeFilterTest() { "count(root.sg1.d2.s2)", "count(root.sg1.d2.s3)", "count(root.sg1.d2.s4)", - "count(root.sg1.d2.s5)" + "count(root.sg1.d2.s5)", + "count(root.sg1.d2.s6)", }; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -1850,89 +1856,90 @@ public void selectAllAlignedWithoutValueFilterAlignByDeviceTest() { public void selectAllAlignedAndNonAlignedAlignByDeviceTest() { String[] retArray = new String[] { - "1,root.sg1.d1,1.0,1,1,true,aligned_test1", - "2,root.sg1.d1,2.0,2,2,null,aligned_test2", - "3,root.sg1.d1,30000.0,null,30000,true,aligned_unseq_test3", - "4,root.sg1.d1,4.0,4,null,true,aligned_test4", - "5,root.sg1.d1,5.0,5,null,true,aligned_test5", - "6,root.sg1.d1,6.0,6,6,true,null", - "7,root.sg1.d1,7.0,7,7,false,aligned_test7", - "8,root.sg1.d1,8.0,8,8,null,aligned_test8", - "9,root.sg1.d1,9.0,9,9,false,aligned_test9", - "10,root.sg1.d1,null,10,10,true,aligned_test10", - "11,root.sg1.d1,11.0,11,11,null,null", - "12,root.sg1.d1,12.0,12,12,null,null", - "13,root.sg1.d1,130000.0,130000,130000,true,aligned_unseq_test13", - "14,root.sg1.d1,14.0,14,14,null,null", - "15,root.sg1.d1,15.0,15,15,null,null", - "16,root.sg1.d1,16.0,16,16,null,null", - "17,root.sg1.d1,17.0,17,17,null,null", - "18,root.sg1.d1,18.0,18,18,null,null", - "19,root.sg1.d1,19.0,19,19,null,null", - "20,root.sg1.d1,20.0,20,20,null,null", - "21,root.sg1.d1,null,null,21,true,null", - "22,root.sg1.d1,null,null,22,true,null", - "23,root.sg1.d1,230000.0,null,230000,false,null", - "24,root.sg1.d1,null,null,24,true,null", - "25,root.sg1.d1,null,null,25,true,null", - "26,root.sg1.d1,null,null,26,false,null", - "27,root.sg1.d1,null,null,27,false,null", - "28,root.sg1.d1,null,null,28,false,null", - "29,root.sg1.d1,null,null,29,false,null", - "30,root.sg1.d1,null,null,30,false,null", - "31,root.sg1.d1,null,31,null,null,aligned_test31", - "32,root.sg1.d1,null,32,null,null,aligned_test32", - "33,root.sg1.d1,null,33,null,null,aligned_test33", - "34,root.sg1.d1,null,34,null,null,aligned_test34", - "35,root.sg1.d1,null,35,null,null,aligned_test35", - "36,root.sg1.d1,null,36,null,null,aligned_test36", - "37,root.sg1.d1,null,37,null,null,aligned_test37", - "38,root.sg1.d1,null,38,null,null,aligned_test38", - "39,root.sg1.d1,null,39,null,null,aligned_test39", - "40,root.sg1.d1,null,40,null,null,aligned_test40", - "1,root.sg1.d2,1.0,1,1,true,non_aligned_test1", - "2,root.sg1.d2,2.0,2,2,null,non_aligned_test2", - "3,root.sg1.d2,3.0,null,3,false,non_aligned_test3", - "4,root.sg1.d2,4.0,4,null,true,non_aligned_test4", - "5,root.sg1.d2,5.0,5,null,true,non_aligned_test5", - "6,root.sg1.d2,6.0,6,6,true,null", - "7,root.sg1.d2,7.0,7,7,false,non_aligned_test7", - "8,root.sg1.d2,8.0,8,8,null,non_aligned_test8", - "9,root.sg1.d2,9.0,9,9,false,non_aligned_test9", - "10,root.sg1.d2,null,10,10,true,non_aligned_test10", - "11,root.sg1.d2,11.0,11,11,null,null", - "12,root.sg1.d2,12.0,12,12,null,null", - "13,root.sg1.d2,13.0,13,13,null,null", - "14,root.sg1.d2,14.0,14,14,null,null", - "15,root.sg1.d2,15.0,15,15,null,null", - "16,root.sg1.d2,16.0,16,16,null,null", - "17,root.sg1.d2,17.0,17,17,null,null", - "18,root.sg1.d2,18.0,18,18,null,null", - "19,root.sg1.d2,19.0,19,19,null,null", - "20,root.sg1.d2,20.0,20,20,null,null", - "21,root.sg1.d2,null,null,21,true,null", - "22,root.sg1.d2,null,null,22,true,null", - "23,root.sg1.d2,null,null,23,true,null", - "24,root.sg1.d2,null,null,24,true,null", - "25,root.sg1.d2,null,null,25,true,null", - "26,root.sg1.d2,null,null,26,false,null", - "27,root.sg1.d2,null,null,27,false,null", - "28,root.sg1.d2,null,null,28,false,null", - "29,root.sg1.d2,null,null,29,false,null", - "30,root.sg1.d2,null,null,30,false,null", - "31,root.sg1.d2,null,31,null,null,non_aligned_test31", - "32,root.sg1.d2,null,32,null,null,non_aligned_test32", - "33,root.sg1.d2,null,33,null,null,non_aligned_test33", - "34,root.sg1.d2,null,34,null,null,non_aligned_test34", - "35,root.sg1.d2,null,35,null,null,non_aligned_test35", - "36,root.sg1.d2,null,36,null,null,non_aligned_test36", - "37,root.sg1.d2,null,37,null,null,non_aligned_test37", - "38,root.sg1.d2,null,38,null,null,non_aligned_test38", - "39,root.sg1.d2,null,39,null,null,non_aligned_test39", - "40,root.sg1.d2,null,40,null,null,non_aligned_test40", + "1,root.sg1.d1,1.0,1,1,true,aligned_test1,null", + "2,root.sg1.d1,2.0,2,2,null,aligned_test2,null", + "3,root.sg1.d1,30000.0,null,30000,true,aligned_unseq_test3,null", + "4,root.sg1.d1,4.0,4,null,true,aligned_test4,null", + "5,root.sg1.d1,5.0,5,null,true,aligned_test5,null", + "6,root.sg1.d1,6.0,6,6,true,null,null", + "7,root.sg1.d1,7.0,7,7,false,aligned_test7,null", + "8,root.sg1.d1,8.0,8,8,null,aligned_test8,null", + "9,root.sg1.d1,9.0,9,9,false,aligned_test9,null", + "10,root.sg1.d1,null,10,10,true,aligned_test10,null", + "11,root.sg1.d1,11.0,11,11,null,null,null", + "12,root.sg1.d1,12.0,12,12,null,null,null", + "13,root.sg1.d1,130000.0,130000,130000,true,aligned_unseq_test13,null", + "14,root.sg1.d1,14.0,14,14,null,null,null", + "15,root.sg1.d1,15.0,15,15,null,null,null", + "16,root.sg1.d1,16.0,16,16,null,null,null", + "17,root.sg1.d1,17.0,17,17,null,null,null", + "18,root.sg1.d1,18.0,18,18,null,null,null", + "19,root.sg1.d1,19.0,19,19,null,null,null", + "20,root.sg1.d1,20.0,20,20,null,null,null", + "21,root.sg1.d1,null,null,21,true,null,null", + "22,root.sg1.d1,null,null,22,true,null,null", + "23,root.sg1.d1,230000.0,null,230000,false,null,null", + "24,root.sg1.d1,null,null,24,true,null,null", + "25,root.sg1.d1,null,null,25,true,null,null", + "26,root.sg1.d1,null,null,26,false,null,null", + "27,root.sg1.d1,null,null,27,false,null,null", + "28,root.sg1.d1,null,null,28,false,null,null", + "29,root.sg1.d1,null,null,29,false,null,null", + "30,root.sg1.d1,null,null,30,false,null,null", + "31,root.sg1.d1,null,31,null,null,aligned_test31,null", + "32,root.sg1.d1,null,32,null,null,aligned_test32,null", + "33,root.sg1.d1,null,33,null,null,aligned_test33,null", + "34,root.sg1.d1,null,34,null,null,aligned_test34,null", + "35,root.sg1.d1,null,35,null,null,aligned_test35,null", + "36,root.sg1.d1,null,36,null,null,aligned_test36,null", + "37,root.sg1.d1,null,37,null,null,aligned_test37,null", + "38,root.sg1.d1,null,38,null,null,aligned_test38,null", + "39,root.sg1.d1,null,39,null,null,aligned_test39,null", + "40,root.sg1.d1,null,40,null,null,aligned_test40,null", + "-40,root.sg1.d2,null,null,null,null,null,40.0", + "1,root.sg1.d2,1.0,1,1,true,non_aligned_test1,null", + "2,root.sg1.d2,2.0,2,2,null,non_aligned_test2,null", + "3,root.sg1.d2,3.0,null,3,false,non_aligned_test3,null", + "4,root.sg1.d2,4.0,4,null,true,non_aligned_test4,null", + "5,root.sg1.d2,5.0,5,null,true,non_aligned_test5,null", + "6,root.sg1.d2,6.0,6,6,true,null,null", + "7,root.sg1.d2,7.0,7,7,false,non_aligned_test7,null", + "8,root.sg1.d2,8.0,8,8,null,non_aligned_test8,null", + "9,root.sg1.d2,9.0,9,9,false,non_aligned_test9,null", + "10,root.sg1.d2,null,10,10,true,non_aligned_test10,null", + "11,root.sg1.d2,11.0,11,11,null,null,null", + "12,root.sg1.d2,12.0,12,12,null,null,null", + "13,root.sg1.d2,13.0,13,13,null,null,null", + "14,root.sg1.d2,14.0,14,14,null,null,null", + "15,root.sg1.d2,15.0,15,15,null,null,null", + "16,root.sg1.d2,16.0,16,16,null,null,null", + "17,root.sg1.d2,17.0,17,17,null,null,null", + "18,root.sg1.d2,18.0,18,18,null,null,null", + "19,root.sg1.d2,19.0,19,19,null,null,null", + "20,root.sg1.d2,20.0,20,20,null,null,null", + "21,root.sg1.d2,null,null,21,true,null,null", + "22,root.sg1.d2,null,null,22,true,null,null", + "23,root.sg1.d2,null,null,23,true,null,null", + "24,root.sg1.d2,null,null,24,true,null,null", + "25,root.sg1.d2,null,null,25,true,null,null", + "26,root.sg1.d2,null,null,26,false,null,null", + "27,root.sg1.d2,null,null,27,false,null,null", + "28,root.sg1.d2,null,null,28,false,null,null", + "29,root.sg1.d2,null,null,29,false,null,null", + "30,root.sg1.d2,null,null,30,false,null,null", + "31,root.sg1.d2,null,31,null,null,non_aligned_test31,null", + "32,root.sg1.d2,null,32,null,null,non_aligned_test32,null", + "33,root.sg1.d2,null,33,null,null,non_aligned_test33,null", + "34,root.sg1.d2,null,34,null,null,non_aligned_test34,null", + "35,root.sg1.d2,null,35,null,null,non_aligned_test35,null", + "36,root.sg1.d2,null,36,null,null,non_aligned_test36,null", + "37,root.sg1.d2,null,37,null,null,non_aligned_test37,null", + "38,root.sg1.d2,null,38,null,null,non_aligned_test38,null", + "39,root.sg1.d2,null,39,null,null,non_aligned_test39,null", + "40,root.sg1.d2,null,40,null,null,non_aligned_test40,null", }; - String[] columnNames = {"Device", "s1", "s2", "s3", "s4", "s5"}; + String[] columnNames = {"Device", "s1", "s2", "s3", "s4", "s5", "s6"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -2518,9 +2525,10 @@ public void countAllAlignedWithoutTimeFilterAlignByDeviceTest() { @Test public void countAllAlignedAndNonAlignedWithoutTimeFilterAlignByDeviceTest() { - String[] retArray = new String[] {"root.sg1.d1,20,29,28,19,20,", "root.sg1.d2,19,29,28,18,19,"}; + String[] retArray = + new String[] {"root.sg1.d1,20,29,28,19,20,null,", "root.sg1.d2,19,29,28,18,19,1,"}; String[] columnNames = { - "Device", "count(s1)", "count(s2)", "count(s3)", "count(s4)", "count(s5)" + "Device", "count(s1)", "count(s2)", "count(s3)", "count(s4)", "count(s5)", "count(s6)" }; try (Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletionIT.java index f02e2fd6ec8ae..cd2ce061c0031 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithDeletionIT.java @@ -125,7 +125,8 @@ public void selectAllAlignedAndNonAlignedLastTest() { "40,root.sg1.d2.s2,40,INT32", "30,root.sg1.d2.s3,30,INT64", "30,root.sg1.d2.s4,false,BOOLEAN", - "40,root.sg1.d2.s5,non_aligned_test40,TEXT")); + "40,root.sg1.d2.s5,non_aligned_test40,TEXT", + "-40,root.sg1.d2.s6,40.0,DOUBLE")); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheIT.java index 25b6a2098c9c3..b9b8cd24bc17f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBLastQueryWithoutLastCacheIT.java @@ -122,7 +122,8 @@ public void selectAllAlignedAndNonAlignedLastTest() { "40,root.sg1.d2.s2,40,INT32", "30,root.sg1.d2.s3,30,INT64", "30,root.sg1.d2.s4,false,BOOLEAN", - "40,root.sg1.d2.s5,non_aligned_test40,TEXT")); + "40,root.sg1.d2.s5,non_aligned_test40,TEXT", + "-40,root.sg1.d2.s6,40.0,DOUBLE")); try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBRawQueryWithoutValueFilterWithDeletionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBRawQueryWithoutValueFilterWithDeletionIT.java index 5bd02dbacacc8..67787a8ddd233 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBRawQueryWithoutValueFilterWithDeletionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aligned/IoTDBRawQueryWithoutValueFilterWithDeletionIT.java @@ -155,46 +155,47 @@ public void selectAllAlignedAndNonAlignedTest() { String[] retArray = new String[] { - "1,null,1,true,aligned_test1,1.0,1,1,true,non_aligned_test1", - "2,null,2,null,aligned_test2,2.0,2,2,null,non_aligned_test2", - "3,null,30000,true,aligned_unseq_test3,3.0,null,3,false,non_aligned_test3", - "4,null,null,true,aligned_test4,4.0,4,null,true,non_aligned_test4", - "5,null,null,true,aligned_test5,5.0,5,null,true,non_aligned_test5", - "6,null,6,true,null,6.0,6,6,true,null", - "7,null,7,false,aligned_test7,7.0,7,7,false,non_aligned_test7", - "8,null,8,null,aligned_test8,8.0,8,8,null,non_aligned_test8", - "9,null,9,false,aligned_test9,9.0,9,9,false,non_aligned_test9", - "10,null,10,true,aligned_test10,null,10,10,true,non_aligned_test10", - "11,null,11,null,null,11.0,11,11,null,null", - "12,null,12,null,null,12.0,12,12,null,null", - "13,null,130000,true,aligned_unseq_test13,13.0,13,13,null,null", - "14,null,14,null,null,14.0,14,14,null,null", - "15,null,15,null,null,15.0,15,15,null,null", - "16,null,16,null,null,16.0,16,16,null,null", - "17,null,17,null,null,17.0,17,17,null,null", - "18,null,18,null,null,18.0,18,18,null,null", - "19,null,19,null,null,19.0,19,19,null,null", - "20,null,20,null,null,20.0,20,20,null,null", - "21,null,21,true,null,null,null,21,true,null", - "22,null,22,true,null,null,null,22,true,null", - "23,230000.0,230000,false,null,null,null,23,true,null", - "24,null,24,true,null,null,null,24,true,null", - "25,null,25,true,null,null,null,25,true,null", - "26,null,26,false,null,null,null,26,false,null", - "27,null,27,false,null,null,null,27,false,null", - "28,null,28,false,null,null,null,28,false,null", - "29,null,29,false,null,null,null,29,false,null", - "30,null,30,false,null,null,null,30,false,null", - "31,null,null,null,null,null,31,null,null,non_aligned_test31", - "32,null,null,null,aligned_test32,null,32,null,null,non_aligned_test32", - "33,null,null,null,aligned_test33,null,33,null,null,non_aligned_test33", - "34,null,null,null,aligned_test34,null,34,null,null,non_aligned_test34", - "35,null,null,null,aligned_test35,null,35,null,null,non_aligned_test35", - "36,null,null,null,aligned_test36,null,36,null,null,non_aligned_test36", - "37,null,null,null,aligned_test37,null,37,null,null,non_aligned_test37", - "38,null,null,null,aligned_test38,null,38,null,null,non_aligned_test38", - "39,null,null,null,aligned_test39,null,39,null,null,non_aligned_test39", - "40,null,null,null,aligned_test40,null,40,null,null,non_aligned_test40", + "-40,null,null,null,null,null,null,null,null,null,40.0", + "1,null,1,true,aligned_test1,1.0,1,1,true,non_aligned_test1,null", + "2,null,2,null,aligned_test2,2.0,2,2,null,non_aligned_test2,null", + "3,null,30000,true,aligned_unseq_test3,3.0,null,3,false,non_aligned_test3,null", + "4,null,null,true,aligned_test4,4.0,4,null,true,non_aligned_test4,null", + "5,null,null,true,aligned_test5,5.0,5,null,true,non_aligned_test5,null", + "6,null,6,true,null,6.0,6,6,true,null,null", + "7,null,7,false,aligned_test7,7.0,7,7,false,non_aligned_test7,null", + "8,null,8,null,aligned_test8,8.0,8,8,null,non_aligned_test8,null", + "9,null,9,false,aligned_test9,9.0,9,9,false,non_aligned_test9,null", + "10,null,10,true,aligned_test10,null,10,10,true,non_aligned_test10,null", + "11,null,11,null,null,11.0,11,11,null,null,null", + "12,null,12,null,null,12.0,12,12,null,null,null", + "13,null,130000,true,aligned_unseq_test13,13.0,13,13,null,null,null", + "14,null,14,null,null,14.0,14,14,null,null,null", + "15,null,15,null,null,15.0,15,15,null,null,null", + "16,null,16,null,null,16.0,16,16,null,null,null", + "17,null,17,null,null,17.0,17,17,null,null,null", + "18,null,18,null,null,18.0,18,18,null,null,null", + "19,null,19,null,null,19.0,19,19,null,null,null", + "20,null,20,null,null,20.0,20,20,null,null,null", + "21,null,21,true,null,null,null,21,true,null,null", + "22,null,22,true,null,null,null,22,true,null,null", + "23,230000.0,230000,false,null,null,null,23,true,null,null", + "24,null,24,true,null,null,null,24,true,null,null", + "25,null,25,true,null,null,null,25,true,null,null", + "26,null,26,false,null,null,null,26,false,null,null", + "27,null,27,false,null,null,null,27,false,null,null", + "28,null,28,false,null,null,null,28,false,null,null", + "29,null,29,false,null,null,null,29,false,null,null", + "30,null,30,false,null,null,null,30,false,null,null", + "31,null,null,null,null,null,31,null,null,non_aligned_test31,null", + "32,null,null,null,aligned_test32,null,32,null,null,non_aligned_test32,null", + "33,null,null,null,aligned_test33,null,33,null,null,non_aligned_test33,null", + "34,null,null,null,aligned_test34,null,34,null,null,non_aligned_test34,null", + "35,null,null,null,aligned_test35,null,35,null,null,non_aligned_test35,null", + "36,null,null,null,aligned_test36,null,36,null,null,non_aligned_test36,null", + "37,null,null,null,aligned_test37,null,37,null,null,non_aligned_test37,null", + "38,null,null,null,aligned_test38,null,38,null,null,non_aligned_test38,null", + "39,null,null,null,aligned_test39,null,39,null,null,non_aligned_test39,null", + "40,null,null,null,aligned_test40,null,40,null,null,non_aligned_test40,null", }; String[] columnNames = { @@ -206,7 +207,8 @@ public void selectAllAlignedAndNonAlignedTest() { "root.sg1.d2.s2", "root.sg1.d2.s3", "root.sg1.d2.s4", - "root.sg1.d2.s5" + "root.sg1.d2.s5", + "root.sg1.d2.s6" }; try (Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryAliasIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryAliasIT.java index 4772804f38480..3e83a4ab5a290 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryAliasIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/last/IoTDBLastQueryAliasIT.java @@ -92,9 +92,13 @@ public void nonAlignedTest() { "1,root.test.d1.alias1,3,INT32,", }; resultSetEqualTest( - "select last alias3,alias2,alias1 from root.test.d1", expectedHeader, retArray); + "select last alias3,alias2,alias1 from root.test.d1 order by timeseries desc", + expectedHeader, + retArray); resultSetEqualTest( - "select last alias3,alias2,alias1 from root.test.d1", expectedHeader, retArray); + "select last alias3,alias2,alias1 from root.test.d1 order by timeseries desc", + expectedHeader, + retArray); retArray = new String[] { @@ -123,9 +127,13 @@ public void alignedTest() { "2,root.test.d2.alias1,4,INT32,", }; resultSetEqualTest( - "select last alias3,alias2,alias1 from root.test.d2", expectedHeader, retArray); + "select last alias3,alias2,alias1 from root.test.d2 order by timeseries desc", + expectedHeader, + retArray); resultSetEqualTest( - "select last alias3,alias2,alias1 from root.test.d2", expectedHeader, retArray); + "select last alias3,alias2,alias1 from root.test.d2 order by timeseries desc", + expectedHeader, + retArray); retArray = new String[] { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java index 7ed80f7ab4ad6..8930985717f5d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/quotas/IoTDBSpaceQuotaIT.java @@ -26,6 +26,7 @@ import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -40,6 +41,7 @@ import java.sql.Statement; import java.util.Arrays; import java.util.List; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -297,8 +299,6 @@ public void setSpaceQuotaTest5() { "create timeseries root.sg0.wf01.wt02.status0 with datatype=BOOLEAN,encoding=PLAIN;"); adminStmt.execute( "create timeseries root.sg0.wf01.wt02.status1 with datatype=BOOLEAN,encoding=PLAIN;"); - Thread.sleep(2000); - ResultSet resultSet1 = adminStmt.executeQuery("show space quota root.sg0;"); String ans1 = "root.sg0,diskSize,0.09765625G,0.0G" + ",\n" @@ -306,8 +306,8 @@ public void setSpaceQuotaTest5() { + ",\n" + "root.sg0,timeSeriesNum,5,3" + ",\n"; - validateResultSet(resultSet1, ans1); - } catch (InterruptedException | SQLException e) { + validateResultSetEventually(adminStmt, "show space quota root.sg0;", ans1); + } catch (SQLException e) { Assert.fail(e.getMessage()); } } @@ -509,6 +509,22 @@ public void showSpaceQuotaTest1() throws SQLException { } } + private void validateResultSetEventually(Statement statement, String sql, String ans) { + Awaitility.await() + .pollInSameThread() + .pollDelay(0, TimeUnit.MILLISECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .atMost(30, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try { + validateResultSet(statement.executeQuery(sql), ans); + } catch (SQLException e) { + Assert.fail(e.getMessage()); + } + }); + } + private void validateResultSet(ResultSet set, String ans) throws SQLException { try { StringBuilder builder = new StringBuilder(); @@ -521,12 +537,12 @@ private void validateResultSet(ResultSet set, String ans) throws SQLException { builder.append("\n"); } String result = builder.toString(); - assertEquals(ans.length(), result.length()); + assertEquals(result, ans.length(), result.length()); List ansLines = Arrays.asList(ans.split("\n")); List resultLines = Arrays.asList(result.split("\n")); - assertEquals(ansLines.size(), resultLines.size()); + assertEquals(result, ansLines.size(), resultLines.size()); for (String resultLine : resultLines) { - assertTrue(ansLines.contains(resultLine)); + assertTrue(result, ansLines.contains(resultLine)); } } finally { set.close(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBMetadataFetchIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBMetadataFetchIT.java index 2f1d4e5bed406..4d0c873664d1f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBMetadataFetchIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBMetadataFetchIT.java @@ -26,6 +26,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runners.Parameterized; @@ -494,6 +495,26 @@ public void showCountTimeSeries() throws SQLException { } } + @Test + @Ignore + public void showCountTimeSeriesExcludeInternalDatabaseAndIncludeView() throws SQLException { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + final long baseVisibleCount = queryCount(statement, "COUNT TIMESERIES root.ln*.**"); + statement.execute("CREATE DATABASE root.count_it"); + statement.execute( + "CREATE TIMESERIES root.count_it.src.s1 WITH DATATYPE = INT32, ENCODING = PLAIN"); + statement.execute( + "CREATE TIMESERIES root.count_it.src.s2 WITH DATATYPE = INT32, ENCODING = PLAIN"); + statement.execute("CREATE VIEW root.count_it.dst.v1 AS SELECT s1 FROM root.count_it.src;"); + + final long localCount = queryCount(statement, "COUNT TIMESERIES root.count_it.**"); + assertEquals(3L, localCount); + assertEquals( + baseVisibleCount + localCount, queryCount(statement, "COUNT TIMESERIES root.**")); + } + } + @Test public void showCountTimeSeriesWithTag() throws SQLException { try (Connection connection = EnvFactory.getEnv().getConnection(); @@ -865,4 +886,11 @@ public void showDeadbandInfo() throws SQLException { } } } + + private long queryCount(final Statement statement, final String sql) throws SQLException { + try (ResultSet resultSet = statement.executeQuery(sql)) { + Assert.assertTrue(resultSet.next()); + return resultSet.getLong(1); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveSchemaQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveSchemaQueryIT.java index 853ef3c87ae00..f062994fa3d46 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveSchemaQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveSchemaQueryIT.java @@ -27,6 +27,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runners.Parameterized; @@ -235,6 +236,52 @@ public void testShowTimeSeries() { } } + @Test + @Ignore + public void testCountTimeSeriesWithTimeConditionIncludesView() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.view_count"); + statement.execute( + "CREATE TIMESERIES root.view_count.src.s1 WITH DATATYPE = INT32, ENCODING = PLAIN"); + statement.execute( + "CREATE TIMESERIES root.view_count.src.s2 WITH DATATYPE = INT32, ENCODING = PLAIN"); + statement.execute("CREATE VIEW root.view_count.dst.v1 AS SELECT s1 FROM root.view_count.src"); + + checkResultSet( + statement, + "count timeseries root.view_count.**", + new HashSet<>(Collections.singletonList("3,"))); + + statement.execute("insert into root.view_count.src(timestamp,s1) values(1,1)"); + + checkResultSet( + statement, + "count timeseries root.view_count.** where time>0", + new HashSet<>(Collections.singletonList("2,"))); + checkResultSet( + statement, + "show timeseries root.view_count.** where time>0", + new HashSet<>( + Arrays.asList( + "root.view_count.src.s1,null,root.view_count,INT32,PLAIN,LZ4,null,null,null,null,BASE,", + "root.view_count.dst.v1,null,root.view_count,INT32,null,null,null,null,null,null,VIEW,"))); + checkResultSet( + statement, + "count timeseries root.view_count.dst.** where time>0", + new HashSet<>(Collections.singletonList("1,"))); + checkResultSet( + statement, + "show timeseries root.view_count.dst.** where time>0", + new HashSet<>( + Collections.singletonList( + "root.view_count.dst.v1,null,root.view_count,INT32,null,null,null,null,null,null,VIEW,"))); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + @Test public void testShowDevices() { try (Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/AlignedWriteUtil.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/AlignedWriteUtil.java index 798f1b2b889d4..b0cee529f0da0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/AlignedWriteUtil.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/AlignedWriteUtil.java @@ -44,6 +44,7 @@ public class AlignedWriteUtil { "create timeseries root.sg1.d2.s3 WITH DATATYPE=INT64", "create timeseries root.sg1.d2.s4 WITH DATATYPE=BOOLEAN", "create timeseries root.sg1.d2.s5 WITH DATATYPE=TEXT", + "create timeseries root.sg1.d2.s6 WITH DATATYPE=DOUBLE", "insert into root.sg1.d1(time, s1, s2, s3, s4, s5) aligned values(1, 1.0, 1, 1, TRUE, 'aligned_test1')", "insert into root.sg1.d1(time, s1, s2, s3, s5) aligned values(2, 2.0, 2, 2, 'aligned_test2')", "insert into root.sg1.d1(time, s1, s3, s4, s5) aligned values(3, 3.0, 3, FALSE, 'aligned_test3')", @@ -130,6 +131,7 @@ public class AlignedWriteUtil { "insert into root.sg1.d2(time, s2, s5) values(38, 38, 'non_aligned_test38')", "insert into root.sg1.d2(time, s2, s5) values(39, 39, 'non_aligned_test39')", "insert into root.sg1.d2(time, s2, s5) values(40, 40, 'non_aligned_test40')", + "insert into root.sg1.d2(time, s6) values(-40, 40)", }; public static void insertData() { diff --git a/integration-test/src/test/java/org/apache/iotdb/libudf/it/dlearn/DLearnIT.java b/integration-test/src/test/java/org/apache/iotdb/libudf/it/dlearn/DLearnIT.java index 480dff4e46dca..b4a1c8daa92ab 100644 --- a/integration-test/src/test/java/org/apache/iotdb/libudf/it/dlearn/DLearnIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/libudf/it/dlearn/DLearnIT.java @@ -94,6 +94,11 @@ private static void createTimeSeries() { + "datatype=double, " + "encoding=plain, " + "compression=uncompressed"); + statement.addBatch( + "create timeseries root.vehicle.d3.s1 with " + + "datatype=int32, " + + "encoding=plain, " + + "compression=uncompressed"); statement.executeBatch(); } catch (SQLException throwable) { fail(throwable.getMessage()); @@ -168,6 +173,16 @@ private static void generateData() { String.format( "insert into root.vehicle.d2(timestamp,s1,s2,s3,s4) values(%d,%d,%d,%d,%d)", 900, 4, 4, 4, 4)); + // Toy series for cluster UDF (l=3, k=2): windows [1,2,3], [10,20,30], [1,5,1]. With + // norm=false, + // k-means groups the first two windows; k-shape / medoidshape group windows 0 and 2 + // (shape-related). + int[] toy = {1, 2, 3, 10, 20, 30, 1, 5, 1}; + for (int i = 0; i < toy.length; i++) { + statement.addBatch( + String.format( + "insert into root.vehicle.d3(timestamp,s1) values(%d,%d)", (i + 1) * 100, toy[i])); + } statement.executeBatch(); } catch (SQLException throwable) { fail(throwable.getMessage()); @@ -179,6 +194,7 @@ private static void registerUDF() { Statement statement = connection.createStatement()) { statement.execute("create function iqr as 'org.apache.iotdb.library.anomaly.UDTFIQR'"); statement.execute("create function ar as 'org.apache.iotdb.library.dlearn.UDTFAR'"); + statement.execute("create function cluster as 'org.apache.iotdb.library.dlearn.UDTFCluster'"); } catch (SQLException throwable) { fail(throwable.getMessage()); } @@ -308,4 +324,70 @@ public void testAR4() { fail(throwable.getMessage()); } } + + @Test + public void testCluster1() { + String sqlStr = + "select cluster(d3.s1, 'l'='3', 'k'='2', 'method'='kmeans', 'norm'='false', " + + "'maxiter'='50', 'output'='label') from root.vehicle"; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sqlStr)) { + resultSet.next(); + int l0 = resultSet.getInt(2); + resultSet.next(); + int l1 = resultSet.getInt(2); + resultSet.next(); + int l2 = resultSet.getInt(2); + Assert.assertFalse(resultSet.next()); + Assert.assertEquals(l0, l2); + Assert.assertNotEquals(l0, l1); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testCluster2() { + String sqlStr = + "select cluster(d3.s1, 'l'='3', 'k'='2', 'method'='kshape', 'norm'='true', " + + "'maxiter'='50', 'output'='label') from root.vehicle"; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sqlStr)) { + resultSet.next(); + int l0 = resultSet.getInt(2); + resultSet.next(); + int l1 = resultSet.getInt(2); + resultSet.next(); + int l2 = resultSet.getInt(2); + Assert.assertFalse(resultSet.next()); + Assert.assertEquals(l0, l1); + Assert.assertNotEquals(l0, l2); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testCluster3() { + String sqlStr = + "select cluster(d3.s1, 'l'='3', 'k'='2', 'method'='medoidshape', 'norm'='true', " + + "'sample_rate'='1', 'maxiter'='50', 'output'='label') from root.vehicle"; + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sqlStr)) { + resultSet.next(); + int l0 = resultSet.getInt(2); + resultSet.next(); + int l1 = resultSet.getInt(2); + resultSet.next(); + int l2 = resultSet.getInt(2); + Assert.assertFalse(resultSet.next()); + Assert.assertEquals(l0, l1); + Assert.assertNotEquals(l0, l2); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java index 3e4200daf323a..b964415938650 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java @@ -534,4 +534,49 @@ public void testAlterPipeSourceAndProcessor() { TestUtils.assertDataEventuallyOnEnv( receiverEnv, "count timeSeries", "count(timeseries),", Collections.singleton("3,")); } + + @Test + public void testAlterPipeRealtime() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Insert data on sender + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "insert into root.db.d1 (time, at1) values (1000, 1), (1500, 2), (2000, 3), (2500, 4), (3000, 5)", + "flush"), + null); + + // Create pipe + final String sql = + String.format( + "create pipe a2b with source ('history.enable'='false') with sink ('node-urls'='%s')", + receiverDataNode.getIpAndPortString()); + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute(sql); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "count timeSeries root.db.**", + "count(timeseries),", + Collections.singleton("0,")); + + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement()) { + statement.execute("alter pipe a2b modify source ('history.enable'='true')"); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "count timeSeries root.db.**", + "count(timeseries),", + Collections.singleton("1,")); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java index dc1d1fc93aec3..09657e2deb2db 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java @@ -21,8 +21,10 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; @@ -33,59 +35,67 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2AutoCreateSchema.class}) public class IoTDBPipeAutoDropIT extends AbstractPipeDualAutoIT { + @Override + protected void setupConfig() { + super.setupConfig(); + senderEnv.getConfig().getCommonConfig().setPipeAutoSplitFullEnabled(true); + receiverEnv.getConfig().getCommonConfig().setPipeAutoSplitFullEnabled(true); + senderEnv.getConfig().getConfigNodeConfig().setLeaderDistributionPolicy("HASH"); + } + @Test public void testAutoDropInHistoricalTransfer() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - TestUtils.executeNonQuery(senderEnv, "insert into root.db.d1(time,s1) values (1,1)", null); - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.mode", "query"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("p1", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("p1").getCode()); - - TestUtils.assertDataEventuallyOnEnv( - receiverEnv, - "select count(*) from root.**", - "count(root.db.d1.s1),", - Collections.singleton("1,")); - - TestUtils.assertDataEventuallyOnEnv( - senderEnv, - "show pipes", - "ID,CreationTime,State,PipeSource,PipeProcessor,PipeSink,ExceptionMessage,RemainingEventCount,EstimatedRemainingSeconds,", - Collections.emptySet()); - } + TestUtils.executeNonQuery( + senderEnv, + String.format( + "create pipe a2b with sink ('node-urls'='%s')", receiverDataNode.getIpAndPortString()), + null); + + TestUtils.executeNonQuery(senderEnv, "insert into root.db.d1(time,s1) values (1,1)", null); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select count(*) from root.**", + "count(root.db.d1.s1),", + Collections.singleton("1,")); + + await() + .pollInSameThread() + .pollDelay(1L, TimeUnit.SECONDS) + .pollInterval(1L, TimeUnit.SECONDS) + .atMost(600, TimeUnit.SECONDS) + .untilAsserted( + () -> { + try (final Connection connection = senderEnv.getConnection(); + final Statement statement = connection.createStatement(); + final ResultSet result = statement.executeQuery("show pipes")) { + int pipeNum = 0; + while (result.next()) { + final String pipeName = result.getString(ColumnHeaderConstant.ID); + if (!pipeName.contains(PipeStaticMeta.CONSENSUS_PIPE_PREFIX) + && pipeName.endsWith("_history")) { + pipeNum++; + } + } + Assert.assertEquals(0, pipeNum); + } + }); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoSplitIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoSplitIT.java index 5f732b6d390c5..ced1e9ee13ce5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoSplitIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoSplitIT.java @@ -23,6 +23,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; @@ -37,6 +38,8 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -78,13 +81,12 @@ public void setUp() { public void testSingleEnv() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - final String sql = - String.format( - "create pipe a2b with source ('source'='iotdb-source') with processor ('processor'='do-nothing-processor') with sink ('node-urls'='%s')", - receiverDataNode.getIpAndPortString()); try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { - statement.execute(sql); + statement.execute( + String.format( + "create pipe a2b with sink ('node-urls'='%s')", + receiverDataNode.getIpAndPortString())); } catch (final SQLException e) { fail(e.getMessage()); } @@ -100,5 +102,48 @@ public void testSingleEnv() throws Exception { || (Objects.equals(showPipeResult.get(1).id, "a2b_history") && Objects.equals(showPipeResult.get(0).id, "a2b_realtime"))); } + + // Do not split for pipes without insertion or non-full + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "drop pipe a2b_history", + "drop pipe a2b_realtime", + String.format( + "create pipe a2b1 with source ('inclusion'='schema') with sink ('node-urls'='%s')", + receiverDataNode.getIpAndPortString()), + String.format( + "create pipe a2b2 with source ('realtime.enable'='false') with sink ('node-urls'='%s')", + receiverDataNode.getIpAndPortString()), + String.format( + "create pipe a2b3 with source ('history.enable'='false') with sink ('node-urls'='%s')", + receiverDataNode.getIpAndPortString())), + null); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + showPipeResult.removeIf(i -> i.getId().startsWith("__consensus")); + Assert.assertEquals(3, showPipeResult.size()); + } + + TestUtils.executeNonQueries( + senderEnv, + Arrays.asList( + "drop pipe a2b1", + "drop pipe a2b2", + "drop pipe a2b3", + "insert into root.test.device(time, field) values(0,1),(1,2)", + "delete from root.test.device.* where time == 0", + String.format( + "create pipe a2b with source ('inclusion'='all') with sink ('node-urls'='%s')", + receiverDataNode.getIpAndPortString())), + null); + + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, + "select * from root.test.device", + "Time,root.test.device.field,", + Collections.singleton("1,2.0,")); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java index 9617de89cd70a..16e21fdb2956d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java @@ -19,11 +19,13 @@ package org.apache.iotdb.pipe.it.autocreate; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.commons.cluster.RegionRoleType; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; @@ -35,6 +37,7 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; @@ -49,18 +52,22 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static org.awaitility.Awaitility.await; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2AutoCreateSchema.class}) public class IoTDBPipeClusterIT extends AbstractPipeDualAutoIT { + private static final double SYNC_LAG_DELTA = 0.001; + @Override @Before public void setUp() { @@ -77,6 +84,10 @@ public void setUp() { .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); + senderEnv + .getConfig() + .getDataNodeConfig() + .setMetricReporterType(Collections.singletonList("PROMETHEUS")); receiverEnv .getConfig() @@ -89,6 +100,10 @@ public void setUp() { .setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS) .setPipeMemoryManagementEnabled(false) .setIsPipeEnableMemoryCheck(false); + receiverEnv + .getConfig() + .getDataNodeConfig() + .setMetricReporterType(Collections.singletonList("PROMETHEUS")); // 10 min, assert that the operations will not time out senderEnv.getConfig().getCommonConfig().setDnConnectionTimeoutMs(600000); @@ -301,41 +316,10 @@ public void testPipeAfterDataRegionLeaderStop() throws Exception { TestUtils.executeNonQueries( senderEnv, Arrays.asList("insert into root.db.d1(time,s1) values (1,1)", "flush"), null); + flushTreeDataRegionReplicasAfterReplicationComplete( + senderEnv, Collections.singletonList("root.db")); - final AtomicInteger leaderPort = new AtomicInteger(-1); - final TShowRegionResp showRegionResp = client.showRegion(new TShowRegionReq()); - showRegionResp - .getRegionInfoList() - .forEach( - regionInfo -> { - if (RegionRoleType.Leader.getRoleType().equals(regionInfo.getRoleType())) { - leaderPort.set(regionInfo.getClientRpcPort()); - } - }); - - int leaderIndex = -1; - for (int i = 0; i < 3; ++i) { - if (senderEnv.getDataNodeWrapper(i).getPort() == leaderPort.get()) { - leaderIndex = i; - try { - senderEnv.shutdownDataNode(i); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - try { - TimeUnit.SECONDS.sleep(1); - } catch (final InterruptedException ignored) { - } - try { - senderEnv.startDataNode(i); - ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); - } catch (final Throwable e) { - e.printStackTrace(); - return; - } - } - } + final int leaderIndex = restartTreeDataRegionLeader(client, "root.db"); if (leaderIndex == -1) { // ensure the leader is stopped fail(); } @@ -352,6 +336,10 @@ public void testPipeAfterDataRegionLeaderStop() throws Exception { "select count(*) from root.db.d1", "count(root.db.d1.s1),", Collections.singleton("2,")); + flushTreeDataRegionReplicasAfterReplicationComplete( + senderEnv, Collections.singletonList("root.db")); + flushTreeDataRegionReplicasAfterReplicationComplete( + receiverEnv, Collections.singletonList("root.db")); } try { @@ -397,6 +385,147 @@ public void testPipeAfterDataRegionLeaderStop() throws Exception { } } + private int restartTreeDataRegionLeader( + final SyncConfigNodeIServiceClient client, final String database) throws TException { + final List leaderRegionInfoList = + showTreeDataRegionLeaders(Collections.singletonList(database), client); + if (leaderRegionInfoList.isEmpty()) { + return -1; + } + + final TRegionInfo targetRegionInfo = + leaderRegionInfoList.stream() + .min(Comparator.comparingInt(regionInfo -> regionInfo.getConsensusGroupId().getId())) + .orElse(null); + if (targetRegionInfo == null) { + return -1; + } + + final int leaderPort = targetRegionInfo.getClientRpcPort(); + for (int i = 0; i < senderEnv.getDataNodeWrapperList().size(); ++i) { + if (senderEnv.getDataNodeWrapper(i).getPort() != leaderPort) { + continue; + } + + try { + senderEnv.shutdownDataNode(i); + } catch (final Throwable e) { + e.printStackTrace(); + return -1; + } + + try { + TimeUnit.SECONDS.sleep(1); + } catch (final InterruptedException ignored) { + Thread.currentThread().interrupt(); + return -1; + } + + try { + senderEnv.startDataNode(i); + ((AbstractEnv) senderEnv).checkClusterStatusWithoutUnknown(); + } catch (final Throwable e) { + e.printStackTrace(); + return -1; + } + return i; + } + return -1; + } + + private void flushTreeDataRegionReplicasAfterReplicationComplete( + final BaseEnv env, final List databases) { + waitForTreeDataRegionReplicationComplete(env, databases); + TestUtils.executeNonQueryWithRetry(env, "flush"); + waitForTreeDataRegionReplicationComplete(env, databases); + } + + private void waitForTreeDataRegionReplicationComplete( + final BaseEnv env, final List databases) { + await() + .pollInterval(500, TimeUnit.MILLISECONDS) + .atMost(2, TimeUnit.MINUTES) + .untilAsserted( + () -> { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { + final List leaderRegionInfoList = + showTreeDataRegionLeaders(databases, client); + Assert.assertFalse( + "No tree DataRegion leader found for databases " + databases, + leaderRegionInfoList.isEmpty()); + + for (final TRegionInfo regionInfo : leaderRegionInfoList) { + final DataNodeWrapper leaderNode = + findDataNodeWrapperByPort(env, regionInfo.getClientRpcPort()); + final String metricsUrl = + "http://" + + leaderNode.getIp() + + ":" + + leaderNode.getMetricPort() + + "/metrics"; + final String metricsContent = env.getUrlContent(metricsUrl, null); + Assert.assertNotNull( + "Failed to fetch metrics from leader DataNode at " + metricsUrl, + metricsContent); + assertSyncLagIsZero(metricsContent, buildDataRegionTag(regionInfo), metricsUrl); + } + } + }); + } + + private List showTreeDataRegionLeaders( + final List databases, final SyncConfigNodeIServiceClient client) throws TException { + final TShowRegionResp showRegionResp = + client.showRegion( + new TShowRegionReq() + .setConsensusGroupType(TConsensusGroupType.DataRegion) + .setDatabases(databases)); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), showRegionResp.getStatus().getCode()); + final List result = new ArrayList<>(); + for (final TRegionInfo regionInfo : showRegionResp.getRegionInfoList()) { + if (RegionRoleType.Leader.getRoleType().equals(regionInfo.getRoleType())) { + result.add(regionInfo); + } + } + return result; + } + + private DataNodeWrapper findDataNodeWrapperByPort(final BaseEnv env, final int port) { + for (final DataNodeWrapper dataNodeWrapper : env.getDataNodeWrapperList()) { + if (dataNodeWrapper.getPort() == port) { + return dataNodeWrapper; + } + } + fail("Failed to find DataNodeWrapper for client rpc port " + port); + return null; + } + + private String buildDataRegionTag(final TRegionInfo regionInfo) { + return "DataRegion[" + regionInfo.getConsensusGroupId().getId() + "]"; + } + + private void assertSyncLagIsZero( + final String metricsContent, final String dataRegionTag, final String metricsUrl) { + for (final String line : metricsContent.split("\\R")) { + if (!line.startsWith("iot_consensus{") + || !line.contains("type=\"syncLag\"") + || !line.contains("region=\"" + dataRegionTag + "\"")) { + continue; + } + final int lastSpaceIndex = line.lastIndexOf(' '); + Assert.assertTrue("Malformed syncLag metric line: " + line, lastSpaceIndex > 0); + Assert.assertEquals( + "Expected syncLag of " + dataRegionTag + " to be 0 at " + metricsUrl + " but got " + line, + 0.0, + Double.parseDouble(line.substring(lastSpaceIndex + 1)), + SYNC_LAG_DELTA); + return; + } + fail("No syncLag metric found for " + dataRegionTag + " at " + metricsUrl); + } + @Test public void testPipeAfterRegisterNewDataNode() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBLegacyPipeReceiverSecurityIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBLegacyPipeReceiverSecurityIT.java new file mode 100644 index 0000000000000..51e1e211535e8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBLegacyPipeReceiverSecurityIT.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.single; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.property.ThriftClientProperty; +import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TSCloseSessionReq; +import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; +import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; +import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion; +import org.apache.iotdb.service.rpc.thrift.TSyncIdentityInfo; +import org.apache.iotdb.service.rpc.thrift.TSyncTransportMetaInfo; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.ZoneId; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class}) +public class IoTDBLegacyPipeReceiverSecurityIT { + + @BeforeClass + public static void setUp() { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testRejectPathTraversalFileNameInLegacyTransportFile() throws Exception { + final DataNodeWrapper dataNode = EnvFactory.getEnv().getDataNodeWrapper(0); + + try (final IoTDBSyncClient client = + new IoTDBSyncClient( + new ThriftClientProperty.Builder().build(), + dataNode.getIp(), + dataNode.getPort(), + false, + null, + null)) { + final TSOpenSessionResp openSessionResp = client.openSession(createOpenSessionReq()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), openSessionResp.getStatus().getCode()); + + try { + final TSStatus handshakeStatus = + client.handshake( + new TSyncIdentityInfo( + "pathTraversalPipe", System.currentTimeMillis(), "UNKNOWN", "")); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), handshakeStatus.getCode()); + + final String maliciousFileName = + ".." + File.separator + ".." + File.separator + "pwned.tsfile"; + final TSStatus status = + client.sendFile( + new TSyncTransportMetaInfo(maliciousFileName, 0), + ByteBuffer.wrap("pwned".getBytes(StandardCharsets.UTF_8))); + + Assert.assertEquals(TSStatusCode.SYNC_FILE_ERROR.getStatusCode(), status.getCode()); + Assert.assertTrue(status.getMessage().contains("Illegal fileName")); + } finally { + client.closeSession(new TSCloseSessionReq(openSessionResp.getSessionId())); + } + } + } + + private TSOpenSessionReq createOpenSessionReq() { + final TSOpenSessionReq req = new TSOpenSessionReq(); + req.setClient_protocol(TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V3); + req.setUsername("root"); + req.setPassword("root"); + req.setZoneId(ZoneId.systemDefault().toString()); + req.putToConfiguration("version", IoTDBConstant.ClientVersion.V_1_0.toString()); + req.putToConfiguration("sql_dialect", "tree"); + return req; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionQueryIT.java index 7ade93490a79d..30544f4f8dc57 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionQueryIT.java @@ -153,10 +153,13 @@ public void rawDataQueryWithTimeRangeTest2() throws IoTDBConnectionException { @Test public void lastQueryTest() throws IoTDBConnectionException { - String[] retArray = new String[] {"23,root.sg1.d1.s1,230000.0,FLOAT"}; + Set retArray = + new HashSet<>( + Arrays.asList("-40,root.sg1.d2.s6,40.0,DOUBLE", "23,root.sg1.d1.s1,230000.0,FLOAT")); - List selectedPaths = Collections.singletonList("root.sg1.d1.s1"); + List selectedPaths = Arrays.asList("root.sg1.d1.s1", "root.sg1.d2.s6"); + // Does not guarantee sequence try (ISession session = EnvFactory.getEnv().getSessionConnection()) { try (SessionDataSet resultSet = session.executeLastDataQuery(selectedPaths)) { assertResultSetEqual(resultSet, lastQueryColumnNames, retArray, true); diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/param/IoTDBEncryptedPasswordPullConsumerIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/param/IoTDBEncryptedPasswordPullConsumerIT.java new file mode 100644 index 0000000000000..9987ebe97322d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/triple/regression/param/IoTDBEncryptedPasswordPullConsumerIT.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.subscription.it.triple.regression.param; + +import org.apache.iotdb.commons.utils.AuthUtils; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegressionMisc; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.session.subscription.consumer.SubscriptionPullConsumer; +import org.apache.iotdb.subscription.it.triple.regression.AbstractSubscriptionRegressionIT; + +import org.apache.thrift.TException; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2SubscriptionRegressionMisc.class}) +public class IoTDBEncryptedPasswordPullConsumerIT extends AbstractSubscriptionRegressionIT { + + private static final String DATABASE = "root.TestEncryptedPasswordPullConsumer"; + private static final String DEVICE = DATABASE + ".d_0"; + private static final String TOPIC_NAME = "TestEncryptedPasswordPullConsumerTopic"; + private static final String USERNAME = "encrypted_user"; + private static final String PASSWORD = "EncryptedUser@123"; + private static final String ENCRYPTED_PASSWORD = AuthUtils.encryptPassword(PASSWORD); + private static final String WRONG_ENCRYPTED_PASSWORD = + AuthUtils.encryptPassword("WrongEncryptedUser@123"); + + private static final List SCHEMA_LIST = new ArrayList<>(); + + static { + SCHEMA_LIST.add(new MeasurementSchema("s_0", TSDataType.INT64)); + SCHEMA_LIST.add(new MeasurementSchema("s_1", TSDataType.DOUBLE)); + } + + private SubscriptionPullConsumer consumer; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + createDB(DATABASE); + createTopic_s(TOPIC_NAME, "root.**", null, null, false); + session_src.createTimeseries( + DEVICE + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_src.createTimeseries( + DEVICE + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZ4); + session_dest.createTimeseries( + DEVICE + ".s_0", TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4); + session_dest.createTimeseries( + DEVICE + ".s_1", TSDataType.DOUBLE, TSEncoding.TS_2DIFF, CompressionType.LZ4); + session_src.executeNonQueryStatement("create user " + USERNAME + " '" + PASSWORD + "'"); + session_src.executeNonQueryStatement("grant read,write on root.** to user " + USERNAME); + Assert.assertTrue(subs.getTopic(TOPIC_NAME).isPresent()); + } + + @Override + @After + public void tearDown() throws Exception { + try { + if (consumer != null) { + consumer.close(); + } + } catch (final Exception ignored) { + } + try { + subs.dropTopic(TOPIC_NAME); + } catch (final Exception ignored) { + } + try { + session_src.executeNonQueryStatement("drop user " + USERNAME); + } catch (final Exception ignored) { + } + dropDB(DATABASE); + super.tearDown(); + } + + @Test + public void testSubscribeWithEncryptedPassword() + throws TException, + IoTDBConnectionException, + IOException, + StatementExecutionException, + InterruptedException { + consumer = createConsumer("encrypted-password-group", ENCRYPTED_PASSWORD); + + consumer.open(); + consumer.subscribe(TOPIC_NAME); + Assert.assertEquals(1, subs.getSubscriptions().size()); + + insertData(1706659200000L); + consume_data(consumer, session_dest); + check_count( + 4, + "select count(s_0) from " + DEVICE + " where time >= 1706659200000", + "encrypted password consumption"); + } + + @Test + public void testSubscribeFailsWithWrongEncryptedPassword() + throws IoTDBConnectionException, StatementExecutionException { + consumer = createConsumer("wrong-encrypted-password-group", WRONG_ENCRYPTED_PASSWORD); + + try { + consumer.open(); + consumer.subscribe(TOPIC_NAME); + Assert.fail("subscribe should fail when encrypted password mismatches"); + } catch (final Exception ignored) { + Assert.assertTrue(subs.getSubscriptions().isEmpty()); + } + } + + private SubscriptionPullConsumer createConsumer( + final String consumerGroupId, final String encryptedPassword) { + return new SubscriptionPullConsumer.Builder() + .host(SRC_HOST) + .port(SRC_PORT) + .username(USERNAME) + .encryptedPassword(encryptedPassword) + .consumerId("consumer_" + consumerGroupId) + .consumerGroupId(consumerGroupId) + .buildPullConsumer(); + } + + private void insertData(long timestamp) + throws IoTDBConnectionException, StatementExecutionException { + final Tablet tablet = new Tablet(DEVICE, SCHEMA_LIST, 10); + for (int row = 0; row < 5; row++) { + final int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue("s_0", rowIndex, row * 20L + row); + tablet.addValue("s_1", rowIndex, row + 2.45); + timestamp += row * 2000; + } + session_src.insertTablet(tablet); + } +} diff --git a/iotdb-client/client-cpp/CMakeLists.txt b/iotdb-client/client-cpp/CMakeLists.txt new file mode 100644 index 0000000000000..8ee20d3d241b2 --- /dev/null +++ b/iotdb-client/client-cpp/CMakeLists.txt @@ -0,0 +1,271 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ============================================================================= +# Apache IoTDB - C++ Session Client (top-level CMake build) +# ============================================================================= + +cmake_minimum_required(VERSION 3.15) +project(iotdb_session CXX C) + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_POLICY_DEFAULT_CMP0091 NEW) + +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) +endif() + +set(IOTDB_EXTRA_CXX_FLAGS "" + CACHE STRING "Extra flags appended to CMAKE_CXX_FLAGS") +option(IOTDB_LIBSTDCXX_CXX11_ABI + "Use libstdc++ dual-ABI new layout (_GLIBCXX_USE_CXX11_ABI=1); release glibc224 zips" + OFF) + +if(NOT MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g -O2") + if(IOTDB_LIBSTDCXX_CXX11_ABI) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=1") + endif() + if(IOTDB_EXTRA_CXX_FLAGS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${IOTDB_EXTRA_CXX_FLAGS}") + endif() + # CI/release scripts read this tag; nm-based checks are unreliable on some toolchains. + if(IOTDB_LIBSTDCXX_CXX11_ABI) + file(WRITE "${CMAKE_BINARY_DIR}/iotdb_libstdcxx_abi_tag" "cxx11\n") + else() + file(WRITE "${CMAKE_BINARY_DIR}/iotdb_libstdcxx_abi_tag" "legacy\n") + endif() +endif() + +if(MSVC) + # /MD: matches default Visual Studio projects; CRT lives in the VC redistributable. + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>DLL") +endif() + +option(WITH_SSL "Build with OpenSSL support" OFF) +option(BUILD_TESTING "Build IT test executables" OFF) +option(IOTDB_OFFLINE "Disable all network access during configure" OFF) +set(IOTDB_SESSION_VERSION "0.0.0" + CACHE STRING "IoTDB C++ session package version") +string(REGEX MATCH "^[0-9]+" IOTDB_SESSION_SOVERSION "${IOTDB_SESSION_VERSION}") +if(NOT IOTDB_SESSION_SOVERSION) + set(IOTDB_SESSION_SOVERSION "0") +endif() + +set(IOTDB_DEPS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third-party" + CACHE PATH "Local tarball cache for third-party dependencies (lives under client-cpp/)") +if(WIN32) + set(_iotdb_default_boost_version "1.74.0") +elseif(APPLE) + set(_iotdb_default_boost_version "1.84.0") +else() + set(_iotdb_default_boost_version "1.60.0") +endif() +set(BOOST_VERSION "${_iotdb_default_boost_version}" + CACHE STRING "Boost version used when downloading / unpacking (Thrift build only)") +set(THRIFT_VERSION "0.21.0" + CACHE STRING "Apache Thrift version used when downloading / building") + +if(WIN32) + set(IOTDB_OS_DEPS_DIR "${IOTDB_DEPS_DIR}/windows") +elseif(APPLE) + set(IOTDB_OS_DEPS_DIR "${IOTDB_DEPS_DIR}/mac") +else() + set(IOTDB_OS_DEPS_DIR "${IOTDB_DEPS_DIR}/linux") +endif() +file(MAKE_DIRECTORY "${IOTDB_OS_DEPS_DIR}") + +if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/install" + CACHE PATH "Install prefix" FORCE) +endif() + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +include(FetchBoost) # -> BOOST_INCLUDE_DIR (Thrift build only) +include(FetchBuildTools) +if(WITH_SSL) + include(FetchOpenSSL) +endif() +include(FetchThrift) +include(GenerateThriftSources) + +file(GLOB SESSION_PUBLIC_SRCS CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/src/session/*.cpp") +file(GLOB SESSION_RPC_SRCS CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/src/rpc/*.cpp") + +add_library(iotdb_session SHARED + ${SESSION_PUBLIC_SRCS} + ${SESSION_RPC_SRCS} + ${THRIFT_GENERATED_SRCS}) + +if(WIN32) + set_target_properties(iotdb_session PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) +endif() + +if(UNIX AND NOT APPLE) + set_target_properties(iotdb_session PROPERTIES + VERSION "${IOTDB_SESSION_VERSION}" + SOVERSION "${IOTDB_SESSION_SOVERSION}") +endif() + +add_dependencies(iotdb_session iotdb_thrift_external iotdb_thrift_codegen) + +target_compile_definitions(iotdb_session PRIVATE THRIFT_STATIC_DEFINE IOTDB_BUILDING_SHARED) + +target_include_directories(iotdb_session + PUBLIC + $ + $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/rpc + ${THRIFT_GEN_CPP_DIR} + ${THRIFT_INCLUDE_DIR} + $<$:${BOOST_INCLUDE_DIR}>) + +if(APPLE) + target_link_libraries(iotdb_session PRIVATE "-Wl,-force_load,${THRIFT_STATIC_LIB_PATH}") +elseif(UNIX AND NOT MSVC) + # whole-archive pulls all Thrift objects into libiotdb_session.so; allow-multiple-definition + # avoids libgcc __morestack_* duplicate symbol errors on some GCC/toolchain combos. + target_link_libraries(iotdb_session PRIVATE + -Wl,--whole-archive + ${THRIFT_STATIC_LIB_PATH} + -Wl,--no-whole-archive + -Wl,--allow-multiple-definition) +else() + target_link_libraries(iotdb_session PRIVATE iotdb_thrift_static) +endif() + +if(WITH_SSL) + target_link_libraries(iotdb_session PUBLIC OpenSSL::SSL OpenSSL::Crypto) + target_compile_definitions(iotdb_session PUBLIC WITH_SSL=1) +else() + target_compile_definitions(iotdb_session PUBLIC WITH_SSL=0) +endif() + +if(UNIX) + target_link_libraries(iotdb_session PUBLIC pthread) +endif() + +include(GNUInstallDirs) + +set(IOTDB_PUBLIC_HEADERS + Export.h + SessionConfig.h + Session.h + Common.h + Optional.h + Date.h + Status.h + Endpoint.h + SessionBuilder.h + AbstractSessionBuilder.h + SessionPool.h + SessionC.h + SessionDataSet.h + Column.h + ColumnDecoder.h + TsBlock.h) + +# Windows: RUNTIME = iotdb_session.dll, ARCHIVE = import .lib (both under lib/ in the zip). +install(TARGETS iotdb_session + RUNTIME DESTINATION lib + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib) + +foreach(_hdr IN LISTS IOTDB_PUBLIC_HEADERS) + install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/src/include/${_hdr}" + DESTINATION include) +endforeach() + +set(IOTDB_SESSION_PC_LIBS "-liotdb_session") +set(IOTDB_SESSION_PC_CFLAGS "") +set(IOTDB_SESSION_CXX11_ABI_COMPILE_DEFINITION "") +if(UNIX) + set(IOTDB_SESSION_PC_LIBS "${IOTDB_SESSION_PC_LIBS} -pthread") +endif() +if(IOTDB_LIBSTDCXX_CXX11_ABI AND NOT MSVC) + set(IOTDB_SESSION_PC_CFLAGS "-D_GLIBCXX_USE_CXX11_ABI=1") + set(IOTDB_SESSION_CXX11_ABI_COMPILE_DEFINITION "_GLIBCXX_USE_CXX11_ABI=1") +endif() +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/iotdb-session-config.cmake.in" + "${CMAKE_BINARY_DIR}/package-metadata/cmake/iotdb-session-config.cmake" + @ONLY) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/iotdb-session.pc.in" + "${CMAKE_BINARY_DIR}/package-metadata/pkgconfig/iotdb-session.pc" + @ONLY) +execute_process( + COMMAND git rev-parse HEAD + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE IOTDB_SESSION_GIT_COMMIT + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) +if(NOT IOTDB_SESSION_GIT_COMMIT) + set(IOTDB_SESSION_GIT_COMMIT "unknown") +endif() +string(TIMESTAMP IOTDB_SESSION_BUILD_TIME "%Y-%m-%dT%H:%M:%SZ" UTC) +set(IOTDB_SESSION_CI_BUILD_ID "$ENV{GITHUB_RUN_ID}") +if(NOT IOTDB_SESSION_CI_BUILD_ID) + set(IOTDB_SESSION_CI_BUILD_ID "local") +endif() +file(WRITE "${CMAKE_BINARY_DIR}/package-metadata/VERSION" + "${IOTDB_SESSION_VERSION}\n") +file(WRITE "${CMAKE_BINARY_DIR}/package-metadata/BUILD-INFO.txt" + "Apache IoTDB C++ session client build information\n\n" + "version=${IOTDB_SESSION_VERSION}\n" + "commit=${IOTDB_SESSION_GIT_COMMIT}\n" + "build.time=${IOTDB_SESSION_BUILD_TIME}\n" + "ci.build.id=${IOTDB_SESSION_CI_BUILD_ID}\n" + "compiler=${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}\n" + "cmake.generator=${CMAKE_GENERATOR}\n" + "cmake.build.type=${CMAKE_BUILD_TYPE}\n" + "with.ssl=${WITH_SSL}\n" + "iotdb.offline=${IOTDB_OFFLINE}\n" + "iotdb.libstdcxx.cxx11.abi=${IOTDB_LIBSTDCXX_CXX11_ABI}\n" + "iotdb.extra.cxx.flags=${IOTDB_EXTRA_CXX_FLAGS}\n") +install(FILES "${CMAKE_BINARY_DIR}/package-metadata/cmake/iotdb-session-config.cmake" + DESTINATION cmake) +install(FILES "${CMAKE_BINARY_DIR}/package-metadata/pkgconfig/iotdb-session.pc" + DESTINATION pkgconfig) +install(FILES + "${CMAKE_BINARY_DIR}/package-metadata/VERSION" + "${CMAKE_BINARY_DIR}/package-metadata/BUILD-INFO.txt" + DESTINATION .) + +if(BUILD_TESTING) + enable_testing() + add_subdirectory(test) +endif() + +message(STATUS "iotdb_session configuration summary:") +message(STATUS " WITH_SSL = ${WITH_SSL}") +message(STATUS " BUILD_TESTING = ${BUILD_TESTING}") +message(STATUS " IOTDB_OFFLINE = ${IOTDB_OFFLINE}") +message(STATUS " IOTDB_DEPS_DIR = ${IOTDB_DEPS_DIR}") +message(STATUS " BOOST_INCLUDE_DIR = ${BOOST_INCLUDE_DIR} (Thrift build only)") +message(STATUS " THRIFT_INCLUDE_DIR = ${THRIFT_INCLUDE_DIR}") +message(STATUS " THRIFT_STATIC_LIB = ${THRIFT_STATIC_LIB_PATH}") +message(STATUS " THRIFT_EXECUTABLE = ${THRIFT_EXECUTABLE}") +message(STATUS " CMAKE_INSTALL_PREFIX = ${CMAKE_INSTALL_PREFIX}") +message(STATUS " IOTDB_LIBSTDCXX_CXX11_ABI = ${IOTDB_LIBSTDCXX_CXX11_ABI}") diff --git a/iotdb-client/client-cpp/README.md b/iotdb-client/client-cpp/README.md index 81983e06e2e4d..1ce08dc560a1a 100644 --- a/iotdb-client/client-cpp/README.md +++ b/iotdb-client/client-cpp/README.md @@ -18,133 +18,351 @@ under the License. --> -# Building C++ Client +# Apache IoTDB C++ Client -## Compile and Test: +The C++ client is built by a single top-level `CMakeLists.txt` in this +directory. The outer Maven POM is a thin wrapper that invokes CMake; you can +also build the client standalone with just `cmake` if you don't have Maven +available. -### Compile +## Build layout at a glance -#### Unix -To compile the cpp client, use the following command: -`mvn clean package -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests` - -#### Windows -To compile on Windows, please install Boost first and add following Maven -settings: ``` --Dboost.include.dir=${your boost header folder} -Dboost.library.dir=${your boost lib (stage) folder}` +iotdb-client/client-cpp/ +├── CMakeLists.txt # single entry point - manages everything +├── cmake/ # helpers (FetchBoost / FetchThrift / ...) +├── third-party/ # local tarball cache (one sub-dir per OS) +│ ├── linux/ mac/ windows/ +├── src/include/ # public API headers (installed to include/) +├── src/session/ # Session, Tablet, and C API implementation (.cpp) +├── src/rpc/ # Thrift RPC layer (private, not installed) +├── test/ # Catch2-based integration tests +└── pom.xml # Maven wrapper (cmake-maven-plugin) ``` -The thrift dependency that the cpp client uses is incompatible with MinGW, please use Visual -Studio. It is highly recommended to use Visual Studio 2022 or later. - -##### Visual Studio 2022 -If you are using Visual Studio 2022, you can compile the cpp client with the following command: - -``` -mvn clean package -P with-cpp -pl iotdb-client/client-cpp -am -DskipTest --D"boost.include.dir"="D:\boost_1_75_0" -D"boost.library.dir"="D:\boost_1_75_0\stage\lib" +During configure CMake will, in order: + +1. Resolve Boost headers (`find_package` → local `third-party//` tarball → + download from `archives.boost.io` when not in offline mode). +2. On Linux/macOS, ensure `m4` / `flex` / `bison` are available; if not, + build them from local tarballs into `build/tools/bin` (no `sudo` + required). +3. Build a static Apache Thrift from source (tarball cache → download fallback). +4. Run the produced `thrift` compiler on + `iotdb-protocol/thrift-{commons,datanode}/src/main/thrift/*.thrift`. +5. Compile `iotdb_session` (the C/C++ session library) and, optionally, + the Catch2 integration test binaries. +6. `cmake --install` lays out the SDK under `target/install/{include,lib}`, + which Maven's assembly step packages into a zip. + +## Build matrix + +| Goal | Command | +|-------------------------------|--------------------------------------------------------------------------------------------------------| +| Library only (Linux/macOS) | `mvn -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests package` | +| Library only (Windows / MSVC) | `mvn -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests "-Dboost.include.dir=C:\boost_1_88_0" package` | +| Library + ITs (Linux/macOS) | `mvn clean install -P with-cpp -pl distribution,iotdb-client/client-cpp -am` then `mvn -P with-cpp -pl iotdb-client/client-cpp -am verify` | +| Direct CMake (no Maven) | `cmake -S iotdb-client/client-cpp -B build && cmake --build build --target install` | + +The Maven build sets `cmake.install.prefix` to `target/install/`. Output zips +land at `iotdb-client/client-cpp/target/iotdb-session-cpp--.zip` +(with a package root directory and a `.sha512` checksum generated alongside), +where `` defaults to the OS name (for example `linux-x86_64`) and +can be overridden with `-Dclient.cpp.package.classifier=...` when building +multiple toolchains on the same platform. + +### Release packages (CI) + +The [C++ Client package](../../.github/workflows/client-cpp-package.yml) workflow +builds one zip per platform/toolchain. Pick the artifact that matches your +deployment environment: + +| Target environment | Zip classifier (suffix) | +|--------------------|-------------------------| +| Linux x86_64, glibc >= 2.28 | `linux-x86_64-glibc2.28` | +| Linux aarch64, glibc >= 2.28 | `linux-aarch64-glibc2.28` | +| macOS x86_64 | `macos-x86_64` | +| macOS arm64 | `macos-aarch64` | +| Windows + Visual Studio 2017 | `windows-x86_64-msvc14.1` | +| Windows + Visual Studio 2019 | `windows-x86_64-msvc14.2` | +| Windows + Visual Studio 2022 | `windows-x86_64-msvc14.3` | +| Windows + Visual Studio 2026 | `windows-x86_64-msvc14.4` | + +Example file name: +`iotdb-session-cpp-1.3.7-SNAPSHOT-linux-x86_64-glibc2.28.zip`. + +**Linux package choice:** The CI Linux packages are built in manylinux_2_28 +containers, so use them on hosts with glibc 2.28 or newer. Build locally with +the same Maven/CMake options if you need a different baseline. + +Thrift **0.21.0** is compiled from source during the CMake configure step (see +`cmake/FetchThrift.cmake`). Older releases that used pre-built +`iotdb-tools-thrift` Maven artifacts and `-Diotdb-tools-thrift.version=...` +for glibc/MSVC compatibility apply only to the **legacy** client-cpp build; +with the current CMake build, compatibility is determined by the **compiler +and OS used to build** the SDK, not by that Maven property. + +### Local build for a specific classifier + +Linux x86_64 (glibc 2.28 baseline): + +```bash +mvn -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests \ + -Dclient.cpp.package.classifier=linux-x86_64-glibc2.28 package ``` -#### Visual Studio 2019 or older -If you are using Visual Studio 2019 or older, the pre-built Thrift library is incompatible. You -will have to compile the thrift library manually: +Windows (match the Visual Studio version you use to build your application): + +```powershell +# Visual Studio 2022 (default on recent Windows) +mvn -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests package -1. Install the dependencies of Thrift: -* flex http://gnuwin32.sourceforge.net/packages/flex.htm -* bison http://gnuwin32.sourceforge.net/packages/bison.htm -* openssl https://slproweb.com/products/Win32OpenSSL.html +# Visual Studio 2019 +mvn -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests ` + "-Dcmake.generator=Visual Studio 16 2019" ` + -Dclient.cpp.package.classifier=windows-x86_64-msvc14.2 package -2. Clone the repository: https://github.com/apache/iotdb-bin-resources. +# Visual Studio 2017 (CMake uses -A x64 on Windows automatically) +mvn -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests ` + "-Dcmake.generator=Visual Studio 15 2017" ` + -Dclient.cpp.package.classifier=windows-x86_64-msvc14.1 package -3. Enter the "iotdb-tools-thrift" folder in the cloned repository; use the following command to - compile the thrift library: +# Debug build +mvn -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests ` + "-Dcmake.build.type=Debug" package +``` -`mvn install` +On Windows, the build passes `-DCMAKE_GENERATOR_PLATFORM=x64` so Visual Studio +generators target **x64** (VS2017 otherwise defaults to Win32). + +## CMake options + +The table below lists CMake cache variables. When building through **Maven**, +pass them as Maven properties (the POM maps them to `-D` options for CMake): + +| CMake variable | Maven property (`-D...`) | +|----------------|--------------------------| +| `WITH_SSL` | `with.ssl` (e.g. `-Dwith.ssl=ON`) | +| `IOTDB_OFFLINE` | `iotdb.offline` | +| `BUILD_TESTING` | `build.tests` | +| `IOTDB_DEPS_DIR` | `iotdb.deps.dir` | +| `BOOST_INCLUDEDIR` | `boost.include.dir` (legacy alias) | + +For a **standalone** `cmake` configure, pass `-DWITH_SSL=ON`, `-DIOTDB_OFFLINE=ON`, +etc. directly. + +| Option | Default | Purpose | +|-----------------------|----------------------------------|----------------------------------------------------------------------------------------------------------| +| `WITH_SSL` | `OFF` | Link against OpenSSL. See *SSL* below. | +| `BUILD_TESTING` | `OFF` (Maven sets `ON` for verify) | Build Catch2 IT executables (Catch2 v2.13.7 header downloaded at configure time). | +| `CATCH2_INCLUDE_DIR` | (unset) | Pre-downloaded Catch2 include dir (Maven sets this under `target/test/catch2`). | +| `IOTDB_OFFLINE` | `OFF` | Disallow any network access during configure. | +| `IOTDB_DEPS_DIR` | `/third-party` | Override the local tarball cache directory. | +| `BOOST_VERSION` | `1.60.0` (`1.84.0` on macOS) | Boost version that CMake will look for / download. | +| `THRIFT_VERSION` | `0.21.0` | Apache Thrift version to build from source. | +| `BOOST_ROOT` | (unset) | Existing Boost install to reuse, equivalent to `-Dboost.include.dir=...` from the legacy build. | +| `OPENSSL_ROOT_DIR` | (unset) | Existing OpenSSL install when `WITH_SSL=ON`. | +| `CMAKE_INSTALL_PREFIX`| `/install` | Install location. | + +## Online build (default) + +CMake will download any missing tarball at configure time. The first run is +slow (≈100 MB download + a Thrift build); subsequent runs reuse the +extracted artifacts under `build/_deps/`. + +```bash +# Linux / macOS +mvn -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests package + +# Windows (Developer Command Prompt for VS, PowerShell, or cmd) +mvn -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests "-Dboost.include.dir=C:\boost_1_88_0" package +``` -4. If you encounter a problem like "cannot find 'unistd.h'", please open the file -"iotdb-bin-resources\iotdb-tools-thrift\target\build\compiler\cpp\thrift\thriftl.cc" and replace -"#include " with "#include " and "#include "; then, rerun the command - in the third step; +## Offline build -5. Return to the cpp client repository and compile it with: +1. Pre-populate the platform-specific sub-directory under `third-party/`: -``` -mvn clean package -P with-cpp -pl iotdb-client/client-cpp -am -DskipTest --D"boost.include.dir"="D:\boost_1_75_0" -D"boost.library.dir"="D:\boost_1_75_0\stage\lib" + | Platform | Required files | + |------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| + | `linux/` | `thrift-0.21.0.tar.gz`, `boost_1_60_0.tar.gz`, `m4-1.4.19.tar.gz`, `flex-2.6.4.tar.gz`, `bison-3.8.tar.gz` (and `openssl-3.5.0.tar.gz` when `WITH_SSL=ON`) | + | `mac/` | `thrift-0.21.0.tar.gz`, `boost_1_84_0.tar.gz` (newer Boost for Xcode/Clang; Apple ships m4/flex/bison; `openssl-3.5.0.tar.gz` optional) | + | `windows/` | `thrift-0.21.0.tar.gz`, `boost_1_60_0.tar.gz` (Boost headers only - no `b2` build required for `iotdb_session`) | + + Reference URLs (the configure step uses the same): + - Apache Thrift 0.21.0: + - Boost 1.60.0: + - GNU m4 1.4.19: + - GNU flex 2.6.4: + - GNU bison 3.8: + - OpenSSL 3.5.0: + +2. Run the build with offline mode enabled: + + ```bash + mvn -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests \ + -Diotdb.offline=ON package + ``` + + or, going straight through CMake: + + ```bash + cmake -S iotdb-client/client-cpp -B build -DIOTDB_OFFLINE=ON + cmake --build build --config Release --target install + ``` + +CI environments can share a single cache by setting +`-DIOTDB_DEPS_DIR=/path/to/cache` instead of copying tarballs around. + +## Platform-specific notes + +### Linux + +- Tested with GCC 7+ and Clang 9+. Anything that can compile Apache Thrift + 0.21.0 works. +- Build deps that must already exist on the host (only required when + CMake auto-builds m4/flex/bison from tarball): `make`, `autoconf`, + `gcc`, plus the standard C/C++ toolchain. `sudo` is **not** required; + the helper tools install under `build/tools/`. +- If you would rather use distro-provided tools (`apt install m4 flex + bison`), CMake will pick them up first. + +### macOS + +- Xcode Command Line Tools provide `m4`, `flex`, `bison`, and `make`, + so the auto-build path normally skips them. +- Homebrew users can `brew install boost` to short-circuit `FetchBoost`. + +### Windows + +Visual Studio **2017, 2019, 2022, or 2026** is supported for building the SDK. +Link your application against the zip built with the **same VS generation** you +use for your project. + +Prerequisites: + +1. **Boost.** Download and extract + + (any 1.60+ release will work). `iotdb_session` only needs Boost + headers, so running `bootstrap.bat` / `b2` is optional. Pass the + location with either `-Dboost.include.dir="C:\boost_1_88_0"` (Maven) + or `-DBOOST_ROOT="C:\boost_1_88_0"` (raw CMake). +2. **flex / bison.** Install + and rename `win_flex.exe`→`flex.exe`, `win_bison.exe`→`bison.exe` on + `PATH`. +3. **OpenSSL** *(only when `WITH_SSL=ON`)*: run the Win64 OpenSSL + installer from , then + pass `-DOPENSSL_ROOT_DIR=...` to CMake. + +On Windows the SDK ships as **`iotdb_session.dll`** plus an import library +**`iotdb_session.lib`**, built with **`/MD`** (dynamic CRT, same as a +default Visual Studio application). Thrift is linked into the DLL; users +do not install separate Thrift headers or libraries. Place +`iotdb_session.dll` next to your `.exe` or on `PATH`. + +Auto-building m4/flex/bison from tarball is **not** supported on Windows; +the GNU autotools tarballs assume a POSIX shell environment. + +## SSL + +Both Thrift and `iotdb_session` build without OpenSSL by default. Enable +SSL with `-Dwith.ssl=ON` (Maven) or `-DWITH_SSL=ON` (standalone CMake). +CMake first calls `find_package(OpenSSL)`; +if nothing is found, it falls back to: + +- **Linux / macOS** – use a local `openssl-.tar.gz` (or download it + when not in offline mode), configure with `no-shared`, install into + `build/_deps/openssl/install`, and link statically. +- **Windows** – fail with a friendly message that points at the Win64 + OpenSSL installer. Building OpenSSL from source via MSVC is out of scope. + +## Tests + +Maven binds `cmake-maven-plugin`'s `test` goal to the `integration-test` +phase and runs `ctest`. `pre-integration-test` spawns a local IoTDB server +from `distribution/target/.../sbin/start-standalone.{sh,bat}`, so make sure +the distribution module is built first: + +```bash +mvn clean install -P with-cpp -pl distribution,iotdb-client/client-cpp -am -DskipTests +mvn -P with-cpp -pl iotdb-client/client-cpp -am verify ``` +Running ctest directly (after a `mvn ... package` build) is also supported: -### Test -First build IoTDB server together with the cpp client. +```bash +cd iotdb-client/client-cpp/target/build/test +ctest --output-on-failure +``` -Explicitly using "install" instead of package in order to be sure we're using libs built on this -machine. +## Code formatting -`mvn clean install -P with-cpp -pl distribution,iotdb-client/client-cpp -am -DskipTests` +We use `clang-format` (pinned by the root POM as `clang.format.version`) +through Maven Spotless. **clang-format 17.0.6** is the version CI runs. -After run verify +```bash +mvn -P with-cpp -pl iotdb-client/client-cpp spotless:check +mvn -P with-cpp -pl iotdb-client/client-cpp spotless:apply +``` -`mvn clean verify -P with-cpp -pl iotdb-client/client-cpp -am` +On JDK 8 the C++ Spotless profile is skipped automatically (Spotless's +clang-format integration requires Spotless 2.44+, which itself requires +JDK 11+). -## Package Hierarchy +## Package layout -If the compilation finishes successfully, the packaged zip file will be placed under -"client-cpp/target/client-cpp-${project.version}-cpp-${os}.zip". +A successful `mvn ... package` produces +`target/iotdb-session-cpp--.zip` with this layout: -On macOS, the hierarchy of the package should look like this: ``` -. -+-- client -| +-- include -| +-- Session.h -| +-- IClientRPCService.h -| +-- client_types.h -| +-- common_types.h -| +-- thrift -| +-- thrift_headers... -| +-- lib -| +-- Release -| +-- libiotdb_session.dylib -| +-- parser.dylib -| +-- thriftmd.dylib -| +-- tutorialgencpp.dylib +iotdb-session-cpp--/ +|-- BUILD-INFO.txt +|-- VERSION +|-- cmake/ +|-- pkgconfig/ +|-- include/ +| |-- Session.h +| |-- SessionC.h +| `-- ... (public API headers only; no Thrift/Boost) +`-- lib/ + |-- libiotdb_session.{so,dylib} (Linux / macOS) + |-- iotdb_session.dll (Windows runtime) + `-- iotdb_session.lib (Windows import library for linking) ``` -## Using C++ Client: -``` -1. Put the zip file "client-cpp-${project.version}-cpp-${os}.zip" wherever you want; - -2. Unzip the archive using the following command, and then you can get the two directories -mentioned above, the header file and the dynamic library: - unzip client-cpp-${project.version}-cpp-${os}.zip +Thrift is embedded inside `iotdb_session` on all platforms; it is not shipped +as a separate install artifact. -3. Write C++ code to call the operation interface of the cpp client to operate IoTDB, - for detail interface information, please refer to the link: https://iotdb.apache.org/zh/UserGuide/Master/API/Programming-Cpp-Native-API.html +## Using the C++ client - E.g: - #include "include/Session.h" +```cpp +#include "Session.h" #include #include int main() { - std::cout << "open session" << std::endl; - std::shared_ptr session(new Session("127.0.0.1", 6667, "root", "root")); + auto session = std::make_shared("127.0.0.1", 6667, "root", "root"); session->open(false); - - std::cout << "setStorageGroup: root.test01" << std::endl; session->setStorageGroup("root.test01"); - if (!session->checkTimeseriesExists("root.test01.d0.s0")) { - session->createTimeseries("root.test01.d0.s0", TSDataType::INT64, TSEncoding::RLE, CompressionType::SNAPPY); - std::cout << "create Timeseries: root.test01.d0.s0" << std::endl; - } - - std::cout << "session close" << std::endl; + session->createTimeseries( + "root.test01.d0.s0", + TSDataType::INT64, + TSEncoding::RLE, + CompressionType::SNAPPY); + } session->close(); } +``` -4. Compile and execute - clang++ -O2 user-cpp-code.cpp -liotdb_session -L/user-unzip-absolute-path/lib -Wl,-rpath /user-unzip-absolute-path/lib -std=c++11 - ./a.out +Compile against the produced SDK: + +```bash +clang++ -O2 user-cpp-code.cpp \ + -I/path/to/sdk/include \ + -L/path/to/sdk/lib \ + -liotdb_session -lpthread \ + -Wl,-rpath,/path/to/sdk/lib \ + -std=c++11 ``` + +For full API documentation see the [C++ Native API guide](https://iotdb.apache.org/UserGuide/latest/API/Programming-Cpp-Native-API.html). diff --git a/iotdb-client/client-cpp/cmake/FetchBoost.cmake b/iotdb-client/client-cpp/cmake/FetchBoost.cmake new file mode 100644 index 0000000000000..d1ab9c18ae1b9 --- /dev/null +++ b/iotdb-client/client-cpp/cmake/FetchBoost.cmake @@ -0,0 +1,130 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ============================================================================= +# FetchBoost.cmake +# +# Resolves the location of Boost headers needed at thrift / iotdb_session +# compile time, in three stages: +# +# 1. Use system / user-provided Boost (find_package, BOOST_ROOT, etc.). +# 2. Fall back to a local tarball under ${IOTDB_OS_DEPS_DIR}/. +# 3. Otherwise download the tarball from archives.boost.io (unless +# IOTDB_OFFLINE is ON). +# +# Output (cache) variables: +# BOOST_INCLUDE_DIR - directory containing headers. +# ============================================================================= + +if(DEFINED BOOST_INCLUDE_DIR AND EXISTS "${BOOST_INCLUDE_DIR}/boost/version.hpp") + message(STATUS "[Boost] reusing cached BOOST_INCLUDE_DIR=${BOOST_INCLUDE_DIR}") + return() +endif() + +# --------------------------------------------------------------------------- +# Stage 1: find_package(Boost) - respects BOOST_ROOT / Boost_INCLUDE_DIR +# --------------------------------------------------------------------------- +find_package(Boost QUIET) +if(Boost_FOUND AND Boost_INCLUDE_DIRS) + set(BOOST_INCLUDE_DIR "${Boost_INCLUDE_DIRS}" CACHE PATH "Boost include directory" FORCE) + message(STATUS "[Boost] using system Boost at ${BOOST_INCLUDE_DIR}") + return() +endif() + +# Allow plain -DBOOST_INCLUDEDIR= / -DBoost_INCLUDE_DIR= as a fast path. +foreach(_hint Boost_INCLUDE_DIR BOOST_INCLUDEDIR BOOST_ROOT) + if(DEFINED ${_hint}) + set(_candidate "${${_hint}}") + if(EXISTS "${_candidate}/boost/version.hpp") + set(BOOST_INCLUDE_DIR "${_candidate}" CACHE PATH "Boost include directory" FORCE) + message(STATUS "[Boost] using hinted path ${BOOST_INCLUDE_DIR}") + return() + elseif(EXISTS "${_candidate}/include/boost/version.hpp") + set(BOOST_INCLUDE_DIR "${_candidate}/include" CACHE PATH "Boost include directory" FORCE) + message(STATUS "[Boost] using hinted path ${BOOST_INCLUDE_DIR}") + return() + endif() + endif() +endforeach() + +# --------------------------------------------------------------------------- +# Stage 2: local tarball cache +# --------------------------------------------------------------------------- +string(REPLACE "." "_" _boost_us "${BOOST_VERSION}") +set(_boost_dirname "boost_${_boost_us}") +set(_boost_tarname_gz "${_boost_dirname}.tar.gz") +set(_boost_tarname_zip "${_boost_dirname}.zip") + +set(_boost_tarball "") +foreach(_name IN ITEMS ${_boost_tarname_gz} ${_boost_tarname_zip}) + if(EXISTS "${IOTDB_OS_DEPS_DIR}/${_name}") + set(_boost_tarball "${IOTDB_OS_DEPS_DIR}/${_name}") + break() + endif() +endforeach() + +# --------------------------------------------------------------------------- +# Stage 3: download +# --------------------------------------------------------------------------- +if(NOT _boost_tarball) + if(IOTDB_OFFLINE) + message(FATAL_ERROR + "[Boost] IOTDB_OFFLINE=ON but no Boost tarball found in " + "${IOTDB_OS_DEPS_DIR}. Expected one of: ${_boost_tarname_gz}, ${_boost_tarname_zip}.") + endif() + + set(_boost_url "https://archives.boost.io/release/${BOOST_VERSION}/source/${_boost_tarname_gz}") + set(_boost_tarball "${IOTDB_OS_DEPS_DIR}/${_boost_tarname_gz}") + message(STATUS "[Boost] downloading ${_boost_url}") + file(DOWNLOAD "${_boost_url}" "${_boost_tarball}" + SHOW_PROGRESS + STATUS _boost_dl_status + TLS_VERIFY ON) + list(GET _boost_dl_status 0 _boost_dl_code) + if(NOT _boost_dl_code EQUAL 0) + list(GET _boost_dl_status 1 _boost_dl_msg) + file(REMOVE "${_boost_tarball}") + message(FATAL_ERROR "[Boost] download failed: ${_boost_dl_msg}") + endif() +endif() + +# --------------------------------------------------------------------------- +# Extract headers-only into ${CMAKE_BINARY_DIR}/_deps/boost +# --------------------------------------------------------------------------- +set(_boost_extract_dir "${CMAKE_BINARY_DIR}/_deps/boost") +set(_boost_marker "${_boost_extract_dir}/.extracted-${BOOST_VERSION}") + +if(NOT EXISTS "${_boost_marker}") + file(REMOVE_RECURSE "${_boost_extract_dir}") + file(MAKE_DIRECTORY "${_boost_extract_dir}") + message(STATUS "[Boost] extracting ${_boost_tarball}") + file(ARCHIVE_EXTRACT + INPUT "${_boost_tarball}" + DESTINATION "${_boost_extract_dir}") + file(TOUCH "${_boost_marker}") +endif() + +set(BOOST_INCLUDE_DIR "${_boost_extract_dir}/${_boost_dirname}" + CACHE PATH "Boost include directory" FORCE) + +if(NOT EXISTS "${BOOST_INCLUDE_DIR}/boost/version.hpp") + message(FATAL_ERROR + "[Boost] Could not locate boost/version.hpp after extraction. " + "Looked in: ${BOOST_INCLUDE_DIR}") +endif() + +message(STATUS "[Boost] BOOST_INCLUDE_DIR = ${BOOST_INCLUDE_DIR}") diff --git a/iotdb-client/client-cpp/cmake/FetchBuildTools.cmake b/iotdb-client/client-cpp/cmake/FetchBuildTools.cmake new file mode 100644 index 0000000000000..c9d7482e906ea --- /dev/null +++ b/iotdb-client/client-cpp/cmake/FetchBuildTools.cmake @@ -0,0 +1,266 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ============================================================================= +# FetchBuildTools.cmake +# +# Apache Thrift's source build needs a working flex / bison toolchain (m4 too +# on Unix). When the host already provides them on PATH we use them as-is. +# Otherwise we provision them locally: +# +# * Linux / macOS - configure-make-install each tool from a tarball into +# ${CMAKE_BINARY_DIR}/tools (no sudo required). +# * Windows - extract the winflexbison zip and copy +# win_flex.exe -> flex.exe, win_bison.exe -> bison.exe. +# +# Tarballs / zips are resolved with the standard three-stage pattern: +# 1. ${IOTDB_OS_DEPS_DIR}/ (or any match for an optional GLOB) +# 2. file(DOWNLOAD) when IOTDB_OFFLINE is OFF +# 3. FATAL_ERROR otherwise +# ============================================================================= + +set(_tools_prefix "${CMAKE_BINARY_DIR}/tools") +set(_tools_bin "${_tools_prefix}/bin") +file(MAKE_DIRECTORY "${_tools_bin}") + +# Make sure any tool we install locally takes precedence over the system PATH +# for the remainder of the configure step (and child ExternalProject calls). +if(WIN32) + set(ENV{PATH} "${_tools_bin};$ENV{PATH}") +else() + set(ENV{PATH} "${_tools_bin}:$ENV{PATH}") +endif() + +set(M4_VERSION "1.4.19" CACHE STRING "GNU m4 version to build when missing") +set(FLEX_VERSION "2.6.4" CACHE STRING "GNU flex version to build when missing") +set(BISON_VERSION "3.8" CACHE STRING "GNU bison version to build when missing") +set(WINFLEXBISON_VERSION "2.5.25" + CACHE STRING "winflexbison version to download when no local zip is present") + +set(_m4_url "https://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz") +set(_flex_url "https://github.com/westes/flex/releases/download/v${FLEX_VERSION}/flex-${FLEX_VERSION}.tar.gz") +set(_bison_url "https://ftp.gnu.org/gnu/bison/bison-${BISON_VERSION}.tar.gz") +set(_winflexbison_url + "https://github.com/lexxmark/winflexbison/releases/download/v${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip") +set(_winflexbison_filename "win_flex_bison-${WINFLEXBISON_VERSION}.zip") + +include(ProcessorCount) +ProcessorCount(_jobs) +if(_jobs LESS 1) + set(_jobs 1) +endif() + +# Resolve tarball: prefer the exact filename in ${IOTDB_OS_DEPS_DIR}/, then +# any path matching GLOB_PATTERN (caller-supplied wildcard for relaxed naming, +# e.g. win_flex_bison*.zip), and finally fall back to a download. +function(_iotdb_resolve_tarball OUT_VAR FILENAME URL) + cmake_parse_arguments(ARG "" "GLOB_PATTERN" "" ${ARGN}) + + set(_local "${IOTDB_OS_DEPS_DIR}/${FILENAME}") + if(EXISTS "${_local}") + set(${OUT_VAR} "${_local}" PARENT_SCOPE) + return() + endif() + + if(ARG_GLOB_PATTERN) + file(GLOB _matches "${IOTDB_OS_DEPS_DIR}/${ARG_GLOB_PATTERN}") + if(_matches) + list(GET _matches 0 _hit) + message(STATUS "[BuildTools] reusing ${_hit}") + set(${OUT_VAR} "${_hit}" PARENT_SCOPE) + return() + endif() + endif() + + if(IOTDB_OFFLINE) + set(_hint "${FILENAME}") + if(ARG_GLOB_PATTERN) + set(_hint "${FILENAME} (or any ${ARG_GLOB_PATTERN})") + endif() + message(FATAL_ERROR + "[BuildTools] IOTDB_OFFLINE=ON but ${_hint} is missing in " + "${IOTDB_OS_DEPS_DIR}.") + endif() + + message(STATUS "[BuildTools] downloading ${URL}") + file(DOWNLOAD "${URL}" "${_local}" SHOW_PROGRESS STATUS _st TLS_VERIFY ON) + list(GET _st 0 _code) + if(NOT _code EQUAL 0) + list(GET _st 1 _msg) + file(REMOVE "${_local}") + message(FATAL_ERROR "[BuildTools] download failed for ${FILENAME}: ${_msg}") + endif() + set(${OUT_VAR} "${_local}" PARENT_SCOPE) +endfunction() + +# Configure-make-install from into ${_tools_prefix}. +function(_iotdb_build_autotools NAME TARBALL EXTRACTED_DIRNAME) + set(_src_root "${CMAKE_BINARY_DIR}/_deps/${NAME}") + set(_marker "${_tools_prefix}/.${NAME}-installed") + if(EXISTS "${_marker}") + return() + endif() + file(REMOVE_RECURSE "${_src_root}") + file(MAKE_DIRECTORY "${_src_root}") + message(STATUS "[BuildTools] extracting ${TARBALL}") + file(ARCHIVE_EXTRACT INPUT "${TARBALL}" DESTINATION "${_src_root}") + set(_src "${_src_root}/${EXTRACTED_DIRNAME}") + if(NOT EXISTS "${_src}/configure") + message(FATAL_ERROR + "[BuildTools] expected configure script at ${_src}/configure") + endif() + message(STATUS "[BuildTools] building ${NAME} -> ${_tools_prefix}") + # flex 2.6.4: reallocarray() needs _GNU_SOURCE on glibc 2.26+ (westes/flex#241). + set(_env_prefix "") + if(NAME STREQUAL "flex" AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(_env_prefix env CFLAGS=-D_GNU_SOURCE CXXFLAGS=-D_GNU_SOURCE) + endif() + if(_env_prefix) + execute_process( + COMMAND ${_env_prefix} ./configure --prefix=${_tools_prefix} + WORKING_DIRECTORY "${_src}" + RESULT_VARIABLE _rc) + else() + execute_process( + COMMAND ./configure --prefix=${_tools_prefix} + WORKING_DIRECTORY "${_src}" + RESULT_VARIABLE _rc) + endif() + if(NOT _rc EQUAL 0) + message(FATAL_ERROR "[BuildTools] configure failed for ${NAME}") + endif() + if(_env_prefix) + execute_process( + COMMAND ${_env_prefix} make -j${_jobs} + WORKING_DIRECTORY "${_src}" + RESULT_VARIABLE _rc) + else() + execute_process( + COMMAND make -j${_jobs} + WORKING_DIRECTORY "${_src}" + RESULT_VARIABLE _rc) + endif() + if(NOT _rc EQUAL 0) + message(FATAL_ERROR "[BuildTools] make failed for ${NAME}") + endif() + if(_env_prefix) + execute_process( + COMMAND ${_env_prefix} make install + WORKING_DIRECTORY "${_src}" + RESULT_VARIABLE _rc) + else() + execute_process( + COMMAND make install + WORKING_DIRECTORY "${_src}" + RESULT_VARIABLE _rc) + endif() + if(NOT _rc EQUAL 0) + message(FATAL_ERROR "[BuildTools] make install failed for ${NAME}") + endif() + file(TOUCH "${_marker}") +endfunction() + +# ============================================================================= +# Windows branch - winflexbison +# ============================================================================= +if(WIN32) + # Stage 1: pick up an existing flex/bison if the host already provides it. + find_program(FLEX_EXECUTABLE NAMES flex win_flex) + find_program(BISON_EXECUTABLE NAMES bison win_bison) + + if(FLEX_EXECUTABLE AND BISON_EXECUTABLE) + message(STATUS "[BuildTools] using system flex = ${FLEX_EXECUTABLE}") + message(STATUS "[BuildTools] using system bison = ${BISON_EXECUTABLE}") + set(IOTDB_LOCAL_TOOLS_BIN "${_tools_bin}" CACHE INTERNAL "") + return() + endif() + + # Stage 2/3: resolve and extract the winflexbison zip into _tools_bin. + _iotdb_resolve_tarball(_wfb_zip + "${_winflexbison_filename}" + "${_winflexbison_url}" + GLOB_PATTERN "win_flex_bison*.zip") + + set(_wfb_marker "${_tools_bin}/.winflexbison-installed") + if(NOT EXISTS "${_wfb_marker}") + message(STATUS "[BuildTools] extracting ${_wfb_zip}") + file(ARCHIVE_EXTRACT INPUT "${_wfb_zip}" DESTINATION "${_tools_bin}") + + if(NOT EXISTS "${_tools_bin}/win_flex.exe" OR NOT EXISTS "${_tools_bin}/win_bison.exe") + message(FATAL_ERROR + "[BuildTools] win_flex.exe / win_bison.exe not found after " + "extracting ${_wfb_zip} into ${_tools_bin}.") + endif() + + # Copy with renamed targets so thrift's CMakeLists sees flex/bison. + execute_process(COMMAND ${CMAKE_COMMAND} -E copy + "${_tools_bin}/win_flex.exe" "${_tools_bin}/flex.exe" + RESULT_VARIABLE _rc) + if(NOT _rc EQUAL 0) + message(FATAL_ERROR "[BuildTools] failed to copy win_flex.exe -> flex.exe") + endif() + execute_process(COMMAND ${CMAKE_COMMAND} -E copy + "${_tools_bin}/win_bison.exe" "${_tools_bin}/bison.exe" + RESULT_VARIABLE _rc) + if(NOT _rc EQUAL 0) + message(FATAL_ERROR "[BuildTools] failed to copy win_bison.exe -> bison.exe") + endif() + file(TOUCH "${_wfb_marker}") + endif() + + find_program(FLEX_EXECUTABLE flex PATHS "${_tools_bin}" NO_DEFAULT_PATH REQUIRED) + find_program(BISON_EXECUTABLE bison PATHS "${_tools_bin}" NO_DEFAULT_PATH REQUIRED) + + message(STATUS "[BuildTools] flex = ${FLEX_EXECUTABLE}") + message(STATUS "[BuildTools] bison = ${BISON_EXECUTABLE}") + set(IOTDB_LOCAL_TOOLS_BIN "${_tools_bin}" CACHE INTERNAL "") + return() +endif() + +# ============================================================================= +# Linux / macOS branch - m4 / flex / bison from autotools tarballs +# ============================================================================= + +# m4 (flex/bison both depend on this) +find_program(M4_EXECUTABLE m4) +if(NOT M4_EXECUTABLE) + _iotdb_resolve_tarball(_m4_tarball "m4-${M4_VERSION}.tar.gz" "${_m4_url}") + _iotdb_build_autotools(m4 "${_m4_tarball}" "m4-${M4_VERSION}") + find_program(M4_EXECUTABLE m4 PATHS "${_tools_bin}" NO_DEFAULT_PATH REQUIRED) +endif() +message(STATUS "[BuildTools] m4 = ${M4_EXECUTABLE}") + +# flex +find_program(FLEX_EXECUTABLE flex) +if(NOT FLEX_EXECUTABLE) + _iotdb_resolve_tarball(_flex_tarball "flex-${FLEX_VERSION}.tar.gz" "${_flex_url}") + _iotdb_build_autotools(flex "${_flex_tarball}" "flex-${FLEX_VERSION}") + find_program(FLEX_EXECUTABLE flex PATHS "${_tools_bin}" NO_DEFAULT_PATH REQUIRED) +endif() +message(STATUS "[BuildTools] flex = ${FLEX_EXECUTABLE}") + +# bison +find_program(BISON_EXECUTABLE bison) +if(NOT BISON_EXECUTABLE) + _iotdb_resolve_tarball(_bison_tarball "bison-${BISON_VERSION}.tar.gz" "${_bison_url}") + _iotdb_build_autotools(bison "${_bison_tarball}" "bison-${BISON_VERSION}") + find_program(BISON_EXECUTABLE bison PATHS "${_tools_bin}" NO_DEFAULT_PATH REQUIRED) +endif() +message(STATUS "[BuildTools] bison = ${BISON_EXECUTABLE}") + +# Expose the bin dir for downstream ExternalProject_Add calls. +set(IOTDB_LOCAL_TOOLS_BIN "${_tools_bin}" CACHE INTERNAL "") diff --git a/iotdb-client/client-cpp/cmake/FetchOpenSSL.cmake b/iotdb-client/client-cpp/cmake/FetchOpenSSL.cmake new file mode 100644 index 0000000000000..575e2803f2bd4 --- /dev/null +++ b/iotdb-client/client-cpp/cmake/FetchOpenSSL.cmake @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ============================================================================= +# FetchOpenSSL.cmake (only included when WITH_SSL=ON) +# +# Resolution order: +# 1. find_package(OpenSSL) - any system / vendor install is taken as-is. +# 2. On Linux/macOS: +# use tarball ${IOTDB_OS_DEPS_DIR}/openssl-${OPENSSL_VERSION}.tar.gz +# or download from openssl.org when not in offline mode, then +# ./Configure && make && make install_sw into ${CMAKE_BINARY_DIR}/_deps/openssl. +# 3. On Windows: emit a FATAL_ERROR with instructions to run the bundled +# Win64OpenSSL installer (or any other prebuilt OpenSSL); building +# OpenSSL from source on MSVC is out of scope. +# +# Side effects: +# Defines imported targets OpenSSL::SSL / OpenSSL::Crypto via find_package +# so callers can just link against them. +# ============================================================================= + +set(OPENSSL_VERSION "3.5.0" CACHE STRING "OpenSSL version to fetch when missing") + +find_package(OpenSSL QUIET) +if(OpenSSL_FOUND) + message(STATUS "[OpenSSL] using system OpenSSL ${OPENSSL_VERSION_MAJOR}.${OPENSSL_VERSION_MINOR}") + return() +endif() + +if(WIN32) + message(FATAL_ERROR + "[OpenSSL] WITH_SSL=ON but no OpenSSL was found on Windows. " + "Please run third-party/windows/Win64OpenSSL-3_5_0.exe (or any " + "OpenSSL installer), then re-run the configure step with " + "-DOPENSSL_ROOT_DIR=.") +endif() + +# --- Linux / macOS fallback: build from source --------------------------- +set(_ossl_tarname "openssl-${OPENSSL_VERSION}.tar.gz") +set(_ossl_tarball "${IOTDB_OS_DEPS_DIR}/${_ossl_tarname}") + +if(NOT EXISTS "${_ossl_tarball}") + if(IOTDB_OFFLINE) + message(FATAL_ERROR + "[OpenSSL] IOTDB_OFFLINE=ON but ${_ossl_tarname} is missing in ${IOTDB_OS_DEPS_DIR}.") + endif() + set(_ossl_url "https://www.openssl.org/source/${_ossl_tarname}") + message(STATUS "[OpenSSL] downloading ${_ossl_url}") + file(DOWNLOAD "${_ossl_url}" "${_ossl_tarball}" + SHOW_PROGRESS TLS_VERIFY ON STATUS _st) + list(GET _st 0 _code) + if(NOT _code EQUAL 0) + list(GET _st 1 _msg) + file(REMOVE "${_ossl_tarball}") + message(FATAL_ERROR "[OpenSSL] download failed: ${_msg}") + endif() +endif() + +set(_ossl_root "${CMAKE_BINARY_DIR}/_deps/openssl") +set(_ossl_src "${_ossl_root}/src/openssl-${OPENSSL_VERSION}") +set(_ossl_inst "${_ossl_root}/install") +set(_ossl_stamp "${_ossl_root}/.built-${OPENSSL_VERSION}") + +if(NOT EXISTS "${_ossl_stamp}") + file(REMOVE_RECURSE "${_ossl_root}/src") + file(MAKE_DIRECTORY "${_ossl_root}/src") + message(STATUS "[OpenSSL] extracting ${_ossl_tarball}") + file(ARCHIVE_EXTRACT INPUT "${_ossl_tarball}" DESTINATION "${_ossl_root}/src") + + include(ProcessorCount) + ProcessorCount(_jobs) + if(_jobs LESS 1) + set(_jobs 1) + endif() + + message(STATUS "[OpenSSL] configuring -> ${_ossl_inst}") + execute_process( + COMMAND ./Configure --prefix=${_ossl_inst} --openssldir=${_ossl_inst}/ssl no-shared + WORKING_DIRECTORY "${_ossl_src}" + RESULT_VARIABLE _rc) + if(NOT _rc EQUAL 0) + message(FATAL_ERROR "[OpenSSL] Configure failed (rc=${_rc})") + endif() + + message(STATUS "[OpenSSL] building (-j${_jobs})") + execute_process( + COMMAND make -j${_jobs} + WORKING_DIRECTORY "${_ossl_src}" + RESULT_VARIABLE _rc) + if(NOT _rc EQUAL 0) + message(FATAL_ERROR "[OpenSSL] make failed (rc=${_rc})") + endif() + + execute_process( + COMMAND make install_sw + WORKING_DIRECTORY "${_ossl_src}" + RESULT_VARIABLE _rc) + if(NOT _rc EQUAL 0) + message(FATAL_ERROR "[OpenSSL] make install_sw failed (rc=${_rc})") + endif() + file(TOUCH "${_ossl_stamp}") +endif() + +set(OPENSSL_ROOT_DIR "${_ossl_inst}" CACHE PATH "OpenSSL root" FORCE) +set(OPENSSL_USE_STATIC_LIBS ON) +find_package(OpenSSL REQUIRED) +message(STATUS "[OpenSSL] built locally at ${OPENSSL_ROOT_DIR}") diff --git a/iotdb-client/client-cpp/cmake/FetchThrift.cmake b/iotdb-client/client-cpp/cmake/FetchThrift.cmake new file mode 100644 index 0000000000000..d17e3075d2199 --- /dev/null +++ b/iotdb-client/client-cpp/cmake/FetchThrift.cmake @@ -0,0 +1,277 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ============================================================================= +# FetchThrift.cmake +# +# Downloads (or uses a local copy of) the Apache Thrift source tarball and +# builds it from source as a static-only, runtime-and-compiler artifact. The +# build runs at configure time so the thrift compiler is available for the +# code generation step that follows. +# +# Exported variables: +# THRIFT_EXECUTABLE - path to the thrift binary that just got built +# THRIFT_INCLUDE_DIR - include directory containing +# THRIFT_STATIC_LIB_PATH- path to libthrift.a / thriftmd.lib +# THRIFT_RUNTIME_LIB_DIR- directory containing the static thrift library +# (and any other build artifacts you may want to +# bundle with the SDK) +# +# Imported targets: +# iotdb_thrift_static - INTERFACE-compatible IMPORTED target that +# propagates include dirs and the static lib. +# iotdb_thrift_external - phony custom target representing the build +# (use add_dependencies( iotdb_thrift_external) +# to ensure the thrift build runs first). +# ============================================================================= + +include(ExternalProject) + +set(_thrift_dirname "thrift-${THRIFT_VERSION}") +set(_thrift_tarname "${_thrift_dirname}.tar.gz") + +# --------------------------------------------------------------------------- +# Resolve tarball (local cache -> download) +# --------------------------------------------------------------------------- +set(_thrift_tarball "${IOTDB_OS_DEPS_DIR}/${_thrift_tarname}") +if(NOT EXISTS "${_thrift_tarball}") + if(IOTDB_OFFLINE) + message(FATAL_ERROR + "[Thrift] IOTDB_OFFLINE=ON but ${_thrift_tarname} is missing in " + "${IOTDB_OS_DEPS_DIR}.") + endif() + set(_thrift_url "https://archive.apache.org/dist/thrift/${THRIFT_VERSION}/${_thrift_tarname}") + message(STATUS "[Thrift] downloading ${_thrift_url}") + file(DOWNLOAD "${_thrift_url}" "${_thrift_tarball}" + SHOW_PROGRESS TLS_VERIFY ON STATUS _thrift_dl) + list(GET _thrift_dl 0 _code) + if(NOT _code EQUAL 0) + list(GET _thrift_dl 1 _msg) + file(REMOVE "${_thrift_tarball}") + message(FATAL_ERROR "[Thrift] download failed: ${_msg}") + endif() +endif() + +# --------------------------------------------------------------------------- +# Extract once into ${CMAKE_BINARY_DIR}/_deps/thrift/src +# --------------------------------------------------------------------------- +set(_thrift_root "${CMAKE_BINARY_DIR}/_deps/thrift") +set(_thrift_src "${_thrift_root}/src/${_thrift_dirname}") +set(_thrift_build "${_thrift_root}/build") +set(_thrift_install "${_thrift_root}/install") +set(_thrift_marker "${_thrift_root}/.extracted-${THRIFT_VERSION}") + +set(_thrift_build_config "Release") +if(MSVC AND CMAKE_BUILD_TYPE) + set(_thrift_build_config "${CMAKE_BUILD_TYPE}") +endif() + +if(NOT EXISTS "${_thrift_marker}") + file(REMOVE_RECURSE "${_thrift_root}/src") + file(MAKE_DIRECTORY "${_thrift_root}/src") + message(STATUS "[Thrift] extracting ${_thrift_tarball}") + file(ARCHIVE_EXTRACT INPUT "${_thrift_tarball}" + DESTINATION "${_thrift_root}/src") + file(TOUCH "${_thrift_marker}") +endif() + +if(NOT EXISTS "${_thrift_src}/CMakeLists.txt") + message(FATAL_ERROR + "[Thrift] could not find ${_thrift_src}/CMakeLists.txt after " + "extracting ${_thrift_tarball}.") +endif() + +# --------------------------------------------------------------------------- +# ExternalProject_Add: build thrift at *configure* time so the produced +# binary / library can immediately drive code generation and linking. +# --------------------------------------------------------------------------- +set(_thrift_cmake_args + # CMake 4.x rejects Thrift 0.21's cmake_minimum_required(3.0); set policy first. + "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" + "-DCMAKE_INSTALL_PREFIX=${_thrift_install}" + "-DCMAKE_BUILD_TYPE=${_thrift_build_config}" + "-DBUILD_JAVA=OFF" + "-DBUILD_NODEJS=OFF" + "-DBUILD_JAVASCRIPT=OFF" + "-DBUILD_PYTHON=OFF" + "-DBUILD_TESTING=OFF" + "-DBUILD_TUTORIALS=OFF" + "-DBUILD_SHARED_LIBS=OFF" + "-DWITH_SHARED_LIB=OFF" + "-DWITH_STATIC_LIB=ON" + "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + "-DCMAKE_POLICY_DEFAULT_CMP0091=NEW" + "-DCMAKE_CXX_STANDARD=11") + +if(BOOST_INCLUDE_DIR) + list(APPEND _thrift_cmake_args + "-DBoost_INCLUDE_DIR=${BOOST_INCLUDE_DIR}" + "-DBOOST_INCLUDEDIR=${BOOST_INCLUDE_DIR}") +endif() + +if(MSVC) + list(APPEND _thrift_cmake_args + "-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded$<$:Debug>DLL") +else() + set(_thrift_cxxflags "-fPIC") + if(IOTDB_LIBSTDCXX_CXX11_ABI) + set(_thrift_cxxflags "-D_GLIBCXX_USE_CXX11_ABI=1 -fPIC") + elseif(IOTDB_EXTRA_CXX_FLAGS MATCHES "_GLIBCXX_USE_CXX11_ABI=0") + set(_thrift_cxxflags "-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC") + elseif(IOTDB_EXTRA_CXX_FLAGS MATCHES "_GLIBCXX_USE_CXX11_ABI=1") + set(_thrift_cxxflags "-D_GLIBCXX_USE_CXX11_ABI=1 -fPIC") + endif() + list(APPEND _thrift_cmake_args + "-DCMAKE_C_FLAGS=-fPIC" + "-DCMAKE_CXX_FLAGS=${_thrift_cxxflags}") +endif() + +if(WITH_SSL) + list(APPEND _thrift_cmake_args "-DWITH_OPENSSL=ON") +else() + list(APPEND _thrift_cmake_args "-DWITH_OPENSSL=OFF") +endif() + +# Run the ExternalProject build immediately so the thrift compiler is +# available for the subsequent code-generation step. We do this by +# invoking cmake twice via execute_process and only register a phony +# ExternalProject for dependency ordering. + +set(_thrift_stamp "${_thrift_build}/.built-${THRIFT_VERSION}-${_thrift_build_config}-mdll") +if(NOT EXISTS "${_thrift_stamp}") + file(MAKE_DIRECTORY "${_thrift_build}") + message(STATUS "[Thrift] configuring ${_thrift_dirname}") + + # When we built m4/flex/bison locally, make sure CMake passes the + # updated PATH down to the nested cmake invocation. + if(IOTDB_LOCAL_TOOLS_BIN) + set(_thrift_env "PATH=${IOTDB_LOCAL_TOOLS_BIN}:$ENV{PATH}") + endif() + + if(CMAKE_GENERATOR_PLATFORM) + set(_gen_platform_arg "-A" "${CMAKE_GENERATOR_PLATFORM}") + else() + set(_gen_platform_arg "") + endif() + + execute_process( + COMMAND ${CMAKE_COMMAND} + -G "${CMAKE_GENERATOR}" + ${_gen_platform_arg} + ${_thrift_cmake_args} + "${_thrift_src}" + WORKING_DIRECTORY "${_thrift_build}" + RESULT_VARIABLE _rc) + if(NOT _rc EQUAL 0) + message(FATAL_ERROR "[Thrift] configure step failed (rc=${_rc})") + endif() + + message(STATUS "[Thrift] building (${_thrift_build_config})") + execute_process( + COMMAND ${CMAKE_COMMAND} --build . --config ${_thrift_build_config} --target install + WORKING_DIRECTORY "${_thrift_build}" + RESULT_VARIABLE _rc) + if(NOT _rc EQUAL 0) + message(FATAL_ERROR "[Thrift] build/install step failed (rc=${_rc})") + endif() + file(TOUCH "${_thrift_stamp}") +endif() + +# --------------------------------------------------------------------------- +# Locate produced artifacts +# --------------------------------------------------------------------------- +set(THRIFT_INCLUDE_DIR "${_thrift_install}/include" CACHE PATH "" FORCE) + +# Thrift binary +if(WIN32) + find_program(THRIFT_EXECUTABLE + NAMES thrift + HINTS "${_thrift_install}/bin" "${_thrift_build}/bin/Release" + "${_thrift_build}/compiler/cpp/Release" + "${_thrift_build}/compiler/cpp/bin/Release" + NO_DEFAULT_PATH) +else() + find_program(THRIFT_EXECUTABLE + NAMES thrift + HINTS "${_thrift_install}/bin" + "${_thrift_build}/bin" + "${_thrift_build}/compiler/cpp/bin" + "${_thrift_build}/compiler/cpp" + NO_DEFAULT_PATH) +endif() + +if(NOT THRIFT_EXECUTABLE) + message(FATAL_ERROR + "[Thrift] could not find the thrift binary under ${_thrift_install} or ${_thrift_build}") +endif() +message(STATUS "[Thrift] THRIFT_EXECUTABLE = ${THRIFT_EXECUTABLE}") + +# Thrift static library (search a few standard install/lib locations). +set(_thrift_libname_candidates) +if(MSVC) + if(_thrift_build_config STREQUAL "Debug") + list(APPEND _thrift_libname_candidates thriftmdd.lib thriftmd.lib thriftmtd.lib thriftmt.lib thrift.lib) + else() + list(APPEND _thrift_libname_candidates thriftmd.lib thriftmt.lib thrift.lib) + endif() +else() + list(APPEND _thrift_libname_candidates libthrift.a) +endif() + +set(THRIFT_STATIC_LIB_PATH "") +foreach(_dir + "${_thrift_install}/lib" + "${_thrift_install}/lib64" + "${_thrift_build}/lib" + "${_thrift_build}/lib/Release" + "${_thrift_build}/lib/release") + if(THRIFT_STATIC_LIB_PATH) + break() + endif() + foreach(_n ${_thrift_libname_candidates}) + if(EXISTS "${_dir}/${_n}") + set(THRIFT_STATIC_LIB_PATH "${_dir}/${_n}") + set(THRIFT_RUNTIME_LIB_DIR "${_dir}") + break() + endif() + endforeach() +endforeach() + +if(NOT THRIFT_STATIC_LIB_PATH) + message(FATAL_ERROR + "[Thrift] could not locate the thrift static library under ${_thrift_install}/lib") +endif() +message(STATUS "[Thrift] THRIFT_STATIC_LIB_PATH = ${THRIFT_STATIC_LIB_PATH}") + +# Cache as well so subsequent reconfigures keep the same values. +set(THRIFT_STATIC_LIB_PATH "${THRIFT_STATIC_LIB_PATH}" CACHE FILEPATH "" FORCE) +set(THRIFT_RUNTIME_LIB_DIR "${THRIFT_RUNTIME_LIB_DIR}" CACHE PATH "" FORCE) + +# --------------------------------------------------------------------------- +# Imported target wrapping the static library. +# --------------------------------------------------------------------------- +if(NOT TARGET iotdb_thrift_static) + add_library(iotdb_thrift_static STATIC IMPORTED GLOBAL) + set_target_properties(iotdb_thrift_static PROPERTIES + IMPORTED_LOCATION "${THRIFT_STATIC_LIB_PATH}" + INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}") +endif() + +# Phony target so downstream code can express ordering deps. +if(NOT TARGET iotdb_thrift_external) + add_custom_target(iotdb_thrift_external ALL) +endif() diff --git a/iotdb-client/client-cpp/cmake/GenerateThriftSources.cmake b/iotdb-client/client-cpp/cmake/GenerateThriftSources.cmake new file mode 100644 index 0000000000000..6a3685af45847 --- /dev/null +++ b/iotdb-client/client-cpp/cmake/GenerateThriftSources.cmake @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ============================================================================= +# GenerateThriftSources.cmake +# +# Generates C++ stubs from the iotdb-protocol Thrift files using the thrift +# compiler that FetchThrift just built. +# +# Inputs (resolved relative to the workspace root): +# iotdb-protocol/thrift-commons/src/main/thrift/common.thrift +# iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +# +# Outputs: +# ${THRIFT_GEN_CPP_DIR}/*.{h,cpp} - generated C++ sources +# THRIFT_GENERATED_SRCS - list of *.cpp files for linking +# THRIFT_GENERATED_HDRS - list of *.h files (informational) +# ============================================================================= + +if(NOT THRIFT_EXECUTABLE) + message(FATAL_ERROR "[GenThrift] THRIFT_EXECUTABLE not set - did FetchThrift run first?") +endif() + +# Anchor the source-of-truth .thrift files via the workspace root. The CMake +# project lives at /iotdb-client/client-cpp/, so the workspace root is +# two levels up. +get_filename_component(_workspace_root "${CMAKE_CURRENT_SOURCE_DIR}/../.." ABSOLUTE) + +set(_common_thrift "${_workspace_root}/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift") +set(_client_thrift "${_workspace_root}/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift") + +foreach(_f IN ITEMS "${_common_thrift}" "${_client_thrift}") + if(NOT EXISTS "${_f}") + message(FATAL_ERROR "[GenThrift] missing thrift input: ${_f}") + endif() +endforeach() + +# Mirror the bash reference script: stage both .thrift files into a flat +# working directory so `include "common.thrift"` resolves without -I args. +set(_thrift_proto_dir "${CMAKE_BINARY_DIR}/thrift-protocols") +file(MAKE_DIRECTORY "${_thrift_proto_dir}") +configure_file("${_common_thrift}" "${_thrift_proto_dir}/common.thrift" COPYONLY) +configure_file("${_client_thrift}" "${_thrift_proto_dir}/client.thrift" COPYONLY) + +set(THRIFT_GEN_CPP_DIR "${CMAKE_BINARY_DIR}/gen-cpp" CACHE PATH "" FORCE) +file(MAKE_DIRECTORY "${THRIFT_GEN_CPP_DIR}") + +# A stamp file lets us re-run thrift only when one of the inputs changes +# (handled by add_custom_command at build time) while also making sure we +# run once at configure time so the initial file(GLOB) finds something. +set(_thrift_stamp "${THRIFT_GEN_CPP_DIR}/.generated-stamp") + +set(_thrift_inputs + "${_thrift_proto_dir}/common.thrift" + "${_thrift_proto_dir}/client.thrift") + +set(_run_thrift FALSE) +if(NOT EXISTS "${_thrift_stamp}") + set(_run_thrift TRUE) +else() + foreach(_in IN LISTS _thrift_inputs) + if("${_in}" IS_NEWER_THAN "${_thrift_stamp}") + set(_run_thrift TRUE) + break() + endif() + endforeach() +endif() + +if(_run_thrift) + message(STATUS "[GenThrift] running ${THRIFT_EXECUTABLE} on common.thrift / client.thrift") + foreach(_in IN LISTS _thrift_inputs) + execute_process( + COMMAND "${THRIFT_EXECUTABLE}" -r --gen cpp:no_skeleton -out "${THRIFT_GEN_CPP_DIR}" "${_in}" + WORKING_DIRECTORY "${_thrift_proto_dir}" + RESULT_VARIABLE _rc) + if(NOT _rc EQUAL 0) + message(FATAL_ERROR "[GenThrift] thrift compile failed for ${_in} (rc=${_rc})") + endif() + endforeach() + # Defensive: remove any accidentally produced server skeleton. + file(GLOB _skeletons "${THRIFT_GEN_CPP_DIR}/*_server.skeleton.cpp") + if(_skeletons) + file(REMOVE ${_skeletons}) + endif() + file(TOUCH "${_thrift_stamp}") +endif() + +# Build-time regeneration: whenever the workspace .thrift files change, rerun +# the thrift compiler. The OUTPUT is the stamp; downstream targets that +# depend on iotdb_thrift_codegen will get the rebuild for free. +add_custom_command( + OUTPUT "${_thrift_stamp}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${_common_thrift}" "${_thrift_proto_dir}/common.thrift" + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${_client_thrift}" "${_thrift_proto_dir}/client.thrift" + COMMAND "${THRIFT_EXECUTABLE}" -r --gen cpp:no_skeleton -out "${THRIFT_GEN_CPP_DIR}" "${_thrift_proto_dir}/common.thrift" + COMMAND "${THRIFT_EXECUTABLE}" -r --gen cpp:no_skeleton -out "${THRIFT_GEN_CPP_DIR}" "${_thrift_proto_dir}/client.thrift" + COMMAND ${CMAKE_COMMAND} -E touch "${_thrift_stamp}" + DEPENDS "${_common_thrift}" "${_client_thrift}" iotdb_thrift_external + WORKING_DIRECTORY "${_thrift_proto_dir}" + COMMENT "Regenerating thrift C++ stubs" + VERBATIM) + +add_custom_target(iotdb_thrift_codegen DEPENDS "${_thrift_stamp}") + +# Glob results with CONFIGURE_DEPENDS so re-cmake picks up newly produced files. +file(GLOB THRIFT_GENERATED_SRCS CONFIGURE_DEPENDS "${THRIFT_GEN_CPP_DIR}/*.cpp") +file(GLOB THRIFT_GENERATED_HDRS CONFIGURE_DEPENDS "${THRIFT_GEN_CPP_DIR}/*.h") + +list(FILTER THRIFT_GENERATED_SRCS EXCLUDE REGEX ".*_server\\.skeleton\\.cpp$") + +message(STATUS "[GenThrift] generated ${THRIFT_GEN_CPP_DIR} with " + "$ cpp files") diff --git a/iotdb-client/client-cpp/cmake/iotdb-session-config.cmake.in b/iotdb-client/client-cpp/cmake/iotdb-session-config.cmake.in new file mode 100644 index 0000000000000..57207aa2fa71a --- /dev/null +++ b/iotdb-client/client-cpp/cmake/iotdb-session-config.cmake.in @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +get_filename_component(_IOTDB_SESSION_PACKAGE_ROOT "${CMAKE_CURRENT_LIST_DIR}/.." ABSOLUTE) + +if(UNIX) + include(CMakeFindDependencyMacro) + find_dependency(Threads) +endif() + +if(WIN32) + set(_IOTDB_SESSION_LIBRARY "${_IOTDB_SESSION_PACKAGE_ROOT}/lib/iotdb_session.lib") +elseif(APPLE) + set(_IOTDB_SESSION_LIBRARY "${_IOTDB_SESSION_PACKAGE_ROOT}/lib/libiotdb_session.dylib") +else() + set(_IOTDB_SESSION_LIBRARY "${_IOTDB_SESSION_PACKAGE_ROOT}/lib/libiotdb_session.so") +endif() + +if(NOT TARGET IoTDB::iotdb_session) + add_library(IoTDB::iotdb_session SHARED IMPORTED) + set_target_properties(IoTDB::iotdb_session PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_IOTDB_SESSION_PACKAGE_ROOT}/include") + set(_IOTDB_SESSION_CXX11_ABI_COMPILE_DEFINITION "@IOTDB_SESSION_CXX11_ABI_COMPILE_DEFINITION@") + if(_IOTDB_SESSION_CXX11_ABI_COMPILE_DEFINITION) + set_target_properties(IoTDB::iotdb_session PROPERTIES + INTERFACE_COMPILE_DEFINITIONS "${_IOTDB_SESSION_CXX11_ABI_COMPILE_DEFINITION}") + endif() + if(UNIX) + set_target_properties(IoTDB::iotdb_session PROPERTIES + INTERFACE_LINK_LIBRARIES Threads::Threads) + endif() + if(WIN32) + set_target_properties(IoTDB::iotdb_session PROPERTIES + IMPORTED_IMPLIB "${_IOTDB_SESSION_LIBRARY}" + IMPORTED_LOCATION "${_IOTDB_SESSION_PACKAGE_ROOT}/lib/iotdb_session.dll") + else() + set_target_properties(IoTDB::iotdb_session PROPERTIES + IMPORTED_LOCATION "${_IOTDB_SESSION_LIBRARY}") + endif() +endif() + +set(iotdb-session_VERSION "@IOTDB_SESSION_VERSION@") diff --git a/iotdb-client/client-cpp/cmake/iotdb-session.pc.in b/iotdb-client/client-cpp/cmake/iotdb-session.pc.in new file mode 100644 index 0000000000000..0cc6833e1ce46 --- /dev/null +++ b/iotdb-client/client-cpp/cmake/iotdb-session.pc.in @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +prefix=${pcfiledir}/.. +libdir=${prefix}/lib +includedir=${prefix}/include + +Name: iotdb-session-cpp +Description: Apache IoTDB C++ session client +Version: @IOTDB_SESSION_VERSION@ +Libs: -L${libdir} @IOTDB_SESSION_PC_LIBS@ +Cflags: -I${includedir} @IOTDB_SESSION_PC_CFLAGS@ diff --git a/iotdb-client/client-cpp/pom.xml b/iotdb-client/client-cpp/pom.xml index af57b78817191..418ef22fdf635 100644 --- a/iotdb-client/client-cpp/pom.xml +++ b/iotdb-client/client-cpp/pom.xml @@ -30,14 +30,38 @@ pom IoTDB: Client: Client for CPP C++ client - + https://github.com/catchorg/Catch2/releases/download/v2.13.7/catch.hpp Release - ${project.build.directory}/dependency/cmake/ - ${project.build.directory}/thrift/bin/${thrift.executable} + ${project.basedir} + ${project.build.directory}/build + ${project.build.directory}/install + ${project.basedir}/third-party + OFF + OFF + + ON ${ctest.skip.tests} + + + + OFF + + + ${os.classifier} + iotdb-session-cpp-${project.version}-${client.cpp.package.classifier} @@ -49,168 +73,30 @@ - - - org.apache.maven.plugins - maven-resources-plugin - - - copy-test-resources - - copy-resources - - validate - - ${project.build.directory}/build/test - - - ${project.basedir}/src/test - true - - - - - - - - org.apache.maven.plugins - maven-resources-plugin - - - copy-cpp-files - - copy-resources - - validate - - ${project.build.directory}/build/main/generated-sources-cpp - - - ${project.basedir}/src/main - - **/*.h - **/*.cpp - **/*.cc - - - - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - get-thrift - - unpack - - generate-sources - - - - org.apache.iotdb.tools - iotdb-tools-thrift - ${iotdb-tools-thrift.version} - ${os.classifier} - zip - true - ${project.build.directory}/thrift - - - - - - com.googlecode.maven-download-plugin download-maven-plugin - get-catch2 wget - generate-test-resources + generate-resources ${catch2.url} false - ${project.build.directory}/build/test/catch2 - ${ctest.skip.tests} + ${project.build.directory}/test/catch2 + ${maven.test.skip} - com.coderplus.maven.plugins - copy-rename-maven-plugin - - - - copy-protocol-thrift-source - - copy - - - - - ../../iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift - ${project.build.directory}/protocols/client.thrift - - - ../../iotdb-protocol/thrift-commons/src/main/thrift/common.thrift - ${project.build.directory}/protocols/common.thrift - - - - - - - - copy-cmakelist-file - - copy - - compile - - - - ${project.basedir}/src/main/CMakeLists.txt - ${project.build.directory}/build/main/CMakeLists.txt - - - - - - - - org.apache.thrift - thrift-maven-plugin - - - generate-thrift-sources-cpp - - compile - - - generate-resources - - cpp:no_skeleton - ${thrift.exec.absolute.path} - ${project.build.directory}/protocols - ${project.build.directory}/build/main/generated-sources-cpp - - - - - - com.googlecode.cmake-maven-project + io.github.cmake-maven-plugin cmake-maven-plugin - + cmake-generate @@ -219,30 +105,25 @@ compile ${cmake.generator} - ${project.build.directory}/build/main - ${project.build.directory}/build/main - - - - - - - - cmake-generate-test - - generate - - test-compile - - ${cmake.generator} - ${project.build.directory}/build/test - ${project.build.directory}/build/test + ${cmake.source.dir} + ${cmake.project.dir} + + + + + + + + + + + - + cmake-compile @@ -251,24 +132,11 @@ compile ${cmake.build.type} - - ${project.build.directory}/build/main - - - - - cmake-compile-test - - compile - - test-compile - - ${cmake.build.type} - ${project.build.directory}/build/test + ${cmake.project.dir} + install - - + cmake-run-test @@ -276,37 +144,18 @@ integration-test - ${project.build.directory}/build/test - ${ctest.skip.tests} - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - package-client-cpp - - single - - package - - ${project.artifactId}-${project.version} - - src/assembly/client-cpp.xml - + ${cmake.build.type} + ${cmake.project.dir}/test + ${maven.test.skip} + com.bazaarvoice.maven.plugins process-exec-maven-plugin - start-iotdb @@ -314,20 +163,18 @@ pre-integration-test - ${ctest.skip.tests} + ${maven.test.skip} iotdb-server false - - 15 - - ${project.build.directory}/build/test/test.log + 45 + ${cmake.project.dir}/test.log - + ${iotdb.shell.command} + ${iotdb.shell.option} ${project.basedir}/../../distribution/target/apache-iotdb-${project.version}-all-bin/apache-iotdb-${project.version}-all-bin/sbin/${iotdb.start.script} - stop-iotdb @@ -338,17 +185,85 @@ ${ctest.skip.tests} iotdb-server false - 5 - - ${project.build.directory}/build/test/test1.log + ${cmake.project.dir}/stop.log + ${iotdb.shell.command} + ${iotdb.shell.option} ${project.basedir}/../../distribution/target/apache-iotdb-${project.version}-all-bin/apache-iotdb-${project.version}-all-bin/sbin/${iotdb.stop.script} + + maven-resources-plugin + + + prepare-client-cpp-package-metadata + + copy-resources + + prepare-package + + ${project.build.directory}/package-metadata + + + ${project.basedir}/src/assembly/package-metadata + true + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + package-client-cpp + + single + + package + + iotdb-session-cpp-${project.version} + + src/assembly/client-cpp.xml + + + + + + + + net.nicoulaj.maven.plugins + checksum-maven-plugin + + + checksum-client-cpp-package + + files + + package + + + SHA-512 + + + + ${project.build.directory} + + ${client.cpp.package.name}.zip + + + + + + + @@ -362,9 +277,12 @@ + bash + -c start-standalone.sh stop-standalone.sh linux + linux-x86_64-glibc2.28 @@ -377,9 +295,12 @@ + bash + -c start-standalone.sh stop-standalone.sh linux + linux-aarch64-glibc2.28 @@ -391,9 +312,12 @@ + bash + -c start-standalone.sh stop-standalone.sh mac + macos-x86_64 @@ -405,9 +329,12 @@ + bash + -c start-standalone.sh stop-standalone.sh mac + macos-aarch64 @@ -418,33 +345,89 @@ + cmd + /c start-standalone.bat stop-standalone.bat win + windows-x86_64-msvc14.3 + - .skipTests + client-cpp-vs-x64 - - skipTests - + + windows + - com.googlecode.cmake-maven-project + io.github.cmake-maven-plugin cmake-maven-plugin - cmake-compile-test - none - + cmake-generate + + + + + + + .skipTests + + + skipTests + + + + OFF + + + + + spotless-cpp + + [11,) + + + + + com.diffplug.spotless + spotless-maven-plugin + 2.44.5 + + + + src/**/*.h + src/**/*.hpp + src/**/*.c + src/**/*.cc + src/**/*.cpp + + + test/catch2/** + + + ${clang.format.version} + + + + + + + + diff --git a/iotdb-client/client-cpp/src/assembly/client-cpp.xml b/iotdb-client/client-cpp/src/assembly/client-cpp.xml index 729fdadde235a..4764523cc3799 100644 --- a/iotdb-client/client-cpp/src/assembly/client-cpp.xml +++ b/iotdb-client/client-cpp/src/assembly/client-cpp.xml @@ -17,51 +17,78 @@ specific language governing permissions and limitations under the License. --> + - cpp-${os.classifier} + ${client.cpp.package.classifier} zip dir - false + true + ${client.cpp.package.name} - ${project.build.directory}/build/main/generated-sources-cpp + ${project.basedir}/../.. - **/*.h + LICENSE + NOTICE - include + ${file.separator} + + + ${project.basedir} + + README.md + + ${file.separator} + + + ${project.build.directory}/package-metadata + + third_party/DEPENDENCIES.md + + ${file.separator} + + + ${project.build.directory}/install + + VERSION + BUILD-INFO.txt + + ${file.separator} - ${project.build.directory}/thrift/include + ${project.build.directory}/install/include ** include - ${project.build.directory}/thrift/lib + ${project.build.directory}/install/lib ** lib - ${project.build.directory}/build/main + ${project.build.directory}/install/cmake - *.so - *.dylib - **/*.dll - **/*.lib + ** - lib + cmake - ${project.basedir}/../compile-tools/thrift/target/build/lib/Release + ${project.build.directory}/install/pkgconfig - **/*.lib + ** - lib + pkgconfig diff --git a/iotdb-client/client-cpp/src/assembly/package-metadata/third_party/DEPENDENCIES.md b/iotdb-client/client-cpp/src/assembly/package-metadata/third_party/DEPENDENCIES.md new file mode 100644 index 0000000000000..e921c7eb94833 --- /dev/null +++ b/iotdb-client/client-cpp/src/assembly/package-metadata/third_party/DEPENDENCIES.md @@ -0,0 +1,34 @@ + +# Third-party Dependencies + +The release library is built with the following third-party components. Some +components are linked into the produced IoTDB C++ session library; this file is +included for provenance. + +| Component | Version | License | +| --- | --- | --- | +| Apache Thrift | 0.21.0 | Apache License 2.0 | +| Boost | 1.60.0 on Linux/Windows, 1.84.0 on macOS by default | Boost Software License 1.0 | +| OpenSSL | 3.5.0 when `WITH_SSL=ON` | Apache License 2.0 | +| GNU m4 | 1.4.19 on Linux build bootstrap | GPL-3.0-or-later | +| GNU flex | 2.6.4 on Linux build bootstrap | BSD-style flex license | +| GNU bison | 3.8 on Linux build bootstrap | GPL-3.0-or-later | diff --git a/iotdb-client/client-cpp/src/include/AbstractSessionBuilder.h b/iotdb-client/client-cpp/src/include/AbstractSessionBuilder.h new file mode 100644 index 0000000000000..8d5ec5729c514 --- /dev/null +++ b/iotdb-client/client-cpp/src/include/AbstractSessionBuilder.h @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef IOTDB_ABSTRACTSESSIONBUILDER_H +#define IOTDB_ABSTRACTSESSIONBUILDER_H + +#include +#include + +#include "SessionConfig.h" + +class AbstractSessionBuilder { +public: + static constexpr const char *DEFAULT_HOST = "localhost"; + static constexpr int DEFAULT_RPC_PORT = 6667; + static constexpr const char *DEFAULT_USERNAME = "root"; + static constexpr const char *DEFAULT_PASSWORD = "root"; + static constexpr int DEFAULT_FETCH_SIZE = iotdb::session::DEFAULT_FETCH_SIZE; + static constexpr int DEFAULT_CONNECT_TIMEOUT_MS = + iotdb::session::DEFAULT_CONNECT_TIMEOUT_MS; + static constexpr int DEFAULT_MAX_RETRIES = + iotdb::session::DEFAULT_MAX_RETRIES; + static constexpr int DEFAULT_RETRY_DELAY_MS = + iotdb::session::DEFAULT_RETRY_DELAY_MS; + static constexpr const char *DEFAULT_SQL_DIALECT = "tree"; + static constexpr bool DEFAULT_ENABLE_AUTO_FETCH = true; + static constexpr bool DEFAULT_ENABLE_REDIRECTIONS = true; + static constexpr bool DEFAULT_ENABLE_RPC_COMPRESSION = false; + + std::string host = DEFAULT_HOST; + int rpcPort = DEFAULT_RPC_PORT; + std::string username = DEFAULT_USERNAME; + std::string password = DEFAULT_PASSWORD; + std::string zoneId = ""; + int fetchSize = DEFAULT_FETCH_SIZE; + int connectTimeoutMs = DEFAULT_CONNECT_TIMEOUT_MS; + int maxRetries = DEFAULT_MAX_RETRIES; + int retryDelayMs = DEFAULT_RETRY_DELAY_MS; + std::string sqlDialect = DEFAULT_SQL_DIALECT; + std::string database = ""; + bool enableAutoFetch = DEFAULT_ENABLE_AUTO_FETCH; + bool enableRedirections = DEFAULT_ENABLE_REDIRECTIONS; + bool enableRPCCompression = DEFAULT_ENABLE_RPC_COMPRESSION; + std::vector nodeUrls; + bool useSSL = false; + std::string trustCertFilePath; +}; + +#endif // IOTDB_ABSTRACTSESSIONBUILDER_H \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/include/Column.h b/iotdb-client/client-cpp/src/include/Column.h new file mode 100644 index 0000000000000..548e577c01a01 --- /dev/null +++ b/iotdb-client/client-cpp/src/include/Column.h @@ -0,0 +1,364 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_COLUMN_H +#define IOTDB_COLUMN_H + +#include +#include +#include +#include + +#include "ColumnDecoder.h" +#include "Common.h" + +enum class ColumnEncoding : uint8_t { + ByteArray, + Int32Array, + Int64Array, + BinaryArray, + Rle +}; + +class Binary { +public: + explicit Binary(std::vector data) : data_(std::move(data)) {} + + const std::vector &getData() const { return data_; } + + std::string getStringValue() const { return {data_.begin(), data_.end()}; } + +private: + std::vector data_; +}; + +const std::map> + kEncodingToDecoder = { + {ColumnEncoding::Int32Array, + std::make_shared()}, + {ColumnEncoding::Int64Array, + std::make_shared()}, + {ColumnEncoding::ByteArray, std::make_shared()}, + {ColumnEncoding::BinaryArray, + std::make_shared()}, + {ColumnEncoding::Rle, std::make_shared()}}; + +const std::map kByteToEncoding = { + {0, ColumnEncoding::ByteArray}, + {1, ColumnEncoding::Int32Array}, + {2, ColumnEncoding::Int64Array}, + {3, ColumnEncoding::BinaryArray}, + {4, ColumnEncoding::Rle}}; + +inline std::shared_ptr +getColumnDecoder(ColumnEncoding encoding) { + auto it = kEncodingToDecoder.find(encoding); + if (it == kEncodingToDecoder.end()) { + throw IoTDBException("Unsupported column encoding"); + } + return it->second; +} + +inline ColumnEncoding getColumnEncodingByByte(uint8_t b) { + auto it = kByteToEncoding.find(b); + if (it == kByteToEncoding.end()) { + throw IoTDBException("Invalid encoding value: " + std::to_string(b)); + } + return it->second; +} + +class Column { +public: + virtual ~Column() = default; + + virtual TSDataType::TSDataType getDataType() const = 0; + virtual ColumnEncoding getEncoding() const = 0; + + virtual bool getBoolean(int32_t position) const { + throw IoTDBException("Unsupported operation: getBoolean"); + } + + virtual int32_t getInt(int32_t position) const { + throw IoTDBException("Unsupported operation: getInt"); + } + + virtual int64_t getLong(int32_t position) const { + throw IoTDBException("Unsupported operation: getLong"); + } + + virtual float getFloat(int32_t position) const { + throw IoTDBException("Unsupported operation: getFloat"); + } + + virtual double getDouble(int32_t position) const { + throw IoTDBException("Unsupported operation: getDouble"); + } + + virtual std::shared_ptr getBinary(int32_t position) const { + throw IoTDBException("Unsupported operation: getBinary"); + } + + virtual std::vector getBooleans() const { + throw IoTDBException("Unsupported operation: getBooleans"); + } + + virtual std::vector getInts() const { + throw IoTDBException("Unsupported operation: getInts"); + } + + virtual std::vector getLongs() const { + throw IoTDBException("Unsupported operation: getLongs"); + } + + virtual std::vector getFloats() const { + throw IoTDBException("Unsupported operation: getFloats"); + } + + virtual std::vector getDoubles() const { + throw IoTDBException("Unsupported operation: getDoubles"); + } + + virtual std::vector> getBinaries() const { + throw IoTDBException("Unsupported operation: getBinaries"); + } + + virtual bool mayHaveNull() const = 0; + virtual bool isNull(int32_t position) const = 0; + virtual std::vector isNulls() const = 0; + + virtual int32_t getPositionCount() const = 0; +}; + +class TimeColumn : public Column { +public: + TimeColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &values); + + TSDataType::TSDataType getDataType() const override; + ColumnEncoding getEncoding() const override; + + int64_t getLong(int32_t position) const override; + + bool mayHaveNull() const override; + bool isNull(int32_t position) const override; + std::vector isNulls() const override; + + int32_t getPositionCount() const override; + + int64_t getStartTime() const; + int64_t getEndTime() const; + + const std::vector &getTimes() const; + std::vector getLongs() const override; + +private: + int32_t arrayOffset_; + int32_t positionCount_; + std::vector values_; +}; + +class BinaryColumn : public Column { +public: + BinaryColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &valueIsNull, + const std::vector> &values); + + TSDataType::TSDataType getDataType() const override; + ColumnEncoding getEncoding() const override; + + std::shared_ptr getBinary(int32_t position) const override; + std::vector> getBinaries() const override; + + bool mayHaveNull() const override; + bool isNull(int32_t position) const override; + std::vector isNulls() const override; + + int32_t getPositionCount() const override; + +private: + int32_t arrayOffset_; + int32_t positionCount_; + std::vector valueIsNull_; + std::vector> values_; +}; + +class IntColumn : public Column { +public: + IntColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &valueIsNull, + const std::vector &values); + + TSDataType::TSDataType getDataType() const override; + ColumnEncoding getEncoding() const override; + + int32_t getInt(int32_t position) const override; + int64_t getLong(int32_t position) const override; + float getFloat(int32_t position) const override; + double getDouble(int32_t position) const override; + std::vector getInts() const override; + + bool mayHaveNull() const override; + bool isNull(int32_t position) const override; + std::vector isNulls() const override; + + int32_t getPositionCount() const override; + +private: + int32_t arrayOffset_; + int32_t positionCount_; + std::vector valueNull_; + std::vector values_; +}; + +class FloatColumn : public Column { +public: + FloatColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &valueIsNull, + const std::vector &values); + + TSDataType::TSDataType getDataType() const override; + ColumnEncoding getEncoding() const override; + + float getFloat(int32_t position) const override; + double getDouble(int32_t position) const override; + std::vector getFloats() const override; + + bool mayHaveNull() const override; + bool isNull(int32_t position) const override; + std::vector isNulls() const override; + + int32_t getPositionCount() const override; + +private: + int32_t arrayOffset_; + int32_t positionCount_; + std::vector valueIsNull_; + std::vector values_; +}; + +class LongColumn : public Column { +public: + LongColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &valueIsNull, + const std::vector &values); + + TSDataType::TSDataType getDataType() const override; + ColumnEncoding getEncoding() const override; + + int64_t getLong(int32_t position) const override; + double getDouble(int32_t position) const override; + std::vector getLongs() const override; + + bool mayHaveNull() const override; + bool isNull(int32_t position) const override; + std::vector isNulls() const override; + + int32_t getPositionCount() const override; + +private: + int32_t arrayOffset_; + int32_t positionCount_; + std::vector valueIsNull_; + std::vector values_; +}; + +class DoubleColumn : public Column { +public: + DoubleColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &valueIsNull, + const std::vector &values); + + TSDataType::TSDataType getDataType() const override; + ColumnEncoding getEncoding() const override; + + double getDouble(int32_t position) const override; + std::vector getDoubles() const override; + + bool mayHaveNull() const override; + bool isNull(int32_t position) const override; + std::vector isNulls() const override; + + int32_t getPositionCount() const override; + +private: + int32_t arrayOffset_; + int32_t positionCount_; + std::vector valueIsNull_; + std::vector values_; +}; + +class BooleanColumn : public Column { +public: + BooleanColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &valueIsNull, + const std::vector &values); + + TSDataType::TSDataType getDataType() const override; + ColumnEncoding getEncoding() const override; + + bool getBoolean(int32_t position) const override; + std::vector getBooleans() const override; + + bool mayHaveNull() const override; + bool isNull(int32_t position) const override; + std::vector isNulls() const override; + + int32_t getPositionCount() const override; + +private: + int32_t arrayOffset_; + int32_t positionCount_; + std::vector valueIsNull_; + std::vector values_; +}; + +class RunLengthEncodedColumn : public Column { +public: + RunLengthEncodedColumn(std::shared_ptr value, int32_t positionCount); + + std::shared_ptr getValue() const; + + TSDataType::TSDataType getDataType() const override; + ColumnEncoding getEncoding() const override; + + bool getBoolean(int32_t position) const override; + int32_t getInt(int32_t position) const override; + int64_t getLong(int32_t position) const override; + float getFloat(int32_t position) const override; + double getDouble(int32_t position) const override; + std::shared_ptr getBinary(int32_t position) const override; + + std::vector getBooleans() const override; + std::vector getInts() const override; + std::vector getLongs() const override; + std::vector getFloats() const override; + std::vector getDoubles() const override; + std::vector> getBinaries() const override; + + bool mayHaveNull() const override; + bool isNull(int32_t position) const override; + std::vector isNulls() const override; + + int32_t getPositionCount() const override; + +private: + std::shared_ptr value_; + int32_t positionCount_; +}; + +#endif diff --git a/iotdb-client/client-cpp/src/include/ColumnDecoder.h b/iotdb-client/client-cpp/src/include/ColumnDecoder.h new file mode 100644 index 0000000000000..6affa72060ef1 --- /dev/null +++ b/iotdb-client/client-cpp/src/include/ColumnDecoder.h @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_COLUMN_DECODER_H +#define IOTDB_COLUMN_DECODER_H + +#include +#include + +#include "Common.h" + +class Column; + +class ColumnDecoder { +public: + virtual ~ColumnDecoder() = default; + virtual std::unique_ptr readColumn(MyStringBuffer &buffer, + TSDataType::TSDataType dataType, + int32_t positionCount) = 0; +}; + +std::vector deserializeNullIndicators(MyStringBuffer &buffer, + int32_t positionCount); +std::vector deserializeBooleanArray(MyStringBuffer &buffer, int32_t size); + +class BaseColumnDecoder : public ColumnDecoder { +public: + std::unique_ptr readColumn(MyStringBuffer &buffer, + TSDataType::TSDataType dataType, + int32_t positionCount) override; +}; + +class Int32ArrayColumnDecoder : public BaseColumnDecoder { +public: + std::unique_ptr readColumn(MyStringBuffer &buffer, + TSDataType::TSDataType dataType, + int32_t positionCount) override; +}; + +class Int64ArrayColumnDecoder : public BaseColumnDecoder { +public: + std::unique_ptr readColumn(MyStringBuffer &buffer, + TSDataType::TSDataType dataType, + int32_t positionCount) override; +}; + +class ByteArrayColumnDecoder : public BaseColumnDecoder { +public: + std::unique_ptr readColumn(MyStringBuffer &buffer, + TSDataType::TSDataType dataType, + int32_t positionCount) override; +}; + +class BinaryArrayColumnDecoder : public BaseColumnDecoder { +public: + std::unique_ptr readColumn(MyStringBuffer &buffer, + TSDataType::TSDataType dataType, + int32_t positionCount) override; +}; + +class RunLengthColumnDecoder : public BaseColumnDecoder { +public: + std::unique_ptr readColumn(MyStringBuffer &buffer, + TSDataType::TSDataType dataType, + int32_t positionCount) override; +}; + +#endif diff --git a/iotdb-client/client-cpp/src/include/Common.h b/iotdb-client/client-cpp/src/include/Common.h new file mode 100644 index 0000000000000..c203edf2de69f --- /dev/null +++ b/iotdb-client/client-cpp/src/include/Common.h @@ -0,0 +1,428 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_COMMON_H +#define IOTDB_COMMON_H + +#include +#include +#include +#include +#include +#include + +#include "Date.h" +#include "Endpoint.h" +#include "Export.h" +#include "Optional.h" +#include "Status.h" + +std::string getTimePrecision(int32_t timeFactor); + +std::string formatDatetime(const std::string &format, + const std::string &precision, int64_t timestamp, + const std::string &zoneId); + +std::tm convertToTimestamp(int64_t value, int32_t timeFactor); +std::tm int32ToDate(int32_t value); + +namespace Version { +enum Version { V_0_12, V_0_13, V_1_0 }; +} + +namespace CompressionType { +enum CompressionType { + UNCOMPRESSED = (char)0, + SNAPPY = (char)1, + GZIP = (char)2, + LZO = (char)3, + SDT = (char)4, + PAA = (char)5, + PLA = (char)6, + LZ4 = (char)7, + ZSTD = (char)8, + LZMA2 = (char)9, +}; +} + +namespace TSDataType { +enum TSDataType { + BOOLEAN = (char)0, + INT32 = (char)1, + INT64 = (char)2, + FLOAT = (char)3, + DOUBLE = (char)4, + TEXT = (char)5, + VECTOR = (char)6, + UNKNOWN = (char)7, + TIMESTAMP = (char)8, + DATE = (char)9, + BLOB = (char)10, + STRING = (char)11, + OBJECT = (char)12 +}; +} + +TSDataType::TSDataType getDataTypeByStr(const std::string &typeStr); + +namespace TSEncoding { +enum TSEncoding { + PLAIN = (char)0, + DICTIONARY = (char)1, + RLE = (char)2, + DIFF = (char)3, + TS_2DIFF = (char)4, + BITMAP = (char)5, + GORILLA_V1 = (char)6, + REGULAR = (char)7, + GORILLA = (char)8, + ZIGZAG = (char)9, + FREQ = (char)10, + INVALID_ENCODING = (char)255 +}; +} + +namespace TSStatusCode { +enum TSStatusCode { + SUCCESS_STATUS = 200, + + INCOMPATIBLE_VERSION = 201, + CONFIGURATION_ERROR = 202, + START_UP_ERROR = 203, + SHUT_DOWN_ERROR = 204, + + UNSUPPORTED_OPERATION = 300, + EXECUTE_STATEMENT_ERROR = 301, + MULTIPLE_ERROR = 302, + ILLEGAL_PARAMETER = 303, + OVERLAP_WITH_EXISTING_TASK = 304, + INTERNAL_SERVER_ERROR = 305, + + REDIRECTION_RECOMMEND = 400, + + DATABASE_NOT_EXIST = 500, + DATABASE_ALREADY_EXISTS = 501, + SERIES_OVERFLOW = 502, + TIMESERIES_ALREADY_EXIST = 503, + TIMESERIES_IN_BLACK_LIST = 504, + ALIAS_ALREADY_EXIST = 505, + PATH_ALREADY_EXIST = 506, + METADATA_ERROR = 507, + PATH_NOT_EXIST = 508, + ILLEGAL_PATH = 509, + CREATE_TEMPLATE_ERROR = 510, + DUPLICATED_TEMPLATE = 511, + UNDEFINED_TEMPLATE = 512, + TEMPLATE_NOT_SET = 513, + DIFFERENT_TEMPLATE = 514, + TEMPLATE_IS_IN_USE = 515, + TEMPLATE_INCOMPATIBLE = 516, + SEGMENT_NOT_FOUND = 517, + PAGE_OUT_OF_SPACE = 518, + RECORD_DUPLICATED = 519, + SEGMENT_OUT_OF_SPACE = 520, + PBTREE_FILE_NOT_EXISTS = 521, + OVERSIZE_RECORD = 522, + PBTREE_FILE_REDO_LOG_BROKEN = 523, + TEMPLATE_NOT_ACTIVATED = 524, + + SYSTEM_READ_ONLY = 600, + STORAGE_ENGINE_ERROR = 601, + STORAGE_ENGINE_NOT_READY = 602, + + PLAN_FAILED_NETWORK_PARTITION = 721 +}; +} + +class Field { +public: + TSDataType::TSDataType dataType = TSDataType::UNKNOWN; + Optional boolV; + Optional intV; + Optional dateV; + Optional longV; + Optional floatV; + Optional doubleV; + Optional stringV; + + explicit Field(TSDataType::TSDataType a) { dataType = a; } + + Field() = default; + + bool isNull() const { + switch (dataType) { + case TSDataType::BOOLEAN: + return !boolV.is_initialized(); + case TSDataType::INT32: + return !intV.is_initialized(); + case TSDataType::INT64: + case TSDataType::TIMESTAMP: + return !longV.is_initialized(); + case TSDataType::FLOAT: + return !floatV.is_initialized(); + case TSDataType::DOUBLE: + return !doubleV.is_initialized(); + case TSDataType::TEXT: + case TSDataType::STRING: + case TSDataType::BLOB: + return !stringV.is_initialized(); + case TSDataType::DATE: + return !dateV.is_initialized(); + default: + return true; + } + } +}; + +enum class ColumnCategory { TAG, FIELD, ATTRIBUTE }; + +class MyStringBuffer { +public: + MyStringBuffer(); + explicit MyStringBuffer(const std::string &str); + + void reserve(size_t n); + void clear(); + bool hasRemaining(); + int getInt(); + IoTDBDate getDate(); + int64_t getInt64(); + float getFloat(); + double getDouble(); + char getChar(); + bool getBool(); + std::string getString(); + + void putInt(int ins); + void putDate(IoTDBDate date); + void putInt64(int64_t ins); + void putFloat(float ins); + void putDouble(double ins); + void putChar(char ins); + void putBool(bool ins); + void putString(const std::string &ins); + void concat(const std::string &ins); + +public: + std::string str; + size_t pos; + +private: + void checkBigEndian(); + const char *getOrderedByte(size_t len); + void putOrderedByte(char *buf, int len); + +private: + bool isBigEndian{}; + char numericBuf[8]{}; +}; + +class BitMap { +public: + explicit BitMap(size_t size = 0); + void resize(size_t size); + bool mark(size_t position); + bool unmark(size_t position); + void markAll(); + void reset(); + bool isMarked(size_t position) const; + bool isAllUnmarked() const; + bool isAllMarked() const; + const std::vector &getByteArray() const; + size_t getSize() const; + +private: + size_t size; + std::vector bits; +}; + +class IoTDBException : public std::exception { +public: + IoTDBException() = default; + + explicit IoTDBException(const std::string &m) : message(m) {} + + explicit IoTDBException(const char *m) : message(m) {} + + virtual const char *what() const noexcept override { return message.c_str(); } + +private: + std::string message; +}; + +std::string extractExceptionMessage(const std::exception &exception); +std::string extractExceptionMessage(const std::exception_ptr &exceptionPtr); + +class DateTimeParseException : public IoTDBException { +private: + std::string parsedString; + int errorIndex; + +public: + explicit DateTimeParseException(const std::string &message, + std::string parsedData, int errorIndex) + : IoTDBException(message), parsedString(std::move(parsedData)), + errorIndex(errorIndex) {} + + explicit DateTimeParseException(const std::string &message, + std::string parsedData, int errorIndex, + const std::exception &cause) + : IoTDBException(message + " [Caused by: " + cause.what() + "]"), + parsedString(std::move(parsedData)), errorIndex(errorIndex) {} + + const std::string &getParsedString() const noexcept { return parsedString; } + + int getErrorIndex() const noexcept { return errorIndex; } + + const char *what() const noexcept override { + static std::string fullMsg; + fullMsg = std::string(IoTDBException::what()) + + "\nParsed data: " + parsedString + + "\nError index: " + std::to_string(errorIndex); + return fullMsg.c_str(); + } +}; + +class IoTDBConnectionException : public IoTDBException { +public: + IoTDBConnectionException() {} + + explicit IoTDBConnectionException(const char *m) : IoTDBException(m) {} + + explicit IoTDBConnectionException(const std::string &m) : IoTDBException(m) {} +}; + +class ExecutionException : public IoTDBException { +public: + ExecutionException() {} + + explicit ExecutionException(const char *m) : IoTDBException(m) {} + + explicit ExecutionException(const std::string &m) : IoTDBException(m) {} + + explicit ExecutionException(const std::string &m, const Status &tsStatus) + : IoTDBException(m), status(tsStatus) {} + + Status status; +}; + +class BatchExecutionException : public IoTDBException { +public: + BatchExecutionException() {} + + explicit BatchExecutionException(const char *m) : IoTDBException(m) {} + + explicit BatchExecutionException(const std::string &m) : IoTDBException(m) {} + + explicit BatchExecutionException(const std::vector &statusList) + : statusList(statusList) {} + + BatchExecutionException(const std::string &m, + const std::vector &statusList) + : IoTDBException(m), statusList(statusList) {} + + std::vector statusList; +}; + +class RedirectException : public IoTDBException { +public: + RedirectException() {} + + explicit RedirectException(const char *m) : IoTDBException(m) {} + + explicit RedirectException(const std::string &m) : IoTDBException(m) {} + + RedirectException(const std::string &m, const Endpoint &endPoint) + : IoTDBException(m), endPoint(endPoint) {} + + RedirectException(const std::string &m, + const std::map &deviceEndPointMap) + : IoTDBException(m), deviceEndPointMap(deviceEndPointMap) {} + + RedirectException(const std::string &m, + const std::vector &endPointList) + : IoTDBException(m), endPointList(endPointList) {} + + Endpoint endPoint; + std::map deviceEndPointMap; + std::vector endPointList; +}; + +class UnSupportedDataTypeException : public IoTDBException { +public: + UnSupportedDataTypeException() {} + + explicit UnSupportedDataTypeException(const char *m) : IoTDBException(m) {} + + explicit UnSupportedDataTypeException(const std::string &m) + : IoTDBException("UnSupported dataType: " + m) {} +}; + +class SchemaNotFoundException : public IoTDBException { +public: + SchemaNotFoundException() {} + + explicit SchemaNotFoundException(const char *m) : IoTDBException(m) {} + + explicit SchemaNotFoundException(const std::string &m) : IoTDBException(m) {} +}; + +class StatementExecutionException : public IoTDBException { +public: + StatementExecutionException() {} + + explicit StatementExecutionException(const char *m) : IoTDBException(m) {} + + explicit StatementExecutionException(const std::string &m) + : IoTDBException(m) {} +}; + +enum LogLevelType { LEVEL_DEBUG = 0, LEVEL_INFO, LEVEL_WARN, LEVEL_ERROR }; + +extern IOTDB_SESSION_API LogLevelType LOG_LEVEL; + +#define log_debug(fmt, ...) \ + do { \ + if (LOG_LEVEL <= LEVEL_DEBUG) { \ + std::string s = std::string("[DEBUG] %s:%d (%s) - ") + fmt + "\n"; \ + printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__); \ + } \ + } while (0) +#define log_info(fmt, ...) \ + do { \ + if (LOG_LEVEL <= LEVEL_INFO) { \ + std::string s = std::string("[INFO] %s:%d (%s) - ") + fmt + "\n"; \ + printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__); \ + } \ + } while (0) +#define log_warn(fmt, ...) \ + do { \ + if (LOG_LEVEL <= LEVEL_WARN) { \ + std::string s = std::string("[WARN] %s:%d (%s) - ") + fmt + "\n"; \ + printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__); \ + } \ + } while (0) +#define log_error(fmt, ...) \ + do { \ + if (LOG_LEVEL <= LEVEL_ERROR) { \ + std::string s = std::string("[ERROR] %s:%d (%s) - ") + fmt + "\n"; \ + printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__); \ + } \ + } while (0) + +#endif diff --git a/iotdb-client/client-cpp/src/include/Date.h b/iotdb-client/client-cpp/src/include/Date.h new file mode 100644 index 0000000000000..91fe286f0c4d0 --- /dev/null +++ b/iotdb-client/client-cpp/src/include/Date.h @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_DATE_H +#define IOTDB_DATE_H + +#include +#include + +constexpr int32_t EMPTY_DATE_INT = 10000101; + +class IoTDBDate { +public: + IoTDBDate() : valid_(false), year_(0), month_(0), day_(0) {} + + IoTDBDate(int year, int month, int day) + : valid_(true), year_(year), month_(month), day_(day) {} + + static IoTDBDate notADate() { return IoTDBDate(); } + + bool is_not_a_date() const { return !valid_; } + + int year() const { return year_; } + + int month() const { return month_; } + + int day() const { return day_; } + + std::string toIsoExtendedString() const; + + friend bool operator==(const IoTDBDate &lhs, const IoTDBDate &rhs) { + return lhs.valid_ == rhs.valid_ && lhs.year_ == rhs.year_ && + lhs.month_ == rhs.month_ && lhs.day_ == rhs.day_; + } + + friend bool operator!=(const IoTDBDate &lhs, const IoTDBDate &rhs) { + return !(lhs == rhs); + } + +private: + bool valid_; + int year_; + int month_; + int day_; +}; + +int32_t parseDateExpressionToInt(const IoTDBDate &date); +IoTDBDate parseIntToDate(int32_t dateInt); + +#endif diff --git a/iotdb-client/client-cpp/src/main/AbstractSessionBuilder.h b/iotdb-client/client-cpp/src/include/Endpoint.h similarity index 61% rename from iotdb-client/client-cpp/src/main/AbstractSessionBuilder.h rename to iotdb-client/client-cpp/src/include/Endpoint.h index 441e3a41f4fac..9c43a6f48c1b8 100644 --- a/iotdb-client/client-cpp/src/main/AbstractSessionBuilder.h +++ b/iotdb-client/client-cpp/src/include/Endpoint.h @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -16,25 +16,29 @@ * specific language governing permissions and limitations * under the License. */ +#ifndef IOTDB_ENDPOINT_H +#define IOTDB_ENDPOINT_H -#ifndef IOTDB_ABSTRACTSESSIONBUILDER_H -#define IOTDB_ABSTRACTSESSIONBUILDER_H - +#include +#include #include -class AbstractSessionBuilder { -public: - std::string host = "localhost"; - int rpcPort = 6667; - std::string username = "root"; - std::string password = "root"; - std::string zoneId = ""; - int fetchSize = 10000; - std::string sqlDialect = "tree"; - std::string database = ""; - bool enableAutoFetch = true; - bool enableRedirections = true; - bool enableRPCCompression = false; +struct Endpoint { + std::string host; + int32_t port = 0; +}; + +struct EndpointHash { + size_t operator()(const Endpoint &endpoint) const { + return std::hash()(endpoint.host) ^ + std::hash()(endpoint.port); + } +}; + +struct EndpointEqual { + bool operator()(const Endpoint &lhs, const Endpoint &rhs) const { + return lhs.host == rhs.host && lhs.port == rhs.port; + } }; -#endif // IOTDB_ABSTRACTSESSIONBUILDER_H \ No newline at end of file +#endif diff --git a/iotdb-client/client-cpp/src/include/Export.h b/iotdb-client/client-cpp/src/include/Export.h new file mode 100644 index 0000000000000..71262d0cffb77 --- /dev/null +++ b/iotdb-client/client-cpp/src/include/Export.h @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_EXPORT_H +#define IOTDB_EXPORT_H + +#if defined(_WIN32) && defined(IOTDB_BUILDING_SHARED) +#define IOTDB_SESSION_API __declspec(dllexport) +#elif defined(_WIN32) +#define IOTDB_SESSION_API __declspec(dllimport) +#else +#define IOTDB_SESSION_API +#endif + +#endif // IOTDB_EXPORT_H diff --git a/iotdb-client/client-cpp/src/include/Optional.h b/iotdb-client/client-cpp/src/include/Optional.h new file mode 100644 index 0000000000000..f8c1d63af8757 --- /dev/null +++ b/iotdb-client/client-cpp/src/include/Optional.h @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_OPTIONAL_H +#define IOTDB_OPTIONAL_H + +#include + +template class Optional { +public: + Optional() : hasValue_(false) {} + + Optional(const T &value) : hasValue_(true), value_(value) {} + + Optional(T &&value) : hasValue_(true), value_(std::move(value)) {} + + static Optional of(const T &value) { return Optional(value); } + + static Optional of(T &&value) { return Optional(std::move(value)); } + + static Optional none() { return Optional(); } + + bool is_initialized() const { return hasValue_; } + + bool has_value() const { return hasValue_; } + + const T &value() const { return value_; } + + T &value() { return value_; } + + const T &get() const { return value_; } + + T &get() { return value_; } + + explicit operator bool() const { return hasValue_; } + +private: + bool hasValue_; + T value_{}; +}; + +#endif diff --git a/iotdb-client/client-cpp/src/include/Session.h b/iotdb-client/client-cpp/src/include/Session.h new file mode 100644 index 0000000000000..af33e343bb7af --- /dev/null +++ b/iotdb-client/client-cpp/src/include/Session.h @@ -0,0 +1,828 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_SESSION_H +#define IOTDB_SESSION_H + +#include "AbstractSessionBuilder.h" +#include "Common.h" +#include "Date.h" +#include "SessionDataSet.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +//== For compatible with Windows OS == +#ifndef LONG_LONG_MIN +#define LONG_LONG_MIN 0x8000000000000000 +#endif + +template +void safe_cast(const T &value, Target &target) { + /* + Target Allowed Source Types + BOOLEAN BOOLEAN + INT32 INT32 + INT64 INT32 INT64 + FLOAT INT32 FLOAT + DOUBLE INT32 INT64 FLOAT DOUBLE + TEXT TEXT + */ + if (std::is_same::value) { + target = *(Target *)&value; + } else if (std::is_same::value && + std::is_array::value && + std::is_same::type>::value) { + std::string tmp((const char *)&value); + target = *(Target *)&tmp; + } else if (std::is_same::value && + std::is_same::value) { + int64_t tmp = *(int32_t *)&value; + target = *(Target *)&tmp; + } else if (std::is_same::value && + std::is_same::value) { + float tmp = *(int32_t *)&value; + target = *(Target *)&tmp; + } else if (std::is_same::value && + std::is_same::value) { + double tmp = *(int32_t *)&value; + target = *(Target *)&tmp; + } else if (std::is_same::value && + std::is_same::value) { + double tmp = *(int64_t *)&value; + target = *(Target *)&tmp; + } else if (std::is_same::value && + std::is_same::value) { + double tmp = *(float *)&value; + target = *(Target *)&tmp; + } else { + throw UnSupportedDataTypeException("Error: Parameter type " + + std::string(typeid(T).name()) + + " cannot be converted to DataType" + + std::string(typeid(Target).name())); + } +} + +/* + * A tablet data of one device, the tablet contains multiple measurements of + * this device that share the same time column. + * + * for example: device root.sg1.d1 + * + * time, m1, m2, m3 + * 1, 1, 2, 3 + * 2, 1, 2, 3 + * 3, 1, 2, 3 + * + * Notice: The tablet should not have empty cell + * + */ +class Tablet { +private: + static const int DEFAULT_ROW_SIZE = 1024; + + void createColumns(); + void deleteColumns(); + +public: + std::string deviceId; // deviceId of this tablet + std::vector> schemas; + // the list of measurement schemas for creating the tablet + std::map + schemaNameIndex; // the map of schema name to index + std::vector + columnTypes; // column category (TAG/FIELD/ATTRIBUTE); tree clients use + // FIELD only + std::vector timestamps; // timestamps in this tablet + std::vector values; // each object is a primitive type array, which + // represents values of one measurement + std::vector bitMaps; // each bitmap represents the existence of each + // value in the current column + size_t rowSize; // the number of rows to include in this tablet + size_t maxRowNumber; // the maximum number of rows for this tablet + bool isAligned; // whether this tablet store data of aligned timeseries or not + + Tablet() = default; + + /** + * Return a tablet with default specified row number. This is the standard + * constructor (all Tablet should be the same size). + * + * @param deviceId the name of the device specified to be written in + * @param timeseries the list of measurement schemas for creating the tablet + */ + Tablet(const std::string &deviceId, + const std::vector> + ×eries) + : Tablet(deviceId, timeseries, DEFAULT_ROW_SIZE) {} + + /** + * Return a tablet with the specified number of rows (maxBatchSize). Only + * call this constructor directly for testing purposes. Tablet should normally + * always be default size. + * + * @param deviceId the name of the device specified to be written in + * @param schemas the list of measurement schemas for creating the row + * batch + * @param columnTypes column category per schema (tree clients use FIELD only) + * @param maxRowNumber the maximum number of rows for this tablet + */ + Tablet(const std::string &deviceId, + const std::vector> + &schemas, + int maxRowNumber) + : Tablet( + deviceId, schemas, + std::vector(schemas.size(), ColumnCategory::FIELD), + maxRowNumber) {} + + Tablet(const std::string &deviceId, + const std::vector> + &schemas, + const std::vector columnTypes, size_t maxRowNumber, + bool _isAligned = false) + : deviceId(deviceId), schemas(schemas), columnTypes(columnTypes), + maxRowNumber(maxRowNumber), isAligned(_isAligned) { + // create timestamp column + timestamps.resize(maxRowNumber); + // create value columns + values.resize(schemas.size()); + createColumns(); + // create bitMaps + bitMaps.resize(schemas.size()); + for (size_t i = 0; i < schemas.size(); i++) { + bitMaps[i].resize(maxRowNumber); + } + // create schemaNameIndex + for (size_t i = 0; i < schemas.size(); i++) { + schemaNameIndex[schemas[i].first] = i; + } + this->rowSize = 0; + } + + Tablet(const Tablet &other) + : deviceId(other.deviceId), schemas(other.schemas), + schemaNameIndex(other.schemaNameIndex), columnTypes(other.columnTypes), + timestamps(other.timestamps), maxRowNumber(other.maxRowNumber), + bitMaps(other.bitMaps), rowSize(other.rowSize), + isAligned(other.isAligned) { + values.resize(other.values.size()); + for (size_t i = 0; i < other.values.size(); ++i) { + if (!other.values[i]) + continue; + TSDataType::TSDataType type = schemas[i].second; + deepCopyTabletColValue(&(other.values[i]), &values[i], type, + maxRowNumber); + } + } + + Tablet &operator=(const Tablet &other) { + if (this != &other) { + deleteColumns(); + deviceId = other.deviceId; + schemas = other.schemas; + schemaNameIndex = other.schemaNameIndex; + columnTypes = other.columnTypes; + timestamps = other.timestamps; + maxRowNumber = other.maxRowNumber; + rowSize = other.rowSize; + isAligned = other.isAligned; + bitMaps = other.bitMaps; + values.resize(other.values.size()); + for (size_t i = 0; i < other.values.size(); ++i) { + if (!other.values[i]) + continue; + TSDataType::TSDataType type = schemas[i].second; + deepCopyTabletColValue(&(other.values[i]), &values[i], type, + maxRowNumber); + } + } + return *this; + } + + ~Tablet() { + try { + deleteColumns(); + } catch (std::exception &e) { + log_debug(std::string("Tablet::~Tablet(), ") + e.what()); + } + } + + void addTimestamp(size_t rowIndex, int64_t timestamp) { + timestamps[rowIndex] = timestamp; + rowSize = (std::max)(rowSize, rowIndex + 1); + } + + static void deepCopyTabletColValue(void *const *srcPtr, void **destPtr, + TSDataType::TSDataType type, + int maxRowNumber); + + template + void addValue(size_t schemaId, size_t rowIndex, const T &value) { + if (schemaId >= schemas.size()) { + char tmpStr[100]; + snprintf(tmpStr, sizeof(tmpStr), + "Tablet::addValue(), schemaId >= schemas.size(). schemaId=%ld, " + "schemas.size()=%ld.", + (long)schemaId, (long)schemas.size()); + throw std::out_of_range(tmpStr); + } + + if (rowIndex >= rowSize) { + char tmpStr[100]; + snprintf(tmpStr, sizeof(tmpStr), + "Tablet::addValue(), rowIndex >= rowSize. rowIndex=%ld, " + "rowSize.size()=%ld.", + (long)rowIndex, (long)rowSize); + throw std::out_of_range(tmpStr); + } + + TSDataType::TSDataType dataType = schemas[schemaId].second; + switch (dataType) { + case TSDataType::BOOLEAN: { + safe_cast(value, ((bool *)values[schemaId])[rowIndex]); + break; + } + case TSDataType::INT32: { + safe_cast(value, ((int *)values[schemaId])[rowIndex]); + break; + } + case TSDataType::DATE: { + safe_cast(value, ((IoTDBDate *)values[schemaId])[rowIndex]); + break; + } + case TSDataType::TIMESTAMP: + case TSDataType::INT64: { + safe_cast(value, ((int64_t *)values[schemaId])[rowIndex]); + break; + } + case TSDataType::FLOAT: { + safe_cast(value, ((float *)values[schemaId])[rowIndex]); + break; + } + case TSDataType::DOUBLE: { + safe_cast(value, ((double *)values[schemaId])[rowIndex]); + break; + } + case TSDataType::BLOB: + case TSDataType::STRING: + case TSDataType::TEXT: { + safe_cast(value, + ((std::string *)values[schemaId])[rowIndex]); + break; + } + default: + throw UnSupportedDataTypeException(std::string("Data type ") + + std::to_string(dataType) + + " is not supported."); + } + } + + // Add a Binary value with extra metadata: [isEOF (1 byte)] + [offset (8 + // bytes)] + [actual content] + void addValue(size_t schemaId, size_t rowIndex, bool isEOF, int64_t offset, + const std::vector &content) { + // Check schemaId bounds + if (schemaId >= schemas.size()) { + char tmpStr[100]; + sprintf(tmpStr, + "Tablet::addBinaryValueWithMeta(), schemaId >= schemas.size(). " + "schemaId=%ld, " + "schemas.size()=%ld.", + schemaId, schemas.size()); + throw std::out_of_range(tmpStr); + } + + // Check rowIndex bounds + if (rowIndex >= rowSize) { + char tmpStr[100]; + sprintf(tmpStr, + "Tablet::addBinaryValueWithMeta(), rowIndex >= rowSize. " + "rowIndex=%ld, rowSize=%ld.", + rowIndex, rowSize); + throw std::out_of_range(tmpStr); + } + + TSDataType::TSDataType dataType = schemas[schemaId].second; + if (dataType != TSDataType::OBJECT) { + throw std::invalid_argument("The data type of schemaId " + + std::to_string(schemaId) + " is not OBJECT."); + } + + // Create a byte array of size [1 (isEOF) + 8 (offset) + content size] + std::vector val(content.size() + 9); + + // Write the isEOF flag (1 byte) + val[0] = isEOF ? 1 : 0; + + // Write the 8-byte offset in big-endian order + for (int i = 0; i < 8; ++i) { + val[1 + i] = static_cast((offset >> (56 - i * 8)) & 0xFF); + } + + // Append the content bytes + std::copy(content.begin(), content.end(), val.begin() + 9); + + // Cast the value array and assign the Binary data (stored as string) + std::string valEncoded = + std::string(reinterpret_cast(val.data()), val.size()); + safe_cast( + valEncoded, ((std::string *)values[schemaId])[rowIndex]); + } + + void addValue(const std::string &schemaName, size_t rowIndex, bool isEOF, + int64_t offset, const std::vector &content) { + if (schemaNameIndex.find(schemaName) == schemaNameIndex.end()) { + throw SchemaNotFoundException(std::string("Schema ") + schemaName + + " not found."); + } + size_t schemaId = schemaNameIndex[schemaName]; + addValue(schemaId, rowIndex, isEOF, offset, content); + } + + template + void addValue(const std::string &schemaName, size_t rowIndex, + const T &value) { + if (schemaNameIndex.find(schemaName) == schemaNameIndex.end()) { + throw SchemaNotFoundException(std::string("Schema ") + schemaName + + " not found."); + } + size_t schemaId = schemaNameIndex[schemaName]; + addValue(schemaId, rowIndex, value); + } + + void *getValue(size_t schemaId, size_t rowIndex, + TSDataType::TSDataType dataType) { + if (schemaId >= schemas.size()) { + throw std::out_of_range("Tablet::getValue schemaId out of range: " + + std::to_string(schemaId)); + } + if (rowIndex >= rowSize) { + throw std::out_of_range("Tablet::getValue rowIndex out of range: " + + std::to_string(rowIndex)); + } + + switch (dataType) { + case TSDataType::BOOLEAN: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + case TSDataType::INT32: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + case TSDataType::DATE: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + case TSDataType::TIMESTAMP: + case TSDataType::INT64: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + case TSDataType::FLOAT: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + case TSDataType::DOUBLE: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + case TSDataType::BLOB: + case TSDataType::STRING: + case TSDataType::OBJECT: + case TSDataType::TEXT: + return &(reinterpret_cast(values[schemaId])[rowIndex]); + default: + throw UnSupportedDataTypeException("Unsupported data type: " + + std::to_string(dataType)); + } + } + + std::vector> + getSchemas() const { + return schemas; + } + + void reset(); // Reset Tablet to the default state - set the rowSize to 0 + + size_t getTimeBytesSize(); + + size_t getValueByteSize(); // total byte size that values occupies + + void setAligned(bool isAligned); +}; + +class SessionUtils { +public: + static std::string getTime(const Tablet &tablet); + + static std::string getValue(const Tablet &tablet); +}; + +class TemplateNode { +public: + explicit TemplateNode(const std::string &name) : name_(name) {} + + const std::string &getName() const { return name_; } + + virtual const std::unordered_map> & + getChildren() const { + throw BatchExecutionException("Should call exact sub class!"); + } + + virtual bool isMeasurement() = 0; + + virtual bool isAligned() { + throw BatchExecutionException("Should call exact sub class!"); + } + + virtual std::string serialize() const { + throw BatchExecutionException("Should call exact sub class!"); + } + +private: + std::string name_; +}; + +class MeasurementNode : public TemplateNode { +public: + MeasurementNode(const std::string &name_, TSDataType::TSDataType data_type_, + TSEncoding::TSEncoding encoding_, + CompressionType::CompressionType compression_type_) + : TemplateNode(name_) { + this->data_type_ = data_type_; + this->encoding_ = encoding_; + this->compression_type_ = compression_type_; + } + + TSDataType::TSDataType getDataType() const { return data_type_; } + + TSEncoding::TSEncoding getEncoding() const { return encoding_; } + + CompressionType::CompressionType getCompressionType() const { + return compression_type_; + } + + bool isMeasurement() override { return true; } + + std::string serialize() const override; + +private: + TSDataType::TSDataType data_type_; + TSEncoding::TSEncoding encoding_; + CompressionType::CompressionType compression_type_; +}; + +class InternalNode : public TemplateNode { +public: + InternalNode(const std::string &name, bool is_aligned) + : TemplateNode(name), is_aligned_(is_aligned) {} + + void addChild(const InternalNode &node) { + if (this->children_.count(node.getName())) { + throw BatchExecutionException("Duplicated child of node in template."); + } + this->children_[node.getName()] = std::make_shared(node); + } + + void addChild(const MeasurementNode &node) { + if (this->children_.count(node.getName())) { + throw BatchExecutionException("Duplicated child of node in template."); + } + this->children_[node.getName()] = std::make_shared(node); + } + + void deleteChild(const TemplateNode &node) { + this->children_.erase(node.getName()); + } + + const std::unordered_map> & + getChildren() const override { + return children_; + } + + bool isMeasurement() override { return false; } + + bool isAligned() override { return is_aligned_; } + +private: + std::unordered_map> children_; + bool is_aligned_; +}; + +namespace TemplateQueryType { +enum TemplateQueryType { + COUNT_MEASUREMENTS, + IS_MEASUREMENT, + PATH_EXIST, + SHOW_MEASUREMENTS +}; +} + +class Template { +public: + Template(const std::string &name, bool is_aligned) + : name_(name), is_aligned_(is_aligned) {} + + const std::string &getName() const { return name_; } + + bool isAligned() const { return is_aligned_; } + + void addToTemplate(const InternalNode &child) { + if (this->children_.count(child.getName())) { + throw BatchExecutionException("Duplicated child of node in template."); + } + this->children_[child.getName()] = std::make_shared(child); + } + + void addToTemplate(const MeasurementNode &child) { + if (this->children_.count(child.getName())) { + throw BatchExecutionException("Duplicated child of node in template."); + } + this->children_[child.getName()] = std::make_shared(child); + } + + std::string serialize() const; + +private: + std::string name_; + std::unordered_map> children_; + bool is_aligned_; +}; + +class SessionConnection; + +class Session { + struct Impl; + std::unique_ptr impl_; + friend class SessionConnection; + +public: + Session(const std::string &host, int rpcPort); + Session(const std::vector &nodeUrls, const std::string &username, + const std::string &password); + Session(const std::string &host, int rpcPort, const std::string &username, + const std::string &password); + Session(const std::string &host, int rpcPort, const std::string &username, + const std::string &password, const std::string &zoneId, + int fetchSize = AbstractSessionBuilder::DEFAULT_FETCH_SIZE); + Session(const std::string &host, const std::string &rpcPort, + const std::string &username = "user", + const std::string &password = "password", + const std::string &zoneId = "", + int fetchSize = AbstractSessionBuilder::DEFAULT_FETCH_SIZE); + Session(AbstractSessionBuilder *builder); + ~Session(); + + void open(); + + void open(bool enableRPCCompression); + + void open(bool enableRPCCompression, int connectionTimeoutInMs); + + void close(); + + void setTimeZone(const std::string &zoneId); + + std::string getTimeZone(); + + void insertRecord(const std::string &deviceId, int64_t time, + const std::vector &measurements, + const std::vector &values); + + void insertRecord(const std::string &deviceId, int64_t time, + const std::vector &measurements, + const std::vector &types, + const std::vector &values); + + void insertAlignedRecord(const std::string &deviceId, int64_t time, + const std::vector &measurements, + const std::vector &values); + + void insertAlignedRecord(const std::string &deviceId, int64_t time, + const std::vector &measurements, + const std::vector &types, + const std::vector &values); + + void + insertRecords(const std::vector &deviceIds, + const std::vector ×, + const std::vector> &measurementsList, + const std::vector> &valuesList); + + void insertRecords( + const std::vector &deviceIds, + const std::vector ×, + const std::vector> &measurementsList, + const std::vector> &typesList, + const std::vector> &valuesList); + + void insertAlignedRecords( + const std::vector &deviceIds, + const std::vector ×, + const std::vector> &measurementsList, + const std::vector> &valuesList); + + void insertAlignedRecords( + const std::vector &deviceIds, + const std::vector ×, + const std::vector> &measurementsList, + const std::vector> &typesList, + const std::vector> &valuesList); + + void insertRecordsOfOneDevice( + const std::string &deviceId, std::vector ×, + std::vector> &measurementsList, + std::vector> &typesList, + std::vector> &valuesList); + + void insertRecordsOfOneDevice( + const std::string &deviceId, std::vector ×, + std::vector> &measurementsList, + std::vector> &typesList, + std::vector> &valuesList, bool sorted); + + void insertAlignedRecordsOfOneDevice( + const std::string &deviceId, std::vector ×, + std::vector> &measurementsList, + std::vector> &typesList, + std::vector> &valuesList); + + void insertAlignedRecordsOfOneDevice( + const std::string &deviceId, std::vector ×, + std::vector> &measurementsList, + std::vector> &typesList, + std::vector> &valuesList, bool sorted); + + void insertTablet(Tablet &tablet); + + void insertTablet(Tablet &tablet, bool sorted); + + void insertAlignedTablet(Tablet &tablet); + + void insertAlignedTablet(Tablet &tablet, bool sorted); + + void insertTablets(std::unordered_map &tablets); + + void insertTablets(std::unordered_map &tablets, + bool sorted); + + void insertAlignedTablets(std::unordered_map &tablets, + bool sorted = false); + + void testInsertRecord(const std::string &deviceId, int64_t time, + const std::vector &measurements, + const std::vector &values); + + void testInsertTablet(const Tablet &tablet); + + void testInsertRecords( + const std::vector &deviceIds, + const std::vector ×, + const std::vector> &measurementsList, + const std::vector> &valuesList); + + void deleteTimeseries(const std::string &path); + + void deleteTimeseries(const std::vector &paths); + + void deleteData(const std::string &path, int64_t endTime); + + void deleteData(const std::vector &paths, int64_t endTime); + + void deleteData(const std::vector &paths, int64_t startTime, + int64_t endTime); + + void setStorageGroup(const std::string &storageGroupId); + + void deleteStorageGroup(const std::string &storageGroup); + + void deleteStorageGroups(const std::vector &storageGroups); + + void createDatabase(const std::string &database); + + void deleteDatabase(const std::string &database); + + void deleteDatabases(const std::vector &databases); + + void createTimeseries(const std::string &path, + TSDataType::TSDataType dataType, + TSEncoding::TSEncoding encoding, + CompressionType::CompressionType compressor); + + void createTimeseries(const std::string &path, + TSDataType::TSDataType dataType, + TSEncoding::TSEncoding encoding, + CompressionType::CompressionType compressor, + std::map *props, + std::map *tags, + std::map *attributes, + const std::string &measurementAlias); + + void createMultiTimeseries( + const std::vector &paths, + const std::vector &dataTypes, + const std::vector &encodings, + const std::vector &compressors, + std::vector> *propsList, + std::vector> *tagsList, + std::vector> *attributesList, + std::vector *measurementAliasList); + + void createAlignedTimeseries( + const std::string &deviceId, const std::vector &measurements, + const std::vector &dataTypes, + const std::vector &encodings, + const std::vector &compressors); + + bool checkTimeseriesExists(const std::string &path); + + std::unique_ptr executeQueryStatement(const std::string &sql); + + std::unique_ptr executeQueryStatement(const std::string &sql, + int64_t timeoutInMs); + + std::unique_ptr + executeQueryStatementMayRedirect(const std::string &sql, int64_t timeoutInMs); + + void executeNonQueryStatement(const std::string &sql); + + std::unique_ptr + executeRawDataQuery(const std::vector &paths, int64_t startTime, + int64_t endTime); + + std::unique_ptr + executeLastDataQuery(const std::vector &paths); + + std::unique_ptr + executeLastDataQuery(const std::vector &paths, int64_t lastTime); + + void createSchemaTemplate(const Template &templ); + + void setSchemaTemplate(const std::string &template_name, + const std::string &prefix_path); + + void unsetSchemaTemplate(const std::string &prefix_path, + const std::string &template_name); + + void addAlignedMeasurementsInTemplate( + const std::string &template_name, + const std::vector &measurements, + const std::vector &dataTypes, + const std::vector &encodings, + const std::vector &compressors); + + void addAlignedMeasurementsInTemplate( + const std::string &template_name, const std::string &measurement, + TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, + CompressionType::CompressionType compressor); + + void addUnalignedMeasurementsInTemplate( + const std::string &template_name, + const std::vector &measurements, + const std::vector &dataTypes, + const std::vector &encodings, + const std::vector &compressors); + + void addUnalignedMeasurementsInTemplate( + const std::string &template_name, const std::string &measurement, + TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, + CompressionType::CompressionType compressor); + + void deleteNodeInTemplate(const std::string &template_name, + const std::string &path); + + int countMeasurementsInTemplate(const std::string &template_name); + + bool isMeasurementInTemplate(const std::string &template_name, + const std::string &path); + + bool isPathExistInTemplate(const std::string &template_name, + const std::string &path); + + std::vector + showMeasurementsInTemplate(const std::string &template_name); + + std::vector + showMeasurementsInTemplate(const std::string &template_name, + const std::string &pattern); + + bool checkTemplateExists(const std::string &template_name); +}; + +#endif // IOTDB_SESSION_H diff --git a/iotdb-client/client-cpp/src/include/SessionBuilder.h b/iotdb-client/client-cpp/src/include/SessionBuilder.h new file mode 100644 index 0000000000000..1598699da451c --- /dev/null +++ b/iotdb-client/client-cpp/src/include/SessionBuilder.h @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef IOTDB_SESSION_BUILDER_H +#define IOTDB_SESSION_BUILDER_H + +#include "AbstractSessionBuilder.h" + +class SessionBuilder : public AbstractSessionBuilder { +public: + SessionBuilder *host(const std::string &host) { + AbstractSessionBuilder::host = host; + return this; + } + + SessionBuilder *rpcPort(int rpcPort) { + AbstractSessionBuilder::rpcPort = rpcPort; + return this; + } + + SessionBuilder *useSSL(bool useSSL) { + AbstractSessionBuilder::useSSL = useSSL; + return this; + } + + SessionBuilder *trustCertFilePath(const std::string &trustCertFilePath) { + AbstractSessionBuilder::trustCertFilePath = trustCertFilePath; + return this; + } + + SessionBuilder *username(const std::string &username) { + AbstractSessionBuilder::username = username; + return this; + } + + SessionBuilder *password(const std::string &password) { + AbstractSessionBuilder::password = password; + return this; + } + + SessionBuilder *zoneId(const std::string &zoneId) { + AbstractSessionBuilder::zoneId = zoneId; + return this; + } + + SessionBuilder *fetchSize(int fetchSize) { + AbstractSessionBuilder::fetchSize = fetchSize; + return this; + } + + SessionBuilder *nodeUrls(const std::vector &nodeUrls) { + AbstractSessionBuilder::nodeUrls = nodeUrls; + return this; + } + + SessionBuilder *enableAutoFetch(bool enableAutoFetch) { + AbstractSessionBuilder::enableAutoFetch = enableAutoFetch; + return this; + } + + SessionBuilder *enableRedirections(bool enableRedirections) { + AbstractSessionBuilder::enableRedirections = enableRedirections; + return this; + } + + SessionBuilder *enableRPCCompression(bool enableRPCCompression) { + AbstractSessionBuilder::enableRPCCompression = enableRPCCompression; + return this; + } + + std::shared_ptr build() { + if (!AbstractSessionBuilder::nodeUrls.empty() && + (AbstractSessionBuilder::host != DEFAULT_HOST || + AbstractSessionBuilder::rpcPort != DEFAULT_RPC_PORT)) { + throw IoTDBException("Session builder does not support setting node urls " + "and host/rpcPort at the same time."); + } + sqlDialect = "tree"; + auto newSession = std::make_shared(this); + newSession->open(false); + return newSession; + } +}; + +#endif // IOTDB_SESSION_BUILDER_H \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/include/SessionC.h b/iotdb-client/client-cpp/src/include/SessionC.h new file mode 100644 index 0000000000000..510fbb01a5719 --- /dev/null +++ b/iotdb-client/client-cpp/src/include/SessionC.h @@ -0,0 +1,399 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef IOTDB_SESSION_C_H +#define IOTDB_SESSION_C_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* ============================================================ + * Error Handling + * ============================================================ */ + +typedef int64_t TsStatus; + +#define TS_OK 0 +#define TS_ERR_CONNECTION -1 +#define TS_ERR_EXECUTION -2 +#define TS_ERR_INVALID_PARAM -3 +#define TS_ERR_NULL_PTR -4 +#define TS_ERR_UNKNOWN -99 + +/** + * Returns the error message from the last failed C API call on the current + * thread. The returned pointer is valid until the next C API call on the same + * thread. + */ +const char *ts_get_last_error(void); + +/* ============================================================ + * Opaque Handle Types + * ============================================================ */ + +typedef struct CSession_ CSession; +typedef struct CTablet_ CTablet; +typedef struct CSessionDataSet_ CSessionDataSet; +typedef struct CRowRecord_ CRowRecord; + +/* ============================================================ + * Enums (values match C++ TSDataType / TSEncoding / CompressionType) + * ============================================================ */ + +typedef enum { + TS_TYPE_BOOLEAN = 0, + TS_TYPE_INT32 = 1, + TS_TYPE_INT64 = 2, + TS_TYPE_FLOAT = 3, + TS_TYPE_DOUBLE = 4, + TS_TYPE_TEXT = 5, + TS_TYPE_TIMESTAMP = 8, + TS_TYPE_DATE = 9, + TS_TYPE_BLOB = 10, + TS_TYPE_STRING = 11, + /** Not a server type; used for invalid arguments / error paths in the C API. + */ + TS_TYPE_INVALID = 255 +} TSDataType_C; + +typedef enum { + TS_ENCODING_PLAIN = 0, + TS_ENCODING_DICTIONARY = 1, + TS_ENCODING_RLE = 2, + TS_ENCODING_DIFF = 3, + TS_ENCODING_TS_2DIFF = 4, + TS_ENCODING_BITMAP = 5, + TS_ENCODING_GORILLA_V1 = 6, + TS_ENCODING_REGULAR = 7, + TS_ENCODING_GORILLA = 8, + TS_ENCODING_ZIGZAG = 9, + TS_ENCODING_FREQ = 10 +} TSEncoding_C; + +typedef enum { + TS_COMPRESSION_UNCOMPRESSED = 0, + TS_COMPRESSION_SNAPPY = 1, + TS_COMPRESSION_GZIP = 2, + TS_COMPRESSION_LZO = 3, + TS_COMPRESSION_SDT = 4, + TS_COMPRESSION_PAA = 5, + TS_COMPRESSION_PLA = 6, + TS_COMPRESSION_LZ4 = 7, + TS_COMPRESSION_ZSTD = 8, + TS_COMPRESSION_LZMA2 = 9 +} TSCompressionType_C; + +/* ============================================================ + * Session Lifecycle — Tree Model + * ============================================================ */ + +CSession *ts_session_new(const char *host, int rpcPort, const char *username, + const char *password); + +CSession *ts_session_new_with_zone(const char *host, int rpcPort, + const char *username, const char *password, + const char *zoneId, int fetchSize); + +CSession *ts_session_new_multi_node(const char *const *nodeUrls, int urlCount, + const char *username, const char *password); + +void ts_session_destroy(CSession *session); + +TsStatus ts_session_open(CSession *session); + +TsStatus ts_session_open_with_compression(CSession *session, + bool enableRPCCompression); + +TsStatus ts_session_close(CSession *session); + +/* ============================================================ + * Timezone + * ============================================================ */ + +TsStatus ts_session_set_timezone(CSession *session, const char *zoneId); + +TsStatus ts_session_get_timezone(CSession *session, char *buf, int bufLen); + +/* ============================================================ + * Database Management (Tree Model) + * ============================================================ */ + +TsStatus ts_session_create_database(CSession *session, const char *database); + +TsStatus ts_session_delete_database(CSession *session, const char *database); + +TsStatus ts_session_delete_databases(CSession *session, + const char *const *databases, int count); + +/* ============================================================ + * Timeseries Management (Tree Model) + * ============================================================ */ + +TsStatus ts_session_create_timeseries(CSession *session, const char *path, + TSDataType_C dataType, + TSEncoding_C encoding, + TSCompressionType_C compressor); + +TsStatus ts_session_create_timeseries_ex( + CSession *session, const char *path, TSDataType_C dataType, + TSEncoding_C encoding, TSCompressionType_C compressor, int propsCount, + const char *const *propKeys, const char *const *propValues, int tagsCount, + const char *const *tagKeys, const char *const *tagValues, int attrsCount, + const char *const *attrKeys, const char *const *attrValues, + const char *measurementAlias); + +TsStatus ts_session_create_multi_timeseries( + CSession *session, int count, const char *const *paths, + const TSDataType_C *dataTypes, const TSEncoding_C *encodings, + const TSCompressionType_C *compressors); + +TsStatus ts_session_create_aligned_timeseries( + CSession *session, const char *deviceId, int count, + const char *const *measurements, const TSDataType_C *dataTypes, + const TSEncoding_C *encodings, const TSCompressionType_C *compressors); + +TsStatus ts_session_check_timeseries_exists(CSession *session, const char *path, + bool *exists); + +TsStatus ts_session_delete_timeseries(CSession *session, const char *path); + +TsStatus ts_session_delete_timeseries_batch(CSession *session, + const char *const *paths, + int count); + +/* ============================================================ + * Tablet Operations + * ============================================================ */ + +CTablet *ts_tablet_new(const char *deviceId, int columnCount, + const char *const *columnNames, + const TSDataType_C *dataTypes, int maxRowNumber); + +void ts_tablet_destroy(CTablet *tablet); + +void ts_tablet_reset(CTablet *tablet); + +int ts_tablet_get_row_count(CTablet *tablet); + +TsStatus ts_tablet_set_row_count(CTablet *tablet, int rowCount); + +TsStatus ts_tablet_add_timestamp(CTablet *tablet, int rowIndex, + int64_t timestamp); + +TsStatus ts_tablet_add_value_bool(CTablet *tablet, int colIndex, int rowIndex, + bool value); + +TsStatus ts_tablet_add_value_int32(CTablet *tablet, int colIndex, int rowIndex, + int32_t value); + +TsStatus ts_tablet_add_value_int64(CTablet *tablet, int colIndex, int rowIndex, + int64_t value); + +TsStatus ts_tablet_add_value_float(CTablet *tablet, int colIndex, int rowIndex, + float value); + +TsStatus ts_tablet_add_value_double(CTablet *tablet, int colIndex, int rowIndex, + double value); + +TsStatus ts_tablet_add_value_string(CTablet *tablet, int colIndex, int rowIndex, + const char *value); + +/* ============================================================ + * Data Insertion — Tree Model (Record) + * ============================================================ */ + +TsStatus ts_session_insert_record_str(CSession *session, const char *deviceId, + int64_t time, int count, + const char *const *measurements, + const char *const *values); + +TsStatus ts_session_insert_record(CSession *session, const char *deviceId, + int64_t time, int count, + const char *const *measurements, + const TSDataType_C *types, + const void *const *values); + +TsStatus ts_session_insert_aligned_record_str(CSession *session, + const char *deviceId, + int64_t time, int count, + const char *const *measurements, + const char *const *values); + +TsStatus ts_session_insert_aligned_record(CSession *session, + const char *deviceId, int64_t time, + int count, + const char *const *measurements, + const TSDataType_C *types, + const void *const *values); + +/* Batch insert — multiple devices (string values) */ +TsStatus ts_session_insert_records_str( + CSession *session, int deviceCount, const char *const *deviceIds, + const int64_t *times, const int *measurementCounts, + const char *const *const *measurementsList, + const char *const *const *valuesList); + +TsStatus ts_session_insert_aligned_records_str( + CSession *session, int deviceCount, const char *const *deviceIds, + const int64_t *times, const int *measurementCounts, + const char *const *const *measurementsList, + const char *const *const *valuesList); + +/* Batch insert — multiple devices (typed values) */ +TsStatus ts_session_insert_records(CSession *session, int deviceCount, + const char *const *deviceIds, + const int64_t *times, + const int *measurementCounts, + const char *const *const *measurementsList, + const TSDataType_C *const *typesList, + const void *const *const *valuesList); + +TsStatus ts_session_insert_aligned_records( + CSession *session, int deviceCount, const char *const *deviceIds, + const int64_t *times, const int *measurementCounts, + const char *const *const *measurementsList, + const TSDataType_C *const *typesList, const void *const *const *valuesList); + +/* Batch insert — single device (typed values) */ +TsStatus ts_session_insert_records_of_one_device( + CSession *session, const char *deviceId, int rowCount, const int64_t *times, + const int *measurementCounts, const char *const *const *measurementsList, + const TSDataType_C *const *typesList, const void *const *const *valuesList, + bool sorted); + +TsStatus ts_session_insert_aligned_records_of_one_device( + CSession *session, const char *deviceId, int rowCount, const int64_t *times, + const int *measurementCounts, const char *const *const *measurementsList, + const TSDataType_C *const *typesList, const void *const *const *valuesList, + bool sorted); + +/* ============================================================ + * Data Insertion — Tree Model (Tablet) + * ============================================================ */ + +TsStatus ts_session_insert_tablet(CSession *session, CTablet *tablet, + bool sorted); + +TsStatus ts_session_insert_aligned_tablet(CSession *session, CTablet *tablet, + bool sorted); + +TsStatus ts_session_insert_tablets(CSession *session, int tabletCount, + const char *const *deviceIds, + CTablet **tablets, bool sorted); + +TsStatus ts_session_insert_aligned_tablets(CSession *session, int tabletCount, + const char *const *deviceIds, + CTablet **tablets, bool sorted); + +/* ============================================================ + * Query — Tree Model + * ============================================================ */ + +TsStatus ts_session_execute_query(CSession *session, const char *sql, + CSessionDataSet **dataSet); + +TsStatus ts_session_execute_query_with_timeout(CSession *session, + const char *sql, + int64_t timeoutInMs, + CSessionDataSet **dataSet); + +TsStatus ts_session_execute_non_query(CSession *session, const char *sql); + +TsStatus ts_session_execute_raw_data_query(CSession *session, int pathCount, + const char *const *paths, + int64_t startTime, int64_t endTime, + CSessionDataSet **dataSet); + +TsStatus ts_session_execute_last_data_query(CSession *session, int pathCount, + const char *const *paths, + CSessionDataSet **dataSet); + +TsStatus ts_session_execute_last_data_query_with_time( + CSession *session, int pathCount, const char *const *paths, + int64_t lastTime, CSessionDataSet **dataSet); + +/* ============================================================ + * SessionDataSet & RowRecord — Result Iteration + * ============================================================ */ + +void ts_dataset_destroy(CSessionDataSet *dataSet); + +/** On failure (null handle, exception), see ts_get_last_error(). */ +bool ts_dataset_has_next(CSessionDataSet *dataSet); + +/** On failure (null handle, exception), see ts_get_last_error(); nullptr may + * also mean end of rows. */ +CRowRecord *ts_dataset_next(CSessionDataSet *dataSet); + +int ts_dataset_get_column_count(CSessionDataSet *dataSet); + +const char *ts_dataset_get_column_name(CSessionDataSet *dataSet, int index); + +const char *ts_dataset_get_column_type(CSessionDataSet *dataSet, int index); + +void ts_dataset_set_fetch_size(CSessionDataSet *dataSet, int fetchSize); + +void ts_row_record_destroy(CRowRecord *record); + +int64_t ts_row_record_get_timestamp(CRowRecord *record); + +int ts_row_record_get_field_count(CRowRecord *record); + +bool ts_row_record_is_null(CRowRecord *record, int index); + +bool ts_row_record_get_bool(CRowRecord *record, int index); + +int32_t ts_row_record_get_int32(CRowRecord *record, int index); + +int64_t ts_row_record_get_int64(CRowRecord *record, int index); + +float ts_row_record_get_float(CRowRecord *record, int index); + +double ts_row_record_get_double(CRowRecord *record, int index); + +const char *ts_row_record_get_string(CRowRecord *record, int index); + +/** Returns TS_TYPE_INVALID if record is null or index is out of range. */ +TSDataType_C ts_row_record_get_data_type(CRowRecord *record, int index); + +/* ============================================================ + * Data Deletion (Tree Model) + * ============================================================ */ + +TsStatus ts_session_delete_data(CSession *session, const char *path, + int64_t endTime); + +TsStatus ts_session_delete_data_batch(CSession *session, int pathCount, + const char *const *paths, + int64_t endTime); + +TsStatus ts_session_delete_data_range(CSession *session, int pathCount, + const char *const *paths, + int64_t startTime, int64_t endTime); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* IOTDB_SESSION_C_H */ diff --git a/iotdb-client/client-cpp/src/include/SessionConfig.h b/iotdb-client/client-cpp/src/include/SessionConfig.h new file mode 100644 index 0000000000000..2106a1b6c7b5a --- /dev/null +++ b/iotdb-client/client-cpp/src/include/SessionConfig.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_SESSION_CONFIG_H +#define IOTDB_SESSION_CONFIG_H + +namespace iotdb { +namespace session { + +// Defaults aligned with org.apache.iotdb.isession.SessionConfig (Java client). +constexpr int DEFAULT_FETCH_SIZE = 5000; +constexpr int DEFAULT_CONNECT_TIMEOUT_MS = 3 * 1000; +constexpr int DEFAULT_MAX_RETRIES = 3; +constexpr int DEFAULT_RETRY_DELAY_MS = 500; + +} // namespace session +} // namespace iotdb + +#endif // IOTDB_SESSION_CONFIG_H diff --git a/iotdb-client/client-cpp/src/include/SessionDataSet.h b/iotdb-client/client-cpp/src/include/SessionDataSet.h new file mode 100644 index 0000000000000..c7eac0f81cb1d --- /dev/null +++ b/iotdb-client/client-cpp/src/include/SessionDataSet.h @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef IOTDB_SESSION_DATA_SET_H +#define IOTDB_SESSION_DATA_SET_H + +#include +#include +#include +#include +#include + +#include "Column.h" +#include "Date.h" +#include "Optional.h" + +class RowRecord { +public: + int64_t timestamp; + std::vector fields; + + explicit RowRecord(int64_t timestamp); + RowRecord(int64_t timestamp, const std::vector &fields); + explicit RowRecord(const std::vector &fields); + RowRecord(); + + void addField(const Field &f); + std::string toString(); +}; + +class Session; + +class SessionDataSet { + struct Impl; + std::unique_ptr impl_; + SessionDataSet() = default; + friend class Session; + friend std::unique_ptr createSessionDataSet( + const std::string &sql, const std::vector &columnNameList, + const std::vector &columnTypeList, + const std::map &columnNameIndex, int64_t queryId, + int64_t statementId, + std::shared_ptr client, int64_t sessionId, + const std::vector &queryResult, bool ignoreTimestamp, + int64_t timeout, bool moreData, int32_t fetchSize, + const std::string &zoneId); + +private: + std::shared_ptr constructRowRecordFromValueArray(); + +public: + ~SessionDataSet(); + + bool hasNext(); + std::shared_ptr next(); + + int getFetchSize(); + void setFetchSize(int fetchSize); + + const std::vector &getColumnNames() const; + const std::vector &getColumnTypeList() const; + void closeOperationHandle(bool forceClose = false); + + class DataIterator { + std::shared_ptr impl_; + + public: + explicit DataIterator(std::shared_ptr impl); + + bool next(); + + bool isNull(const std::string &columnName); + bool isNullByIndex(int32_t columnIndex); + + Optional getBooleanByIndex(int32_t columnIndex); + Optional getBoolean(const std::string &columnName); + + Optional getDoubleByIndex(int32_t columnIndex); + Optional getDouble(const std::string &columnName); + + Optional getFloatByIndex(int32_t columnIndex); + Optional getFloat(const std::string &columnName); + + Optional getIntByIndex(int32_t columnIndex); + Optional getInt(const std::string &columnName); + + Optional getLongByIndex(int32_t columnIndex); + Optional getLong(const std::string &columnName); + + Optional getStringByIndex(int32_t columnIndex); + Optional getString(const std::string &columnName); + + Optional getTimestampByIndex(int32_t columnIndex); + Optional getTimestamp(const std::string &columnName); + + Optional getDateByIndex(int32_t columnIndex); + Optional getDate(const std::string &columnName); + + int32_t findColumn(const std::string &columnName); + const std::vector &getColumnNames() const; + const std::vector &getColumnTypeList() const; + }; + + // The returned iterator must not outlive this SessionDataSet. + DataIterator getIterator(); +}; + +#endif diff --git a/iotdb-client/client-cpp/src/include/SessionPool.h b/iotdb-client/client-cpp/src/include/SessionPool.h new file mode 100644 index 0000000000000..7c83fcaa87233 --- /dev/null +++ b/iotdb-client/client-cpp/src/include/SessionPool.h @@ -0,0 +1,352 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef IOTDB_SESSIONPOOL_H +#define IOTDB_SESSIONPOOL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Session.h" + +/* + * A thread-safe pool of opened Session objects. + * + * A Session is NOT safe to use from multiple threads concurrently. SessionPool + * solves this by lending each Session to exactly one borrower at a time and + * reclaiming it afterwards, so many application threads can share a bounded set + * of physical connections without external locking. + * + * Two usage styles are supported: + * + * 1. RAII lease (recommended for arbitrary calls): + * { + * PooledSession s = pool.getSession(); // blocks up to the timeout + * s->insertTablet(tablet); // call any Session method + * } // automatically returned + * here + * + * 2. Convenience wrappers / generic execute() (recommended for hot paths): + * pool.insertTablet(tablet); + * pool.execute([&](Session& s) { s.insertRecord(...); }); + * + * Both styles evict a Session from the pool (instead of recycling it) when the + * operation throws IoTDBConnectionException, so a dead connection is never + * handed to the next borrower; a fresh one is created lazily on demand. + * + * Lifetime: a PooledSession returns its Session to the owning SessionPool when + * destroyed, so every PooledSession must not outlive the pool it came from. + */ +class SessionPool; + +class PooledSession { +public: + PooledSession() noexcept + : pool_(nullptr), session_(nullptr), broken_(false) {} + + PooledSession(SessionPool *pool, std::shared_ptr session) + : pool_(pool), session_(std::move(session)), broken_(false) {} + + // Non-copyable: a leased Session is owned by exactly one borrower. + PooledSession(const PooledSession &) = delete; + PooledSession &operator=(const PooledSession &) = delete; + + PooledSession(PooledSession &&other) noexcept + : pool_(other.pool_), session_(std::move(other.session_)), + broken_(other.broken_) { + other.pool_ = nullptr; + other.session_ = nullptr; + other.broken_ = false; + } + + PooledSession &operator=(PooledSession &&other) noexcept { + if (this != &other) { + reset(); + pool_ = other.pool_; + session_ = std::move(other.session_); + broken_ = other.broken_; + other.pool_ = nullptr; + other.session_ = nullptr; + other.broken_ = false; + } + return *this; + } + + ~PooledSession() { reset(); } + + Session *operator->() const { return session_.get(); } + + Session &operator*() const { return *session_; } + + explicit operator bool() const { return static_cast(session_); } + + // Mark the underlying connection as unusable so it is discarded (not + // recycled) when this lease is returned. Call this if you caught a connection + // error. + void markBroken() { broken_ = true; } + + // Eagerly return the Session to the pool before scope exit. + void release() { reset(); } + +private: + void reset(); + + SessionPool *pool_; + std::shared_ptr session_; + bool broken_; +}; + +/* + * Couples a query result set with the pooled Session that produced it. + * + * A SessionDataSet lazily fetches further result blocks over its Session's + * connection, so that Session must stay exclusively leased until iteration is + * finished. This wrapper holds the lease for exactly that long and returns the + * Session to the pool when destroyed. + */ +class PooledSessionDataSet { +public: + PooledSessionDataSet(PooledSession session, + std::unique_ptr dataSet) + : session_(std::move(session)), dataSet_(std::move(dataSet)) {} + + PooledSessionDataSet(const PooledSessionDataSet &) = delete; + PooledSessionDataSet &operator=(const PooledSessionDataSet &) = delete; + PooledSessionDataSet(PooledSessionDataSet &&) noexcept = default; + PooledSessionDataSet &operator=(PooledSessionDataSet &&) noexcept = default; + + SessionDataSet *operator->() const { return dataSet_.get(); } + SessionDataSet &operator*() const { return *dataSet_; } + +private: + PooledSession session_; + std::unique_ptr dataSet_; +}; + +class SessionPool { +public: + static constexpr size_t DEFAULT_MAX_SIZE = 5; + static constexpr int64_t DEFAULT_WAIT_TIMEOUT_MS = 60 * 1000; + + // Single-host constructor. + SessionPool(std::string host, int rpcPort, std::string username, + std::string password, size_t maxSize = DEFAULT_MAX_SIZE); + + ~SessionPool(); + + // Non-copyable, non-movable: the pool owns mutex/condition state. + SessionPool(const SessionPool &) = delete; + SessionPool &operator=(const SessionPool &) = delete; + + // ---- configuration (apply before the first getSession()) ---- + SessionPool &setFetchSize(int fetchSize); + SessionPool &setZoneId(std::string zoneId); + SessionPool &setSqlDialect(std::string sqlDialect); + SessionPool &setDatabase(std::string database); + SessionPool &setEnableRedirection(bool enable); + SessionPool &setEnableAutoFetch(bool enable); + SessionPool &setEnableRPCCompression(bool enable); + SessionPool &setConnectTimeoutMs(int connectTimeoutMs); + SessionPool &setWaitToGetSessionTimeoutMs(int64_t timeoutMs); + + // Borrow a Session. Blocks until one is free or a new one can be created, + // up to timeoutMs (<= 0 means use the pool default). Throws IoTDBException on + // timeout or when the pool is closed. + PooledSession getSession(); + PooledSession getSession(int64_t timeoutMs); + + // Generic helper: borrow a Session, run func(Session&), return/evict it, and + // forward the result. Evicts the Session on IoTDBConnectionException. + template + auto execute(Func &&func) -> decltype(func(std::declval())); + + // ---- convenience wrappers for common operations (with eviction on failure) + // ---- + void insertTablet(Tablet &tablet, bool sorted = false); + void insertAlignedTablet(Tablet &tablet, bool sorted = false); + void insertTablets(std::unordered_map &tablets, + bool sorted = false); + void insertRecord(const std::string &deviceId, int64_t time, + const std::vector &measurements, + const std::vector &values); + void + insertRecords(const std::vector &deviceIds, + const std::vector ×, + const std::vector> &measurementsList, + const std::vector> &valuesList); + void executeNonQueryStatement(const std::string &sql); + // The returned wrapper keeps the underlying Session leased until it is + // destroyed, so it is safe to iterate the result set across multiple fetches. + PooledSessionDataSet executeQueryStatement(const std::string &sql); + PooledSessionDataSet executeQueryStatement(const std::string &sql, + int64_t timeoutInMs); + + // Close the pool: idle Sessions are closed immediately, in-use Sessions are + // closed when they are returned. Idempotent. + void close(); + + // ---- observability ---- + size_t getMaxSize() const { return maxSize_; } + // Number of Sessions currently borrowed. + size_t activeCount(); + +private: + friend class PooledSession; + + std::shared_ptr constructNewSession(); + std::shared_ptr acquire(int64_t timeoutMs); + void putBack(const std::shared_ptr &session, bool broken); + + // connection parameters + std::string host_; + int rpcPort_; + std::string username_; + std::string password_; + std::string zoneId_; + int fetchSize_ = 10000; + std::string sqlDialect_ = "tree"; + std::string database_; + bool enableRedirection_ = true; + bool enableAutoFetch_ = true; + bool enableRPCCompression_ = false; + int connectTimeoutMs_ = 3000; + + // pool sizing / waiting policy + size_t maxSize_; + int64_t waitTimeoutMs_ = DEFAULT_WAIT_TIMEOUT_MS; + + // pool state, guarded by mutex_ + std::mutex mutex_; + std::condition_variable cv_; + std::deque> idleQueue_; + size_t size_ = 0; // total live Sessions (idle + borrowed) + bool closed_ = false; +}; + +template +auto SessionPool::execute(Func &&func) + -> decltype(func(std::declval())) { + PooledSession lease = getSession(); + try { + return func(*lease); + } catch (const IoTDBConnectionException &) { + lease.markBroken(); + throw; + } +} + +/* + * Fluent builder for SessionPool, mirroring SessionBuilder / + * TableSessionBuilder. + * + * auto pool = SessionPoolBuilder() + * .host("127.0.0.1")->rpcPort(6667) + * ->username("root")->password("root") + * ->maxSize(10)->build(); + */ +class SessionPoolBuilder : public AbstractSessionBuilder { +public: + SessionPoolBuilder *host(const std::string &v) { + AbstractSessionBuilder::host = v; + return this; + } + SessionPoolBuilder *rpcPort(int v) { + AbstractSessionBuilder::rpcPort = v; + return this; + } + SessionPoolBuilder *username(const std::string &v) { + AbstractSessionBuilder::username = v; + return this; + } + SessionPoolBuilder *password(const std::string &v) { + AbstractSessionBuilder::password = v; + return this; + } + SessionPoolBuilder *zoneId(const std::string &v) { + AbstractSessionBuilder::zoneId = v; + return this; + } + SessionPoolBuilder *fetchSize(int v) { + AbstractSessionBuilder::fetchSize = v; + return this; + } + SessionPoolBuilder *database(const std::string &v) { + AbstractSessionBuilder::database = v; + return this; + } + SessionPoolBuilder *enableAutoFetch(bool v) { + AbstractSessionBuilder::enableAutoFetch = v; + return this; + } + SessionPoolBuilder *enableRedirections(bool v) { + AbstractSessionBuilder::enableRedirections = v; + return this; + } + SessionPoolBuilder *enableRPCCompression(bool v) { + AbstractSessionBuilder::enableRPCCompression = v; + return this; + } + SessionPoolBuilder *connectTimeoutMs(int v) { + connectTimeoutMs_ = v; + return this; + } + SessionPoolBuilder *maxSize(size_t v) { + maxSize_ = v; + return this; + } + SessionPoolBuilder *waitToGetSessionTimeoutMs(int64_t v) { + waitTimeoutMs_ = v; + return this; + } + SessionPoolBuilder *sqlDialect(const std::string &v) { + AbstractSessionBuilder::sqlDialect = v; + return this; + } + + std::shared_ptr build() { + auto pool = std::make_shared( + AbstractSessionBuilder::host, AbstractSessionBuilder::rpcPort, + AbstractSessionBuilder::username, AbstractSessionBuilder::password, + maxSize_); + pool->setFetchSize(AbstractSessionBuilder::fetchSize) + .setZoneId(AbstractSessionBuilder::zoneId) + .setSqlDialect(AbstractSessionBuilder::sqlDialect) + .setDatabase(AbstractSessionBuilder::database) + .setEnableRedirection(AbstractSessionBuilder::enableRedirections) + .setEnableAutoFetch(AbstractSessionBuilder::enableAutoFetch) + .setEnableRPCCompression(AbstractSessionBuilder::enableRPCCompression) + .setConnectTimeoutMs(connectTimeoutMs_) + .setWaitToGetSessionTimeoutMs(waitTimeoutMs_); + return pool; + } + +private: + size_t maxSize_ = SessionPool::DEFAULT_MAX_SIZE; + int64_t waitTimeoutMs_ = SessionPool::DEFAULT_WAIT_TIMEOUT_MS; + int connectTimeoutMs_ = 3000; +}; + +#endif // IOTDB_SESSIONPOOL_H diff --git a/iotdb-client/client-cpp/src/include/Status.h b/iotdb-client/client-cpp/src/include/Status.h new file mode 100644 index 0000000000000..147c20cffd21c --- /dev/null +++ b/iotdb-client/client-cpp/src/include/Status.h @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_STATUS_H +#define IOTDB_STATUS_H + +#include +#include + +struct Status { + int32_t code = 0; + std::string message; +}; + +#endif diff --git a/iotdb-client/client-cpp/src/include/TsBlock.h b/iotdb-client/client-cpp/src/include/TsBlock.h new file mode 100644 index 0000000000000..482aeacc3f3b2 --- /dev/null +++ b/iotdb-client/client-cpp/src/include/TsBlock.h @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef IOTDB_TS_BLOCK_H +#define IOTDB_TS_BLOCK_H + +#include "Column.h" +#include +#include + +class TsBlock { +public: + static std::shared_ptr + create(int32_t positionCount, std::shared_ptr timeColumn, + std::vector> valueColumns); + + static std::shared_ptr deserialize(const std::string &data); + + int32_t getPositionCount() const; + int64_t getStartTime() const; + int64_t getEndTime() const; + bool isEmpty() const; + int64_t getTimeByIndex(int32_t index) const; + int32_t getValueColumnCount() const; + const std::shared_ptr getTimeColumn() const; + const std::vector> &getValueColumns() const; + const std::shared_ptr getColumn(int32_t columnIndex) const; + +private: + TsBlock(int32_t positionCount, std::shared_ptr timeColumn, + std::vector> valueColumns); + + std::shared_ptr timeColumn_; + std::vector> valueColumns_; + int32_t positionCount_; +}; + +#endif diff --git a/iotdb-client/client-cpp/src/main/CMakeLists.txt b/iotdb-client/client-cpp/src/main/CMakeLists.txt deleted file mode 100644 index c6924060ba9b2..0000000000000 --- a/iotdb-client/client-cpp/src/main/CMakeLists.txt +++ /dev/null @@ -1,53 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -CMAKE_MINIMUM_REQUIRED(VERSION 3.7) -PROJECT(iotdb_session CXX) -SET(CMAKE_CXX_STANDARD 11) -SET(CMAKE_CXX_STANDARD_REQUIRED ON) -SET(CMAKE_POSITION_INDEPENDENT_CODE ON) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -g -O2 ") - -# Add Thrift include directory -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/../../thrift/include) - -# Add Boost include path for MacOS -INCLUDE_DIRECTORIES(/usr/local/include) - -# Add Boost library headers for MaxOS -FIND_PACKAGE(Boost REQUIRED) -IF (DEFINED BOOST_INCLUDEDIR) - INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIR}) -ENDIF() - -IF(MSVC) - SET(THRIFT_STATIC_LIB "${CMAKE_SOURCE_DIR}/../../thrift/lib/Release/thriftmd.lib") -ELSE() - SET(THRIFT_STATIC_LIB "${CMAKE_SOURCE_DIR}/../../thrift/lib/libthrift.a") -ENDIF() - -# Add the generated source files to the sources for the library. -AUX_SOURCE_DIRECTORY(./generated-sources-cpp SESSION_SRCS) -IF(MSVC) - ADD_LIBRARY(iotdb_session ${SESSION_SRCS}) -ELSE() - ADD_LIBRARY(iotdb_session SHARED ${SESSION_SRCS}) -ENDIF() - -# Link with Thrift static library -TARGET_LINK_LIBRARIES(iotdb_session ${THRIFT_STATIC_LIB}) diff --git a/iotdb-client/client-cpp/src/main/Column.cpp b/iotdb-client/client-cpp/src/main/Column.cpp deleted file mode 100644 index 4c533676d075a..0000000000000 --- a/iotdb-client/client-cpp/src/main/Column.cpp +++ /dev/null @@ -1,359 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "Column.h" -#include "ColumnDecoder.h" - -TimeColumn::TimeColumn(int32_t arrayOffset, int32_t positionCount, const std::vector& values) - : arrayOffset_(arrayOffset), positionCount_(positionCount), values_(values) { - if (arrayOffset < 0) throw IoTDBException("arrayOffset is negative"); - if (positionCount < 0) throw IoTDBException("positionCount is negative"); - if (static_cast(values.size()) - arrayOffset < positionCount) { - throw IoTDBException("values length is less than positionCount"); - } -} - -TSDataType::TSDataType TimeColumn::getDataType() const { return TSDataType::INT64; } -ColumnEncoding TimeColumn::getEncoding() const { return ColumnEncoding::Int64Array; } - -int64_t TimeColumn::getLong(int32_t position) const { - return values_[position + arrayOffset_]; -} - -bool TimeColumn::mayHaveNull() const { return false; } -bool TimeColumn::isNull(int32_t position) const { return false; } -std::vector TimeColumn::isNulls() const { return {}; } - -int32_t TimeColumn::getPositionCount() const { return positionCount_; } - -int64_t TimeColumn::getStartTime() const { return values_[arrayOffset_]; } -int64_t TimeColumn::getEndTime() const { return values_[positionCount_ + arrayOffset_ - 1]; } - -const std::vector& TimeColumn::getTimes() const { return values_; } -std::vector TimeColumn::getLongs() const { return getTimes(); } - -BinaryColumn::BinaryColumn(int32_t arrayOffset, int32_t positionCount, - const std::vector& valueIsNull, const std::vector>& values) - : arrayOffset_(arrayOffset), positionCount_(positionCount), - valueIsNull_(valueIsNull), values_(values) { - if (arrayOffset < 0) throw IoTDBException("arrayOffset is negative"); - if (positionCount < 0) throw IoTDBException("positionCount is negative"); - if (static_cast(values.size()) - arrayOffset < positionCount) { - throw IoTDBException("values length is less than positionCount"); - } - if (!valueIsNull.empty() && static_cast(valueIsNull.size()) - arrayOffset < positionCount) { - throw IoTDBException("isNull length is less than positionCount"); - } -} - -TSDataType::TSDataType BinaryColumn::getDataType() const { return TSDataType::TSDataType::TEXT; } -ColumnEncoding BinaryColumn::getEncoding() const { return ColumnEncoding::BinaryArray; } - -std::shared_ptr BinaryColumn::getBinary(int32_t position) const { - return values_[position + arrayOffset_]; -} - -std::vector> BinaryColumn::getBinaries() const { return values_; } - - -bool BinaryColumn::mayHaveNull() const { return !valueIsNull_.empty(); } - -bool BinaryColumn::isNull(int32_t position) const { - return !valueIsNull_.empty() && valueIsNull_[position + arrayOffset_]; -} - -std::vector BinaryColumn::isNulls() const { - if (!valueIsNull_.empty()) return valueIsNull_; - - std::vector result(positionCount_, false); - return result; -} - -int32_t BinaryColumn::getPositionCount() const { return positionCount_; } - -IntColumn::IntColumn(int32_t arrayOffset, int32_t positionCount, - const std::vector& valueIsNull, const std::vector& values) - : arrayOffset_(arrayOffset), positionCount_(positionCount), - valueNull_(valueIsNull), values_(values) { - if (arrayOffset < 0) throw IoTDBException("arrayOffset is negative"); - if (positionCount < 0) throw IoTDBException("positionCount is negative"); - if (static_cast(values.size()) - arrayOffset < positionCount) { - throw IoTDBException("values length is less than positionCount"); - } - if (!valueIsNull.empty() && static_cast(valueIsNull.size()) - arrayOffset < positionCount) { - throw IoTDBException("isNull length is less than positionCount"); - } -} - -TSDataType::TSDataType IntColumn::getDataType() const { return TSDataType::INT32; } -ColumnEncoding IntColumn::getEncoding() const { return ColumnEncoding::Int32Array; } - -int32_t IntColumn::getInt(int32_t position) const { - return values_[position + arrayOffset_]; -} - -std::vector IntColumn::getInts() const { return values_; } - -bool IntColumn::mayHaveNull() const { return !valueNull_.empty(); } - -bool IntColumn::isNull(int32_t position) const { - return !valueNull_.empty() && valueNull_[position + arrayOffset_]; -} - -std::vector IntColumn::isNulls() const { - if (!valueNull_.empty()) return valueNull_; - - std::vector result(positionCount_, false); - return result; -} - -int32_t IntColumn::getPositionCount() const { return positionCount_; } - -FloatColumn::FloatColumn(int32_t arrayOffset, int32_t positionCount, - const std::vector& valueIsNull, const std::vector& values) - : arrayOffset_(arrayOffset), positionCount_(positionCount), - valueIsNull_(valueIsNull), values_(values) { - if (arrayOffset < 0) throw IoTDBException("arrayOffset is negative"); - if (positionCount < 0) throw IoTDBException("positionCount is negative"); - if (static_cast(values.size()) - arrayOffset < positionCount) { - throw IoTDBException("values length is less than positionCount"); - } - if (!valueIsNull.empty() && static_cast(valueIsNull.size()) - arrayOffset < positionCount) { - throw IoTDBException("isNull length is less than positionCount"); - } -} - -TSDataType::TSDataType FloatColumn::getDataType() const { return TSDataType::TSDataType::FLOAT; } -ColumnEncoding FloatColumn::getEncoding() const { return ColumnEncoding::Int32Array; } - -float FloatColumn::getFloat(int32_t position) const { - return values_[position + arrayOffset_]; -} - -std::vector FloatColumn::getFloats() const { return values_; } - -bool FloatColumn::mayHaveNull() const { return !valueIsNull_.empty(); } - -bool FloatColumn::isNull(int32_t position) const { - return !valueIsNull_.empty() && valueIsNull_[position + arrayOffset_]; -} - -std::vector FloatColumn::isNulls() const { - if (!valueIsNull_.empty()) return valueIsNull_; - - std::vector result(positionCount_, false); - return result; -} - -int32_t FloatColumn::getPositionCount() const { return positionCount_; } - -LongColumn::LongColumn(int32_t arrayOffset, int32_t positionCount, - const std::vector& valueIsNull, const std::vector& values) - : arrayOffset_(arrayOffset), positionCount_(positionCount), - valueIsNull_(valueIsNull), values_(values) { - if (arrayOffset < 0) throw IoTDBException("arrayOffset is negative"); - if (positionCount < 0) throw IoTDBException("positionCount is negative"); - if (static_cast(values.size()) - arrayOffset < positionCount) { - throw IoTDBException("values length is less than positionCount"); - } - if (!valueIsNull.empty() && static_cast(valueIsNull.size()) - arrayOffset < positionCount) { - throw IoTDBException("isNull length is less than positionCount"); - } -} - -TSDataType::TSDataType LongColumn::getDataType() const { return TSDataType::TSDataType::INT64; } -ColumnEncoding LongColumn::getEncoding() const { return ColumnEncoding::Int64Array; } - -int64_t LongColumn::getLong(int32_t position) const { - return values_[position + arrayOffset_]; -} - -std::vector LongColumn::getLongs() const { return values_; } - -bool LongColumn::mayHaveNull() const { return !valueIsNull_.empty(); } - -bool LongColumn::isNull(int32_t position) const { - return !valueIsNull_.empty() && valueIsNull_[position + arrayOffset_]; -} - -std::vector LongColumn::isNulls() const { - if (!valueIsNull_.empty()) return valueIsNull_; - - std::vector result(positionCount_, false); - return result; -} - -int32_t LongColumn::getPositionCount() const { return positionCount_; } - -DoubleColumn::DoubleColumn(int32_t arrayOffset, int32_t positionCount, - const std::vector& valueIsNull, const std::vector& values) - : arrayOffset_(arrayOffset), positionCount_(positionCount), - valueIsNull_(valueIsNull), values_(values) { - if (arrayOffset < 0) throw IoTDBException("arrayOffset is negative"); - if (positionCount < 0) throw IoTDBException("positionCount is negative"); - if (static_cast(values.size()) - arrayOffset < positionCount) { - throw IoTDBException("values length is less than positionCount"); - } - if (!valueIsNull.empty() && static_cast(valueIsNull.size()) - arrayOffset < positionCount) { - throw IoTDBException("isNull length is less than positionCount"); - } -} - -TSDataType::TSDataType DoubleColumn::getDataType() const { return TSDataType::TSDataType::DOUBLE; } -ColumnEncoding DoubleColumn::getEncoding() const { return ColumnEncoding::Int64Array; } - -double DoubleColumn::getDouble(int32_t position) const { - return values_[position + arrayOffset_]; -} - -std::vector DoubleColumn::getDoubles() const { return values_; } - -bool DoubleColumn::mayHaveNull() const { return !valueIsNull_.empty(); } - -bool DoubleColumn::isNull(int32_t position) const { - return !valueIsNull_.empty() && valueIsNull_[position + arrayOffset_]; -} - -std::vector DoubleColumn::isNulls() const { - if (!valueIsNull_.empty()) return valueIsNull_; - - std::vector result(positionCount_, false); - return result; -} - -int32_t DoubleColumn::getPositionCount() const { return positionCount_; } - -BooleanColumn::BooleanColumn(int32_t arrayOffset, int32_t positionCount, - const std::vector& valueIsNull, const std::vector& values) - : arrayOffset_(arrayOffset), positionCount_(positionCount), - valueIsNull_(valueIsNull), values_(values) { - if (arrayOffset < 0) throw IoTDBException("arrayOffset is negative"); - if (positionCount < 0) throw IoTDBException("positionCount is negative"); - if (static_cast(values.size()) - arrayOffset < positionCount) { - throw IoTDBException("values length is less than positionCount"); - } - if (!valueIsNull.empty() && static_cast(valueIsNull.size()) - arrayOffset < positionCount) { - throw IoTDBException("isNull length is less than positionCount"); - } -} - -TSDataType::TSDataType BooleanColumn::getDataType() const { return TSDataType::TSDataType::BOOLEAN; } -ColumnEncoding BooleanColumn::getEncoding() const { return ColumnEncoding::ByteArray; } - -bool BooleanColumn::getBoolean(int32_t position) const { - return values_[position + arrayOffset_]; -} - -std::vector BooleanColumn::getBooleans() const { return values_; } - -bool BooleanColumn::mayHaveNull() const { return !valueIsNull_.empty(); } - -bool BooleanColumn::isNull(int32_t position) const { - return !valueIsNull_.empty() && valueIsNull_[position + arrayOffset_]; -} - -std::vector BooleanColumn::isNulls() const { - if (!valueIsNull_.empty()) return valueIsNull_; - - std::vector result(positionCount_, false); - return result; -} - -int32_t BooleanColumn::getPositionCount() const { return positionCount_; } - -RunLengthEncodedColumn::RunLengthEncodedColumn(std::shared_ptr value, int32_t positionCount) - : value_(value), positionCount_(positionCount) { - if (!value) throw IoTDBException("value is null"); - if (value->getPositionCount() != 1) { - throw IoTDBException("Expected value to contain a single position"); - } - if (positionCount < 0) throw IoTDBException("positionCount is negative"); -} - -std::shared_ptr RunLengthEncodedColumn::getValue() const { return value_; } - -TSDataType::TSDataType RunLengthEncodedColumn::getDataType() const { return value_->getDataType(); } -ColumnEncoding RunLengthEncodedColumn::getEncoding() const { return ColumnEncoding::Rle; } - -bool RunLengthEncodedColumn::getBoolean(int32_t position) const { - return value_->getBoolean(0); -} - -int32_t RunLengthEncodedColumn::getInt(int32_t position) const { - return value_->getInt(0); -} - -int64_t RunLengthEncodedColumn::getLong(int32_t position) const { - return value_->getLong(0); -} - -float RunLengthEncodedColumn::getFloat(int32_t position) const { - return value_->getFloat(0); -} - -double RunLengthEncodedColumn::getDouble(int32_t position) const { - return value_->getDouble(0); -} - -std::shared_ptr RunLengthEncodedColumn::getBinary(int32_t position) const { - return value_->getBinary(0); -} - -std::vector RunLengthEncodedColumn::getBooleans() const { - bool v = value_->getBoolean(0); - return std::vector(positionCount_, v); -} - -std::vector RunLengthEncodedColumn::getInts() const { - int32_t v = value_->getInt(0); - return std::vector(positionCount_, v); -} - -std::vector RunLengthEncodedColumn::getLongs() const { - int64_t v = value_->getLong(0); - return std::vector(positionCount_, v); -} - -std::vector RunLengthEncodedColumn::getFloats() const { - float v = value_->getFloat(0); - return std::vector(positionCount_, v); -} - -std::vector RunLengthEncodedColumn::getDoubles() const { - double v = value_->getDouble(0); - return std::vector(positionCount_, v); -} - -std::vector> RunLengthEncodedColumn::getBinaries() const { - auto v = value_->getBinary(0); - return std::vector>(positionCount_, v); -} - -bool RunLengthEncodedColumn::mayHaveNull() const { return value_->mayHaveNull(); } - -bool RunLengthEncodedColumn::isNull(int32_t position) const { - return value_->isNull(0); -} - -std::vector RunLengthEncodedColumn::isNulls() const { - bool v = value_->isNull(0); - return std::vector(positionCount_, v); -} - -int32_t RunLengthEncodedColumn::getPositionCount() const { return positionCount_; } diff --git a/iotdb-client/client-cpp/src/main/Column.h b/iotdb-client/client-cpp/src/main/Column.h deleted file mode 100644 index 04f611f41a874..0000000000000 --- a/iotdb-client/client-cpp/src/main/Column.h +++ /dev/null @@ -1,353 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#ifndef IOTDB_COLUMN_H -#define IOTDB_COLUMN_H - -#include -#include -#include -#include - -#include "Common.h" -#include "ColumnDecoder.h" - -enum class ColumnEncoding : uint8_t { - ByteArray, - Int32Array, - Int64Array, - BinaryArray, - Rle -}; - -class Binary { -public: - explicit Binary(std::vector data) : data_(std::move(data)) { - } - - const std::vector& getData() const { return data_; } - - std::string getStringValue() const { - return {data_.begin(), data_.end()}; - } - -private: - std::vector data_; -}; - - -const std::map> kEncodingToDecoder = { - {ColumnEncoding::Int32Array, std::make_shared()}, - {ColumnEncoding::Int64Array, std::make_shared()}, - {ColumnEncoding::ByteArray, std::make_shared()}, - {ColumnEncoding::BinaryArray, std::make_shared()}, - {ColumnEncoding::Rle, std::make_shared()} -}; - -const std::map kByteToEncoding = { - {0, ColumnEncoding::ByteArray}, - {1, ColumnEncoding::Int32Array}, - {2, ColumnEncoding::Int64Array}, - {3, ColumnEncoding::BinaryArray}, - {4, ColumnEncoding::Rle} -}; - -inline std::shared_ptr getColumnDecoder(ColumnEncoding encoding) { - auto it = kEncodingToDecoder.find(encoding); - if (it == kEncodingToDecoder.end()) { - throw IoTDBException("Unsupported column encoding"); - } - return it->second; -} - -inline ColumnEncoding getColumnEncodingByByte(uint8_t b) { - auto it = kByteToEncoding.find(b); - if (it == kByteToEncoding.end()) { - throw IoTDBException("Invalid encoding value: " + std::to_string(b)); - } - return it->second; -} - -class Column { -public: - virtual ~Column() = default; - - virtual TSDataType::TSDataType getDataType() const = 0; - virtual ColumnEncoding getEncoding() const = 0; - - virtual bool getBoolean(int32_t position) const { - throw IoTDBException("Unsupported operation: getBoolean"); - } - - virtual int32_t getInt(int32_t position) const { - throw IoTDBException("Unsupported operation: getInt"); - } - - virtual int64_t getLong(int32_t position) const { - throw IoTDBException("Unsupported operation: getLong"); - } - - virtual float getFloat(int32_t position) const { - throw IoTDBException("Unsupported operation: getFloat"); - } - - virtual double getDouble(int32_t position) const { - throw IoTDBException("Unsupported operation: getDouble"); - } - - virtual std::shared_ptr getBinary(int32_t position) const { - throw IoTDBException("Unsupported operation: getBinary"); - } - - virtual std::vector getBooleans() const { - throw IoTDBException("Unsupported operation: getBooleans"); - } - - virtual std::vector getInts() const { - throw IoTDBException("Unsupported operation: getInts"); - } - - virtual std::vector getLongs() const { - throw IoTDBException("Unsupported operation: getLongs"); - } - - virtual std::vector getFloats() const { - throw IoTDBException("Unsupported operation: getFloats"); - } - - virtual std::vector getDoubles() const { - throw IoTDBException("Unsupported operation: getDoubles"); - } - - virtual std::vector> getBinaries() const { - throw IoTDBException("Unsupported operation: getBinaries"); - } - - virtual bool mayHaveNull() const = 0; - virtual bool isNull(int32_t position) const = 0; - virtual std::vector isNulls() const = 0; - - virtual int32_t getPositionCount() const = 0; -}; - -class TimeColumn : public Column { -public: - TimeColumn(int32_t arrayOffset, int32_t positionCount, const std::vector& values); - - TSDataType::TSDataType getDataType() const override; - ColumnEncoding getEncoding() const override; - - int64_t getLong(int32_t position) const override; - - bool mayHaveNull() const override; - bool isNull(int32_t position) const override; - std::vector isNulls() const override; - - int32_t getPositionCount() const override; - - int64_t getStartTime() const; - int64_t getEndTime() const; - - const std::vector& getTimes() const; - std::vector getLongs() const override; - -private: - int32_t arrayOffset_; - int32_t positionCount_; - std::vector values_; -}; - -class BinaryColumn : public Column { -public: - BinaryColumn(int32_t arrayOffset, int32_t positionCount, - const std::vector& valueIsNull, const std::vector>& values); - - TSDataType::TSDataType getDataType() const override; - ColumnEncoding getEncoding() const override; - - std::shared_ptr getBinary(int32_t position) const override; - std::vector> getBinaries() const override; - - bool mayHaveNull() const override; - bool isNull(int32_t position) const override; - std::vector isNulls() const override; - - int32_t getPositionCount() const override; - -private: - int32_t arrayOffset_; - int32_t positionCount_; - std::vector valueIsNull_; - std::vector> values_; -}; - -class IntColumn : public Column { -public: - IntColumn(int32_t arrayOffset, int32_t positionCount, - const std::vector& valueIsNull, const std::vector& values); - - TSDataType::TSDataType getDataType() const override; - ColumnEncoding getEncoding() const override; - - int32_t getInt(int32_t position) const override; - std::vector getInts() const override; - - bool mayHaveNull() const override; - bool isNull(int32_t position) const override; - std::vector isNulls() const override; - - int32_t getPositionCount() const override; - -private: - int32_t arrayOffset_; - int32_t positionCount_; - std::vector valueNull_; - std::vector values_; -}; - -class FloatColumn : public Column { -public: - FloatColumn(int32_t arrayOffset, int32_t positionCount, - const std::vector& valueIsNull, const std::vector& values); - - TSDataType::TSDataType getDataType() const override; - ColumnEncoding getEncoding() const override; - - float getFloat(int32_t position) const override; - std::vector getFloats() const override; - - bool mayHaveNull() const override; - bool isNull(int32_t position) const override; - std::vector isNulls() const override; - - int32_t getPositionCount() const override; - -private: - int32_t arrayOffset_; - int32_t positionCount_; - std::vector valueIsNull_; - std::vector values_; -}; - -class LongColumn : public Column { -public: - LongColumn(int32_t arrayOffset, int32_t positionCount, - const std::vector& valueIsNull, const std::vector& values); - - TSDataType::TSDataType getDataType() const override; - ColumnEncoding getEncoding() const override; - - int64_t getLong(int32_t position) const override; - std::vector getLongs() const override; - - bool mayHaveNull() const override; - bool isNull(int32_t position) const override; - std::vector isNulls() const override; - - int32_t getPositionCount() const override; - -private: - int32_t arrayOffset_; - int32_t positionCount_; - std::vector valueIsNull_; - std::vector values_; -}; - -class DoubleColumn : public Column { -public: - DoubleColumn(int32_t arrayOffset, int32_t positionCount, - const std::vector& valueIsNull, const std::vector& values); - - TSDataType::TSDataType getDataType() const override; - ColumnEncoding getEncoding() const override; - - double getDouble(int32_t position) const override; - std::vector getDoubles() const override; - - bool mayHaveNull() const override; - bool isNull(int32_t position) const override; - std::vector isNulls() const override; - - int32_t getPositionCount() const override; - -private: - int32_t arrayOffset_; - int32_t positionCount_; - std::vector valueIsNull_; - std::vector values_; -}; - -class BooleanColumn : public Column { -public: - BooleanColumn(int32_t arrayOffset, int32_t positionCount, - const std::vector& valueIsNull, const std::vector& values); - - TSDataType::TSDataType getDataType() const override; - ColumnEncoding getEncoding() const override; - - bool getBoolean(int32_t position) const override; - std::vector getBooleans() const override; - - bool mayHaveNull() const override; - bool isNull(int32_t position) const override; - std::vector isNulls() const override; - - int32_t getPositionCount() const override; - -private: - int32_t arrayOffset_; - int32_t positionCount_; - std::vector valueIsNull_; - std::vector values_; -}; - -class RunLengthEncodedColumn : public Column { -public: - RunLengthEncodedColumn(std::shared_ptr value, int32_t positionCount); - - std::shared_ptr getValue() const; - - TSDataType::TSDataType getDataType() const override; - ColumnEncoding getEncoding() const override; - - bool getBoolean(int32_t position) const override; - int32_t getInt(int32_t position) const override; - int64_t getLong(int32_t position) const override; - float getFloat(int32_t position) const override; - double getDouble(int32_t position) const override; - std::shared_ptr getBinary(int32_t position) const override; - - std::vector getBooleans() const override; - std::vector getInts() const override; - std::vector getLongs() const override; - std::vector getFloats() const override; - std::vector getDoubles() const override; - std::vector> getBinaries() const override; - - bool mayHaveNull() const override; - bool isNull(int32_t position) const override; - std::vector isNulls() const override; - - int32_t getPositionCount() const override; - -private: - std::shared_ptr value_; - int32_t positionCount_; -}; - -#endif diff --git a/iotdb-client/client-cpp/src/main/ColumnDecoder.cpp b/iotdb-client/client-cpp/src/main/ColumnDecoder.cpp deleted file mode 100644 index e45cb49409a0d..0000000000000 --- a/iotdb-client/client-cpp/src/main/ColumnDecoder.cpp +++ /dev/null @@ -1,178 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "ColumnDecoder.h" - -#include "Column.h" - -std::vector deserializeNullIndicators(MyStringBuffer& buffer, int32_t positionCount) { - uint8_t mayHaveNullByte = buffer.getChar(); - - bool mayHaveNull = mayHaveNullByte != 0; - if (!mayHaveNull) { - return {}; - } - - return deserializeBooleanArray(buffer, positionCount); -} - -std::vector deserializeBooleanArray(MyStringBuffer& buffer, int32_t size) { - const int32_t packedSize = (size + 7) / 8; - std::vector packedBytes(packedSize); - for (int i = 0; i < packedSize; i++) { - packedBytes[i] = buffer.getChar(); - } - - std::vector output(size); - int currentByte = 0; - const int fullGroups = size & ~0b111; - - for (int pos = 0; pos < fullGroups; pos += 8) { - const uint8_t b = packedBytes[currentByte++]; - output[pos + 0] = (b & 0b10000000) != 0; - output[pos + 1] = (b & 0b01000000) != 0; - output[pos + 2] = (b & 0b00100000) != 0; - output[pos + 3] = (b & 0b00010000) != 0; - output[pos + 4] = (b & 0b00001000) != 0; - output[pos + 5] = (b & 0b00000100) != 0; - output[pos + 6] = (b & 0b00000010) != 0; - output[pos + 7] = (b & 0b00000001) != 0; - } - - if ((size & 0b111) > 0) { - const uint8_t b = packedBytes.back(); - uint8_t mask = 0b10000000; - - for (int pos = fullGroups; pos < size; pos++) { - output[pos] = (b & mask) != 0; - mask >>= 1; - } - } - - return output; -} - -std::unique_ptr BaseColumnDecoder::readColumn( - MyStringBuffer& buffer, TSDataType::TSDataType dataType, int32_t positionCount) { - return nullptr; -} - -std::unique_ptr Int32ArrayColumnDecoder::readColumn( - MyStringBuffer& buffer, TSDataType::TSDataType dataType, int32_t positionCount) { - auto nullIndicators = deserializeNullIndicators(buffer, positionCount); - - switch (dataType) { - case TSDataType::INT32: - case TSDataType::DATE: { - std::vector intValues(positionCount); - for (int32_t i = 0; i < positionCount; i++) { - if (!nullIndicators.empty() && nullIndicators[i]) continue; - intValues[i] = buffer.getInt(); - } - return std::unique_ptr(new IntColumn(0, positionCount, nullIndicators, intValues)); - } - case TSDataType::FLOAT: { - std::vector floatValues(positionCount); - for (int32_t i = 0; i < positionCount; i++) { - if (!nullIndicators.empty() && nullIndicators[i]) continue; - floatValues[i] = buffer.getFloat(); - } - return std::unique_ptr(new FloatColumn(0, positionCount, nullIndicators, floatValues)); - } - default: - throw IoTDBException("Invalid data type for Int32ArrayColumnDecoder"); - } -} - -std::unique_ptr Int64ArrayColumnDecoder::readColumn( - MyStringBuffer& buffer, TSDataType::TSDataType dataType, int32_t positionCount) { - auto nullIndicators = deserializeNullIndicators(buffer, positionCount); - - switch (dataType) { - case TSDataType::INT64: - case TSDataType::TIMESTAMP: { - std::vector values(positionCount); - for (int32_t i = 0; i < positionCount; i++) { - if (!nullIndicators.empty() && nullIndicators[i]) continue; - values[i] = buffer.getInt64(); - } - return std::unique_ptr(new LongColumn(0, positionCount, nullIndicators, values)); - } - case TSDataType::DOUBLE: { - std::vector values(positionCount); - for (int32_t i = 0; i < positionCount; i++) { - if (!nullIndicators.empty() && nullIndicators[i]) continue; - values[i] = buffer.getDouble(); - } - return std::unique_ptr(new DoubleColumn(0, positionCount, nullIndicators, values)); - } - default: - throw IoTDBException("Invalid data type for Int64ArrayColumnDecoder"); - } -} - -std::unique_ptr ByteArrayColumnDecoder::readColumn( - MyStringBuffer& buffer, TSDataType::TSDataType dataType, int32_t positionCount) { - if (dataType != TSDataType::BOOLEAN) { - throw IoTDBException("Invalid data type for ByteArrayColumnDecoder"); - } - - auto nullIndicators = deserializeNullIndicators(buffer, positionCount); - auto values = deserializeBooleanArray(buffer, positionCount); - return std::unique_ptr(new BooleanColumn(0, positionCount, nullIndicators, values)); -} - -std::unique_ptr BinaryArrayColumnDecoder::readColumn( - MyStringBuffer& buffer, TSDataType::TSDataType dataType, int32_t positionCount) { - if (dataType != TSDataType::TEXT) { - throw IoTDBException("Invalid data type for BinaryArrayColumnDecoder"); - } - - auto nullIndicators = deserializeNullIndicators(buffer, positionCount); - std::vector> values(positionCount); - - for (int32_t i = 0; i < positionCount; i++) { - if (!nullIndicators.empty() && nullIndicators[i]) continue; - - int32_t length = buffer.getInt(); - - std::vector value(length); - for (int32_t j = 0; j < length; j++) { - value[j] = buffer.getChar(); - } - - values[i] = std::make_shared(value); - } - - return std::unique_ptr(new BinaryColumn(0, positionCount, nullIndicators, values)); -} - -std::unique_ptr RunLengthColumnDecoder::readColumn( - MyStringBuffer& buffer, TSDataType::TSDataType dataType, int32_t positionCount) { - uint8_t encodingByte = buffer.getChar(); - - auto columnEncoding = static_cast(encodingByte); - auto decoder = getColumnDecoder(columnEncoding); - - auto column = decoder->readColumn(buffer, dataType, 1); - if (!column) { - throw IoTDBException("Failed to read inner column"); - } - return std::unique_ptr(new RunLengthEncodedColumn(move(column), positionCount)); -} diff --git a/iotdb-client/client-cpp/src/main/ColumnDecoder.h b/iotdb-client/client-cpp/src/main/ColumnDecoder.h deleted file mode 100644 index f5340d1e400ac..0000000000000 --- a/iotdb-client/client-cpp/src/main/ColumnDecoder.h +++ /dev/null @@ -1,75 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#ifndef IOTDB_COLUMN_DECODER_H -#define IOTDB_COLUMN_DECODER_H - -#include -#include - -#include "Common.h" - -class Column; - -class ColumnDecoder { -public: - virtual ~ColumnDecoder() = default; - virtual std::unique_ptr readColumn( - MyStringBuffer& buffer, TSDataType::TSDataType dataType, int32_t positionCount) = 0; -}; - -std::vector deserializeNullIndicators(MyStringBuffer& buffer, int32_t positionCount); -std::vector deserializeBooleanArray(MyStringBuffer& buffer, int32_t size); - -class BaseColumnDecoder : public ColumnDecoder { -public: - std::unique_ptr readColumn( - MyStringBuffer& buffer, TSDataType::TSDataType dataType, int32_t positionCount) override; -}; - -class Int32ArrayColumnDecoder : public BaseColumnDecoder { -public: - std::unique_ptr readColumn( - MyStringBuffer& buffer, TSDataType::TSDataType dataType, int32_t positionCount) override; -}; - -class Int64ArrayColumnDecoder : public BaseColumnDecoder { -public: - std::unique_ptr readColumn( - MyStringBuffer& buffer, TSDataType::TSDataType dataType, int32_t positionCount) override; -}; - -class ByteArrayColumnDecoder : public BaseColumnDecoder { -public: - std::unique_ptr readColumn( - MyStringBuffer& buffer, TSDataType::TSDataType dataType, int32_t positionCount) override; -}; - -class BinaryArrayColumnDecoder : public BaseColumnDecoder { -public: - std::unique_ptr readColumn( - MyStringBuffer& buffer, TSDataType::TSDataType dataType, int32_t positionCount) override; -}; - -class RunLengthColumnDecoder : public BaseColumnDecoder { -public: - std::unique_ptr readColumn( - MyStringBuffer& buffer, TSDataType::TSDataType dataType, int32_t positionCount) override; -}; - -#endif diff --git a/iotdb-client/client-cpp/src/main/Common.cpp b/iotdb-client/client-cpp/src/main/Common.cpp deleted file mode 100644 index 38e8a31d2ff37..0000000000000 --- a/iotdb-client/client-cpp/src/main/Common.cpp +++ /dev/null @@ -1,456 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "Common.h" -#include - -int32_t parseDateExpressionToInt(const boost::gregorian::date& date) { - if (date.is_not_a_date()) { - throw IoTDBException("Date expression is null or empty."); - } - - const int year = date.year(); - if (year < 1000 || year > 9999) { - throw DateTimeParseException( - "Year must be between 1000 and 9999.", - boost::gregorian::to_iso_extended_string(date), - 0 - ); - } - - const int64_t result = static_cast(year) * 10000 + - date.month() * 100 + - date.day(); - if (result > INT32_MAX || result < INT32_MIN) { - throw DateTimeParseException( - "Date value overflow. ", - boost::gregorian::to_iso_extended_string(date), - 0 - ); - } - return static_cast(result); -} - -boost::gregorian::date parseIntToDate(int32_t dateInt) { - if (dateInt == EMPTY_DATE_INT) { - return boost::gregorian::date(boost::date_time::not_a_date_time); - } - int year = dateInt / 10000; - int month = (dateInt % 10000) / 100; - int day = dateInt % 100; - return boost::gregorian::date(year, month, day); -} - -std::string getTimePrecision(int32_t timeFactor) { - if (timeFactor >= 1000000) return "us"; - if (timeFactor >= 1000) return "ms"; - return "s"; -} - -std::string formatDatetime(const std::string& format, const std::string& precision, - int64_t timestamp, const std::string& zoneId) { - // Simplified implementation - in real code you'd use proper timezone handling - std::time_t time = static_cast(timestamp); - std::tm* tm = std::localtime(&time); - char buffer[80]; - strftime(buffer, sizeof(buffer), format.c_str(), tm); - return std::string(buffer); -} - -std::tm convertToTimestamp(int64_t value, int32_t timeFactor) { - std::time_t time = static_cast(value / timeFactor); - return *std::localtime(&time); -} - -TSDataType::TSDataType getDataTypeByStr(const std::string& typeStr) { - if (typeStr == "BOOLEAN") return TSDataType::BOOLEAN; - if (typeStr == "INT32") return TSDataType::INT32; - if (typeStr == "INT64") return TSDataType::INT64; - if (typeStr == "FLOAT") return TSDataType::FLOAT; - if (typeStr == "DOUBLE") return TSDataType::DOUBLE; - if (typeStr == "TEXT") return TSDataType::TEXT; - if (typeStr == "TIMESTAMP") return TSDataType::TIMESTAMP; - if (typeStr == "DATE") return TSDataType::DATE; - if (typeStr == "BLOB") return TSDataType::BLOB; - if (typeStr == "STRING") return TSDataType::STRING; - return TSDataType::UNKNOWN; -} - -std::tm int32ToDate(int32_t value) { - // Convert days since epoch (1970-01-01) to tm struct - std::time_t time = static_cast(value) * 86400; // seconds per day - return *std::localtime(&time); -} - -void RpcUtils::verifySuccess(const TSStatus& status) { - if (status.code == TSStatusCode::MULTIPLE_ERROR) { - verifySuccess(status.subStatus); - return; - } - if (status.code != TSStatusCode::SUCCESS_STATUS - && status.code != TSStatusCode::REDIRECTION_RECOMMEND) { - throw ExecutionException(to_string(status.code) + ": " + status.message, status); - } -} - -void RpcUtils::verifySuccessWithRedirection(const TSStatus& status) { - verifySuccess(status); - if (status.__isset.redirectNode) { - throw RedirectException(to_string(status.code) + ": " + status.message, status.redirectNode); - } - if (status.__isset.subStatus) { - auto statusSubStatus = status.subStatus; - vector endPointList(statusSubStatus.size()); - int count = 0; - for (TSStatus subStatus : statusSubStatus) { - if (subStatus.__isset.redirectNode) { - endPointList[count++] = subStatus.redirectNode; - } - else { - TEndPoint endPoint; - endPointList[count++] = endPoint; - } - } - if (!endPointList.empty()) { - throw RedirectException(to_string(status.code) + ": " + status.message, endPointList); - } - } -} - -void RpcUtils::verifySuccessWithRedirectionForMultiDevices(const TSStatus& status, vector devices) { - verifySuccess(status); - - if (status.code == TSStatusCode::MULTIPLE_ERROR - || status.code == TSStatusCode::REDIRECTION_RECOMMEND) { - map deviceEndPointMap; - vector statusSubStatus; - for (int i = 0; i < statusSubStatus.size(); i++) { - TSStatus subStatus = statusSubStatus[i]; - if (subStatus.__isset.redirectNode) { - deviceEndPointMap.insert(make_pair(devices[i], subStatus.redirectNode)); - } - } - throw RedirectException(to_string(status.code) + ": " + status.message, deviceEndPointMap); - } - - if (status.__isset.redirectNode) { - throw RedirectException(to_string(status.code) + ": " + status.message, status.redirectNode); - } - if (status.__isset.subStatus) { - auto statusSubStatus = status.subStatus; - vector endPointList(statusSubStatus.size()); - int count = 0; - for (TSStatus subStatus : statusSubStatus) { - if (subStatus.__isset.redirectNode) { - endPointList[count++] = subStatus.redirectNode; - } - else { - TEndPoint endPoint; - endPointList[count++] = endPoint; - } - } - if (!endPointList.empty()) { - throw RedirectException(to_string(status.code) + ": " + status.message, endPointList); - } - } -} - -void RpcUtils::verifySuccess(const vector& statuses) { - for (const TSStatus& status : statuses) { - if (status.code != TSStatusCode::SUCCESS_STATUS) { - throw BatchExecutionException(status.message, statuses); - } - } -} - -TSStatus RpcUtils::getStatus(TSStatusCode::TSStatusCode tsStatusCode) { - TSStatus status; - status.__set_code(tsStatusCode); - return status; -} - -TSStatus RpcUtils::getStatus(int code, const string& message) { - TSStatus status; - status.__set_code(code); - status.__set_message(message); - return status; -} - -shared_ptr RpcUtils::getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode) { - TSStatus status = getStatus(tsStatusCode); - return getTSExecuteStatementResp(status); -} - -shared_ptr -RpcUtils::getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode, const string& message) { - TSStatus status = getStatus(tsStatusCode, message); - return getTSExecuteStatementResp(status); -} - -shared_ptr RpcUtils::getTSExecuteStatementResp(const TSStatus& status) { - shared_ptr resp(new TSExecuteStatementResp()); - TSStatus tsStatus(status); - resp->__set_status(status); - return resp; -} - -shared_ptr RpcUtils::getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode) { - TSStatus status = getStatus(tsStatusCode); - return getTSFetchResultsResp(status); -} - -shared_ptr -RpcUtils::getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode, const string& appendMessage) { - TSStatus status = getStatus(tsStatusCode, appendMessage); - return getTSFetchResultsResp(status); -} - -shared_ptr RpcUtils::getTSFetchResultsResp(const TSStatus& status) { - shared_ptr resp(new TSFetchResultsResp()); - TSStatus tsStatus(status); - resp->__set_status(tsStatus); - return resp; -} - -MyStringBuffer::MyStringBuffer() : pos(0) { - checkBigEndian(); -} - -MyStringBuffer::MyStringBuffer(const std::string& str) : str(str), pos(0) { - checkBigEndian(); -} - -void MyStringBuffer::reserve(size_t n) { - str.reserve(n); -} - -void MyStringBuffer::clear() { - str.clear(); - pos = 0; -} - -bool MyStringBuffer::hasRemaining() { - return pos < str.size(); -} - -int MyStringBuffer::getInt() { - return *(int*)getOrderedByte(4); -} - -boost::gregorian::date MyStringBuffer::getDate() { - return parseIntToDate(getInt()); -} - -int64_t MyStringBuffer::getInt64() { -#ifdef ARCH32 - const char *buf_addr = getOrderedByte(8); - if (reinterpret_cast(buf_addr) % 4 == 0) { - return *(int64_t *)buf_addr; - } else { - char tmp_buf[8]; - memcpy(tmp_buf, buf_addr, 8); - return *(int64_t*)tmp_buf; - } -#else - return *(int64_t*)getOrderedByte(8); -#endif -} - -float MyStringBuffer::getFloat() { - return *(float*)getOrderedByte(4); -} - -double MyStringBuffer::getDouble() { -#ifdef ARCH32 - const char *buf_addr = getOrderedByte(8); - if (reinterpret_cast(buf_addr) % 4 == 0) { - return *(double*)buf_addr; - } else { - char tmp_buf[8]; - memcpy(tmp_buf, buf_addr, 8); - return *(double*)tmp_buf; - } -#else - return *(double*)getOrderedByte(8); -#endif -} - -char MyStringBuffer::getChar() { - return str[pos++]; -} - -bool MyStringBuffer::getBool() { - return getChar() == 1; -} - -std::string MyStringBuffer::getString() { - size_t len = getInt(); - size_t tmpPos = pos; - pos += len; - return str.substr(tmpPos, len); -} - -void MyStringBuffer::putInt(int ins) { - putOrderedByte((char*)&ins, 4); -} - -void MyStringBuffer::putDate(boost::gregorian::date date) { - putInt(parseDateExpressionToInt(date)); -} - -void MyStringBuffer::putInt64(int64_t ins) { - putOrderedByte((char*)&ins, 8); -} - -void MyStringBuffer::putFloat(float ins) { - putOrderedByte((char*)&ins, 4); -} - -void MyStringBuffer::putDouble(double ins) { - putOrderedByte((char*)&ins, 8); -} - -void MyStringBuffer::putChar(char ins) { - str += ins; -} - -void MyStringBuffer::putBool(bool ins) { - char tmp = ins ? 1 : 0; - str += tmp; -} - -void MyStringBuffer::putString(const std::string& ins) { - putInt((int)(ins.size())); - str += ins; -} - -void MyStringBuffer::concat(const std::string& ins) { - str.append(ins); -} - -void MyStringBuffer::checkBigEndian() { - static int chk = 0x0201; //used to distinguish CPU's type (BigEndian or LittleEndian) - isBigEndian = (0x01 != *(char*)(&chk)); -} - -const char* MyStringBuffer::getOrderedByte(size_t len) { - const char* p = nullptr; - if (isBigEndian) { - p = str.c_str() + pos; - } - else { - const char* tmp = str.c_str(); - for (size_t i = pos; i < pos + len; i++) { - numericBuf[pos + len - 1 - i] = tmp[i]; - } - p = numericBuf; - } - pos += len; - return p; -} - -void MyStringBuffer::putOrderedByte(char* buf, int len) { - if (isBigEndian) { - str.assign(buf, len); - } - else { - for (int i = len - 1; i > -1; i--) { - str += buf[i]; - } - } -} - -BitMap::BitMap(size_t size) { - resize(size); -} - -void BitMap::resize(size_t size) { - this->size = size; - this->bits.resize((size >> 3) + 1); // equal to "size/8 + 1" - reset(); -} - -bool BitMap::mark(size_t position) { - if (position >= size) - return false; - - bits[position >> 3] |= (char)1 << (position % 8); - return true; -} - -bool BitMap::unmark(size_t position) { - if (position >= size) - return false; - - bits[position >> 3] &= ~((char)1 << (position % 8)); - return true; -} - -void BitMap::markAll() { - std::fill(bits.begin(), bits.end(), (char)0XFF); -} - -void BitMap::reset() { - std::fill(bits.begin(), bits.end(), (char)0); -} - -bool BitMap::isMarked(size_t position) const { - if (position >= size) - return false; - - return (bits[position >> 3] & ((char)1 << (position % 8))) != 0; -} - -bool BitMap::isAllUnmarked() const { - size_t j; - for (j = 0; j < size >> 3; j++) { - if (bits[j] != (char)0) { - return false; - } - } - for (j = 0; j < size % 8; j++) { - if ((bits[size >> 3] & ((char)1 << j)) != 0) { - return false; - } - } - return true; -} - -bool BitMap::isAllMarked() const { - size_t j; - for (j = 0; j < size >> 3; j++) { - if (bits[j] != (char)0XFF) { - return false; - } - } - for (j = 0; j < size % 8; j++) { - if ((bits[size >> 3] & ((char)1 << j)) == 0) { - return false; - } - } - return true; -} - -const std::vector& BitMap::getByteArray() const { - return this->bits; -} - -size_t BitMap::getSize() const { - return this->size; -} diff --git a/iotdb-client/client-cpp/src/main/Common.h b/iotdb-client/client-cpp/src/main/Common.h deleted file mode 100644 index a9f4552ecc5fd..0000000000000 --- a/iotdb-client/client-cpp/src/main/Common.h +++ /dev/null @@ -1,484 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#ifndef IOTDB_COMMON_H -#define IOTDB_COMMON_H - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "client_types.h" -#include "common_types.h" - -using namespace std; - -using ::apache::thrift::protocol::TBinaryProtocol; -using ::apache::thrift::protocol::TCompactProtocol; -using ::apache::thrift::transport::TSocket; -using ::apache::thrift::transport::TTransport; -using ::apache::thrift::transport::TTransportException; -using ::apache::thrift::transport::TBufferedTransport; -using ::apache::thrift::transport::TFramedTransport; -using ::apache::thrift::TException; - -using namespace std; - -constexpr int32_t EMPTY_DATE_INT = 10000101; - -int32_t parseDateExpressionToInt(const boost::gregorian::date& date); -boost::gregorian::date parseIntToDate(int32_t dateInt); - -std::string getTimePrecision(int32_t timeFactor); - -std::string formatDatetime(const std::string& format, const std::string& precision, - int64_t timestamp, const std::string& zoneId); - -std::tm convertToTimestamp(int64_t value, int32_t timeFactor); -std::tm int32ToDate(int32_t value); - -namespace Version { -enum Version { - V_0_12, V_0_13, V_1_0 -}; -} - -namespace CompressionType { -enum CompressionType { - UNCOMPRESSED = (char)0, - SNAPPY = (char)1, - GZIP = (char)2, - LZO = (char)3, - SDT = (char)4, - PAA = (char)5, - PLA = (char)6, - LZ4 = (char)7, - ZSTD = (char)8, - LZMA2 = (char)9, -}; -} - -namespace TSDataType { -enum TSDataType { - BOOLEAN = (char)0, - INT32 = (char)1, - INT64 = (char)2, - FLOAT = (char)3, - DOUBLE = (char)4, - TEXT = (char)5, - VECTOR = (char)6, - UNKNOWN = (char)7, - TIMESTAMP = (char)8, - DATE = (char)9, - BLOB = (char)10, - STRING = (char)11 -}; -} - -TSDataType::TSDataType getDataTypeByStr(const std::string& typeStr); - -namespace TSEncoding { -enum TSEncoding { - PLAIN = (char)0, - DICTIONARY = (char)1, - RLE = (char)2, - DIFF = (char)3, - TS_2DIFF = (char)4, - BITMAP = (char)5, - GORILLA_V1 = (char)6, - REGULAR = (char)7, - GORILLA = (char)8, - ZIGZAG = (char)9, - FREQ = (char)10, - INVALID_ENCODING = (char)255 -}; -} - -namespace TSStatusCode { -enum TSStatusCode { - SUCCESS_STATUS = 200, - - // System level - INCOMPATIBLE_VERSION = 201, - CONFIGURATION_ERROR = 202, - START_UP_ERROR = 203, - SHUT_DOWN_ERROR = 204, - - // General Error - UNSUPPORTED_OPERATION = 300, - EXECUTE_STATEMENT_ERROR = 301, - MULTIPLE_ERROR = 302, - ILLEGAL_PARAMETER = 303, - OVERLAP_WITH_EXISTING_TASK = 304, - INTERNAL_SERVER_ERROR = 305, - - // Client, - REDIRECTION_RECOMMEND = 400, - - // Schema Engine - DATABASE_NOT_EXIST = 500, - DATABASE_ALREADY_EXISTS = 501, - SERIES_OVERFLOW = 502, - TIMESERIES_ALREADY_EXIST = 503, - TIMESERIES_IN_BLACK_LIST = 504, - ALIAS_ALREADY_EXIST = 505, - PATH_ALREADY_EXIST = 506, - METADATA_ERROR = 507, - PATH_NOT_EXIST = 508, - ILLEGAL_PATH = 509, - CREATE_TEMPLATE_ERROR = 510, - DUPLICATED_TEMPLATE = 511, - UNDEFINED_TEMPLATE = 512, - TEMPLATE_NOT_SET = 513, - DIFFERENT_TEMPLATE = 514, - TEMPLATE_IS_IN_USE = 515, - TEMPLATE_INCOMPATIBLE = 516, - SEGMENT_NOT_FOUND = 517, - PAGE_OUT_OF_SPACE = 518, - RECORD_DUPLICATED = 519, - SEGMENT_OUT_OF_SPACE = 520, - PBTREE_FILE_NOT_EXISTS = 521, - OVERSIZE_RECORD = 522, - PBTREE_FILE_REDO_LOG_BROKEN = 523, - TEMPLATE_NOT_ACTIVATED = 524, - - // Storage Engine - SYSTEM_READ_ONLY = 600, - STORAGE_ENGINE_ERROR = 601, - STORAGE_ENGINE_NOT_READY = 602, - - // Query Engine - PLAN_FAILED_NETWORK_PARTITION = 721 -}; -} - -class Field { -public: - TSDataType::TSDataType dataType = TSDataType::UNKNOWN; - bool boolV{}; - int intV{}; - boost::gregorian::date dateV; - int64_t longV{}; - float floatV{}; - double doubleV{}; - std::string stringV; - - explicit Field(TSDataType::TSDataType a) { - dataType = a; - } - - Field() = default; -}; - -enum class ColumnCategory { - TAG, - FIELD, - ATTRIBUTE -}; - -class MyStringBuffer { -public: - MyStringBuffer(); - explicit MyStringBuffer(const std::string& str); - - void reserve(size_t n); - void clear(); - bool hasRemaining(); - int getInt(); - boost::gregorian::date getDate(); - int64_t getInt64(); - float getFloat(); - double getDouble(); - char getChar(); - bool getBool(); - std::string getString(); - - void putInt(int ins); - void putDate(boost::gregorian::date date); - void putInt64(int64_t ins); - void putFloat(float ins); - void putDouble(double ins); - void putChar(char ins); - void putBool(bool ins); - void putString(const std::string& ins); - void concat(const std::string& ins); - -public: - std::string str; - size_t pos; - -private: - void checkBigEndian(); - const char* getOrderedByte(size_t len); - void putOrderedByte(char* buf, int len); - -private: - bool isBigEndian{}; - char numericBuf[8]{}; //only be used by int, long, float, double etc. -}; - -class BitMap { -public: - explicit BitMap(size_t size = 0); - void resize(size_t size); - bool mark(size_t position); - bool unmark(size_t position); - void markAll(); - void reset(); - bool isMarked(size_t position) const; - bool isAllUnmarked() const; - bool isAllMarked() const; - const std::vector& getByteArray() const; - size_t getSize() const; - -private: - size_t size; - std::vector bits; -}; - -class IoTDBException : public std::exception { -public: - IoTDBException() = default; - - explicit IoTDBException(const std::string& m) : message(m) { - } - - explicit IoTDBException(const char* m) : message(m) { - } - - virtual const char* what() const noexcept override { - return message.c_str(); - } - -private: - std::string message; -}; - -class DateTimeParseException : public IoTDBException { -private: - std::string parsedString; - int errorIndex; - -public: - explicit DateTimeParseException(const std::string& message, - std::string parsedData, - int errorIndex) - : IoTDBException(message), - parsedString(std::move(parsedData)), - errorIndex(errorIndex) { - } - - explicit DateTimeParseException(const std::string& message, - std::string parsedData, - int errorIndex, - const std::exception& cause) - : IoTDBException(message + " [Caused by: " + cause.what() + "]"), - parsedString(std::move(parsedData)), - errorIndex(errorIndex) { - } - - const std::string& getParsedString() const noexcept { - return parsedString; - } - - int getErrorIndex() const noexcept { - return errorIndex; - } - - const char* what() const noexcept override { - static std::string fullMsg; - fullMsg = std::string(IoTDBException::what()) + - "\nParsed data: " + parsedString + - "\nError index: " + std::to_string(errorIndex); - return fullMsg.c_str(); - } -}; - -class IoTDBConnectionException : public IoTDBException { -public: - IoTDBConnectionException() { - } - - explicit IoTDBConnectionException(const char* m) : IoTDBException(m) { - } - - explicit IoTDBConnectionException(const std::string& m) : IoTDBException(m) { - } -}; - -class ExecutionException : public IoTDBException { -public: - ExecutionException() { - } - - explicit ExecutionException(const char* m) : IoTDBException(m) { - } - - explicit ExecutionException(const std::string& m) : IoTDBException(m) { - } - - explicit ExecutionException(const std::string& m, const TSStatus& tsStatus) : IoTDBException(m), status(tsStatus) { - } - - TSStatus status; -}; - -class BatchExecutionException : public IoTDBException { -public: - BatchExecutionException() { - } - - explicit BatchExecutionException(const char* m) : IoTDBException(m) { - } - - explicit BatchExecutionException(const std::string& m) : IoTDBException(m) { - } - - explicit BatchExecutionException(const std::vector& statusList) : statusList(statusList) { - } - - BatchExecutionException(const std::string& m, const std::vector& statusList) : IoTDBException(m), - statusList(statusList) { - } - - std::vector statusList; -}; - -class RedirectException : public IoTDBException { -public: - RedirectException() { - } - - explicit RedirectException(const char* m) : IoTDBException(m) { - } - - explicit RedirectException(const std::string& m) : IoTDBException(m) { - } - - RedirectException(const std::string& m, const TEndPoint& endPoint) : IoTDBException(m), endPoint(endPoint) { - } - - RedirectException(const std::string& m, const map& deviceEndPointMap) : IoTDBException(m), - deviceEndPointMap(deviceEndPointMap) { - } - - RedirectException(const std::string& m, const vector& endPointList) : IoTDBException(m), - endPointList(endPointList) { - } - - TEndPoint endPoint; - map deviceEndPointMap; - vector endPointList; -}; - -class UnSupportedDataTypeException : public IoTDBException { -public: - UnSupportedDataTypeException() { - } - - explicit UnSupportedDataTypeException(const char* m) : IoTDBException(m) { - } - - explicit UnSupportedDataTypeException(const std::string& m) : IoTDBException("UnSupported dataType: " + m) { - } -}; - -class SchemaNotFoundException : public IoTDBException { -public: - SchemaNotFoundException() { - } - - explicit SchemaNotFoundException(const char* m) : IoTDBException(m) { - } - - explicit SchemaNotFoundException(const std::string& m) : IoTDBException(m) { - } -}; - -class StatementExecutionException : public IoTDBException { -public: - StatementExecutionException() { - } - - explicit StatementExecutionException(const char* m) : IoTDBException(m) { - } - - explicit StatementExecutionException(const std::string& m) : IoTDBException(m) { - } -}; - -enum LogLevelType { - LEVEL_DEBUG = 0, - LEVEL_INFO, - LEVEL_WARN, - LEVEL_ERROR -}; - -extern LogLevelType LOG_LEVEL; - -#define log_debug(fmt,...) do {if(LOG_LEVEL <= LEVEL_DEBUG) {string s=string("[DEBUG] %s:%d (%s) - ") + fmt + "\n"; printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__);}} while(0) -#define log_info(fmt,...) do {if(LOG_LEVEL <= LEVEL_INFO) {string s=string("[INFO] %s:%d (%s) - ") + fmt + "\n"; printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__);}} while(0) -#define log_warn(fmt,...) do {if(LOG_LEVEL <= LEVEL_WARN) {string s=string("[WARN] %s:%d (%s) - ") + fmt + "\n"; printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__);}} while(0) -#define log_error(fmt,...) do {if(LOG_LEVEL <= LEVEL_ERROR) {string s=string("[ERROR] %s:%d (%s) - ") + fmt + "\n"; printf(s.c_str(), __FILE__, __LINE__, __FUNCTION__, ##__VA_ARGS__);}} while(0) - -class RpcUtils { -public: - std::shared_ptr SUCCESS_STATUS; - - RpcUtils() { - SUCCESS_STATUS = std::make_shared(); - SUCCESS_STATUS->__set_code(TSStatusCode::SUCCESS_STATUS); - } - - static void verifySuccess(const TSStatus& status); - - static void verifySuccessWithRedirection(const TSStatus& status); - - static void verifySuccessWithRedirectionForMultiDevices(const TSStatus& status, vector devices); - - static void verifySuccess(const std::vector& statuses); - - static TSStatus getStatus(TSStatusCode::TSStatusCode tsStatusCode); - - static TSStatus getStatus(int code, const std::string& message); - - static std::shared_ptr getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode); - - static std::shared_ptr - getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode, const std::string& message); - - static std::shared_ptr getTSExecuteStatementResp(const TSStatus& status); - - static std::shared_ptr getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode); - - static std::shared_ptr - getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode, const std::string& appendMessage); - - static std::shared_ptr getTSFetchResultsResp(const TSStatus& status); -}; - - -#endif diff --git a/iotdb-client/client-cpp/src/main/DeviceID.h b/iotdb-client/client-cpp/src/main/DeviceID.h deleted file mode 100644 index df2682cd5199e..0000000000000 --- a/iotdb-client/client-cpp/src/main/DeviceID.h +++ /dev/null @@ -1,161 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#ifndef IOTDB_DEVICEID_H -#define IOTDB_DEVICEID_H - -#include -#include -#include -#include -#include - -namespace storage { - -static const int DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME = 3; -static const std::string PATH_SEPARATOR = "."; - -class IDeviceID { -public: - virtual ~IDeviceID() = default; - virtual std::string get_table_name() { return ""; } - virtual int segment_num() { return 0; } - virtual const std::vector& get_segments() const { - return empty_segments_; - } - virtual std::string get_device_name() const { return ""; }; - virtual bool operator<(const IDeviceID& other) { return 0; } - virtual bool operator==(const IDeviceID& other) { return false; } - virtual bool operator!=(const IDeviceID& other) { return false; } - -protected: - IDeviceID() : empty_segments_() {} - -private: - const std::vector empty_segments_; -}; - -struct IDeviceIDComparator { - bool operator()(const std::shared_ptr& lhs, - const std::shared_ptr& rhs) const { - return *lhs < *rhs; - } -}; - -class StringArrayDeviceID : public IDeviceID { -public: - explicit StringArrayDeviceID(const std::vector& segments) - : segments_(formalize(segments)) {} - - explicit StringArrayDeviceID() : segments_() {} - - ~StringArrayDeviceID() override = default; - - std::string get_device_name() const override { - return segments_.empty() ? "" : std::accumulate(std::next(segments_.begin()), segments_.end(), - segments_.front(), - [](std::string a, const std::string& b) { - return std::move(a) + "." + b; - }); - }; - - std::string get_table_name() override { - return segments_.empty() ? "" : segments_[0]; - } - - int segment_num() override { return static_cast(segments_.size()); } - - const std::vector& get_segments() const override { - return segments_; - } - - bool operator<(const IDeviceID& other) override { - auto other_segments = other.get_segments(); - return std::lexicographical_compare(segments_.begin(), segments_.end(), - other_segments.begin(), - other_segments.end()); - } - - bool operator==(const IDeviceID& other) override { - auto other_segments = other.get_segments(); - return (segments_.size() == other_segments.size()) && - std::equal(segments_.begin(), segments_.end(), - other_segments.begin()); - } - - bool operator!=(const IDeviceID& other) override { - return !(*this == other); - } - -private: - std::vector segments_; - - std::vector formalize( - const std::vector& segments) { - auto it = - std::find_if(segments.rbegin(), segments.rend(), - [](const std::string& seg) { return !seg.empty(); }); - return std::vector(segments.begin(), it.base()); - } - - std::vector split_device_id_string( - const std::vector& splits) { - size_t segment_cnt = splits.size(); - std::vector final_segments; - - if (segment_cnt == 0) { - return final_segments; - } - - if (segment_cnt == 1) { - // "root" -> {"root"} - final_segments.push_back(splits[0]); - } else if (segment_cnt < static_cast( - DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME + 1)) { - // "root.a" -> {"root", "a"} - // "root.a.b" -> {"root.a", "b"} - std::string table_name = std::accumulate( - splits.begin(), splits.end() - 1, std::string(), - [](const std::string& a, const std::string& b) { - return a.empty() ? b : a + PATH_SEPARATOR + b; - }); - final_segments.push_back(table_name); - final_segments.push_back(splits.back()); - } else { - // "root.a.b.c" -> {"root.a.b", "c"} - // "root.a.b.c.d" -> {"root.a.b", "c", "d"} - std::string table_name = std::accumulate( - splits.begin(), - splits.begin() + DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME, - std::string(), [](const std::string& a, const std::string& b) { - return a.empty() ? b : a + PATH_SEPARATOR + b; - }); - - final_segments.emplace_back(std::move(table_name)); - final_segments.insert( - final_segments.end(), - splits.begin() + DEFAULT_SEGMENT_NUM_FOR_TABLE_NAME, - splits.end()); - } - - return final_segments; - } -}; -} - -#endif \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/main/IoTDBRpcDataSet.cpp b/iotdb-client/client-cpp/src/main/IoTDBRpcDataSet.cpp deleted file mode 100644 index 237bcb803586d..0000000000000 --- a/iotdb-client/client-cpp/src/main/IoTDBRpcDataSet.cpp +++ /dev/null @@ -1,558 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include -#include -#include - -#include "IoTDBRpcDataSet.h" -#include "Column.h" - -const int32_t IoTDBRpcDataSet::START_INDEX = 2; -const std::string IoTDBRpcDataSet::TIMESTAMP_STR = "Time"; -const std::string IoTDBRpcDataSet::DEFAULT_TIME_FORMAT = "default"; - -IoTDBRpcDataSet::IoTDBRpcDataSet(const std::string& sql, - const std::vector& columnNameList, - const std::vector& columnTypeList, - const std::map& columnNameIndex, - bool ignoreTimestamp, - bool moreData, - int64_t queryId, - int64_t statementId, - std::shared_ptr client, - int64_t sessionId, - const std::vector& queryResult, - int32_t fetchSize, - int64_t timeout, - const std::string& zoneId, - const std::string& timeFormat) - : sql_(sql), - isClosed_(false), - client_(client), - fetchSize_(fetchSize), - timeout_(timeout), - hasCachedRecord_(false), - lastReadWasNull_(false), - columnSize_(static_cast(columnNameList.size())), - sessionId_(sessionId), - queryId_(queryId), - statementId_(statementId), - time_(0), - ignoreTimestamp_(ignoreTimestamp), - moreData_(moreData), - queryResult_(queryResult), - curTsBlock_(nullptr), - queryResultSize_(static_cast(queryResult.size())), - queryResultIndex_(0), - tsBlockSize_(0), - tsBlockIndex_(-1), - timeZoneId_(zoneId), - timeFormat_(timeFormat) { - if (!ignoreTimestamp) { - columnNameList_.push_back(TIMESTAMP_STR); - columnTypeList_.emplace_back("INT64"); - columnOrdinalMap_[TIMESTAMP_STR] = 1; - } - - // Process column names and types - if (!columnNameIndex.empty()) { - // Deduplicate column types - std::set uniqueValues; - for (const auto& entry : columnNameIndex) { - uniqueValues.insert(entry.second); - } - int deduplicatedColumnSize = static_cast(uniqueValues.size()); - columnTypeDeduplicatedList_.resize(deduplicatedColumnSize); - for (size_t i = 0; i < columnNameList.size(); ++i) { - const std::string& name = columnNameList[i]; - columnNameList_.push_back(name); - columnTypeList_.push_back(columnTypeList[i]); - // Update ordinal map and deduplicated types - if (!columnOrdinalMap_.count(name)) { - int index = columnNameIndex.at(name); - if (std::none_of(columnOrdinalMap_.begin(), columnOrdinalMap_.end(), - [index](const std::pair& entry) { - return entry.second == (index + START_INDEX); - })) { - columnTypeDeduplicatedList_[index] = getDataTypeByStr(columnTypeList[i]); - } - columnOrdinalMap_[name] = index + START_INDEX; - } - } - } - else { - // Handle case without column name index - int32_t currentIndex = START_INDEX; - for (size_t i = 0; i < columnNameList.size(); ++i) { - std::string name = columnNameList[i]; - columnNameList_.push_back(name); - columnTypeList_.push_back(columnTypeList[i]); - if (!columnOrdinalMap_.count(name)) { - columnOrdinalMap_[name] = currentIndex++; - columnTypeDeduplicatedList_.push_back(getDataTypeByStr(columnTypeList[i])); - } - } - } - - columnSize_ = static_cast(columnNameList_.size()); -} - -IoTDBRpcDataSet::~IoTDBRpcDataSet() { - if (!isClosed_) { - close(); - } -} - -bool IoTDBRpcDataSet::next() { - if (hasCachedBlock()) { - lastReadWasNull_ = false; - constructOneRow(); - return true; - } - - if (hasCachedByteBuffer()) { - constructOneTsBlock(); - constructOneRow(); - return true; - } - - if (moreData_) { - bool hasResultSet = fetchResults(); - if (hasResultSet && hasCachedByteBuffer()) { - constructOneTsBlock(); - constructOneRow(); - return true; - } - } - - close(); - return false; -} - -void IoTDBRpcDataSet::close(bool forceClose) { - if ((!forceClose) && isClosed_) { - return; - } - TSCloseOperationReq closeReq; - closeReq.__set_sessionId(sessionId_); - closeReq.__set_statementId(statementId_); - closeReq.__set_queryId(queryId_); - TSStatus tsStatus; - try { - client_->closeOperation(tsStatus, closeReq); - RpcUtils::verifySuccess(tsStatus); - } - catch (const TTransportException& e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException& e) { - log_debug(e.what()); - throw; - } catch (exception& e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } - isClosed_ = true; - client_ = nullptr; -} - -bool IoTDBRpcDataSet::fetchResults() { - if (isClosed_) { - throw IoTDBException("This data set is already closed"); - } - - TSFetchResultsReq req; - req.__set_sessionId(sessionId_); - req.__set_statement(sql_); - req.__set_fetchSize(fetchSize_); - req.__set_queryId(queryId_); - req.__set_isAlign(true); - req.__set_timeout(timeout_); - TSFetchResultsResp resp; - client_->fetchResultsV2(resp, req); - RpcUtils::verifySuccess(resp.status); - moreData_ = resp.moreData; - if (!resp.hasResultSet) { - close(); - } - else { - queryResult_ = resp.queryResult; - queryResultIndex_ = 0; - if (!queryResult_.empty()) { - queryResultSize_ = queryResult_.size(); - } - else { - queryResultSize_ = 0; - } - tsBlockIndex_ = -1; - tsBlockSize_ = 0; - } - return resp.hasResultSet; -} - -void IoTDBRpcDataSet::constructOneRow() { - tsBlockIndex_++; - hasCachedRecord_ = true; - time_ = curTsBlock_->getTimeColumn()->getLong(tsBlockIndex_); -} - -void IoTDBRpcDataSet::constructOneTsBlock() { - lastReadWasNull_ = false; - const auto& curTsBlockBytes = queryResult_[queryResultIndex_]; - queryResultIndex_++; - curTsBlock_ = TsBlock::deserialize(curTsBlockBytes); - tsBlockIndex_ = -1; - tsBlockSize_ = curTsBlock_->getPositionCount(); -} - -bool IoTDBRpcDataSet::isNullByIndex(int32_t columnIndex) { - int index = columnOrdinalMap_[findColumnNameByIndex(columnIndex)] - START_INDEX; - // time column will never be null - if (index < 0) { - return false; - } - return isNull(index, tsBlockIndex_); -} - -bool IoTDBRpcDataSet::isNullByColumnName(const std::string& columnName) { - int index = columnOrdinalMap_[columnName] - START_INDEX; - // time column will never be null - if (index < 0) { - return false; - } - return isNull(index, tsBlockIndex_); -} - -bool IoTDBRpcDataSet::isNull(int32_t index, int32_t rowNum) { - return index >= 0 && curTsBlock_->getColumn(index)->isNull(rowNum); -} - -bool IoTDBRpcDataSet::getBooleanByIndex(int32_t columnIndex) { - return getBoolean(findColumnNameByIndex(columnIndex)); -} - -bool IoTDBRpcDataSet::getBoolean(const std::string& columnName) { - int index = columnOrdinalMap_[columnName] - START_INDEX; - return getBooleanByTsBlockColumnIndex(index); -} - -bool IoTDBRpcDataSet::getBooleanByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { - checkRecord(); - if (!isNull(tsBlockColumnIndex, tsBlockIndex_)) { - lastReadWasNull_ = false; - return curTsBlock_->getColumn(tsBlockColumnIndex)->getBoolean(tsBlockIndex_); - } - else { - lastReadWasNull_ = true; - return false; - } -} - -double IoTDBRpcDataSet::getDoubleByIndex(int32_t columnIndex) { - return getDouble(findColumnNameByIndex(columnIndex)); -} - -double IoTDBRpcDataSet::getDouble(const std::string& columnName) { - int index = columnOrdinalMap_[columnName] - START_INDEX; - return getDoubleByTsBlockColumnIndex(index); -} - -double IoTDBRpcDataSet::getDoubleByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { - checkRecord(); - if (!isNull(tsBlockColumnIndex, tsBlockIndex_)) { - lastReadWasNull_ = false; - return curTsBlock_->getColumn(tsBlockColumnIndex)->getDouble(tsBlockIndex_); - } - else { - lastReadWasNull_ = true; - return 0.0; - } -} - -float IoTDBRpcDataSet::getFloatByIndex(int32_t columnIndex) { - return getFloat(findColumnNameByIndex(columnIndex)); -} - -float IoTDBRpcDataSet::getFloat(const std::string& columnName) { - int index = columnOrdinalMap_[columnName] - START_INDEX; - return getFloatByTsBlockColumnIndex(index); -} - -float IoTDBRpcDataSet::getFloatByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { - checkRecord(); - if (!isNull(tsBlockColumnIndex, tsBlockIndex_)) { - lastReadWasNull_ = false; - return curTsBlock_->getColumn(tsBlockColumnIndex)->getFloat(tsBlockIndex_); - } - else { - lastReadWasNull_ = true; - return 0.0f; - } -} - -int32_t IoTDBRpcDataSet::getIntByIndex(int32_t columnIndex) { - return getInt(findColumnNameByIndex(columnIndex)); -} - -int32_t IoTDBRpcDataSet::getInt(const std::string& columnName) { - int index = columnOrdinalMap_[columnName] - START_INDEX; - return getIntByTsBlockColumnIndex(index); -} - -int32_t IoTDBRpcDataSet::getIntByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { - checkRecord(); - if (!isNull(tsBlockColumnIndex, tsBlockIndex_)) { - lastReadWasNull_ = false; - TSDataType::TSDataType dataType = curTsBlock_->getColumn(tsBlockColumnIndex)->getDataType(); - if (dataType == TSDataType::INT64) { - return static_cast(curTsBlock_->getColumn(tsBlockColumnIndex)->getLong(tsBlockIndex_)); - } - return curTsBlock_->getColumn(tsBlockColumnIndex)->getInt(tsBlockIndex_); - } - else { - lastReadWasNull_ = true; - return 0; - } -} - -int64_t IoTDBRpcDataSet::getLongByIndex(int32_t columnIndex) { - return getLong(findColumnNameByIndex(columnIndex)); -} - -int64_t IoTDBRpcDataSet::getLong(const std::string& columnName) { - int index = columnOrdinalMap_[columnName] - START_INDEX; - return getLongByTsBlockColumnIndex(index); -} - -int64_t IoTDBRpcDataSet::getLongByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { - checkRecord(); - if (tsBlockColumnIndex < 0) { - lastReadWasNull_ = false; - return curTsBlock_->getTimeByIndex(tsBlockIndex_); - } - if (!isNull(tsBlockColumnIndex, tsBlockIndex_)) { - lastReadWasNull_ = false; - TSDataType::TSDataType dataType = curTsBlock_->getColumn(tsBlockColumnIndex)->getDataType(); - if (dataType == TSDataType::INT32) { - return static_cast(curTsBlock_->getColumn(tsBlockColumnIndex)->getInt(tsBlockIndex_)); - } - return curTsBlock_->getColumn(tsBlockColumnIndex)->getLong(tsBlockIndex_); - } - else { - lastReadWasNull_ = true; - return 0; - } -} - -std::shared_ptr IoTDBRpcDataSet::getBinaryByIndex(int32_t columnIndex) { - return getBinary(findColumnNameByIndex(columnIndex)); -} - -std::shared_ptr IoTDBRpcDataSet::getBinary(const std::string& columnName) { - int index = columnOrdinalMap_[columnName] - START_INDEX; - return getBinaryByTsBlockColumnIndex(index); -} - -std::shared_ptr IoTDBRpcDataSet::getBinaryByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { - checkRecord(); - if (!isNull(tsBlockColumnIndex, tsBlockIndex_)) { - lastReadWasNull_ = false; - return curTsBlock_->getColumn(tsBlockColumnIndex)->getBinary(tsBlockIndex_); - } - else { - lastReadWasNull_ = true; - return nullptr; - } -} - -std::string IoTDBRpcDataSet::getStringByIndex(int32_t columnIndex) { - return getString(findColumnNameByIndex(columnIndex)); -} - -std::string IoTDBRpcDataSet::getString(const std::string& columnName) { - int index = columnOrdinalMap_[columnName] - START_INDEX; - return getStringByTsBlockColumnIndex(index); -} - -std::string IoTDBRpcDataSet::getStringByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { - checkRecord(); - if (tsBlockColumnIndex == -1) { - int64_t timestamp = curTsBlock_->getTimeByIndex(tsBlockIndex_); - return std::to_string(timestamp); - } - if (isNull(tsBlockColumnIndex, tsBlockIndex_)) { - lastReadWasNull_ = true; - return ""; - } - lastReadWasNull_ = false; - return getStringByTsBlockColumnIndexAndDataType(tsBlockColumnIndex, - getDataTypeByIndex(tsBlockColumnIndex)); -} - -std::string IoTDBRpcDataSet::getStringByTsBlockColumnIndexAndDataType(int32_t index, - TSDataType::TSDataType tsDataType) { - switch (tsDataType) { - case TSDataType::BOOLEAN: - return std::to_string(curTsBlock_->getColumn(index)->getBoolean(tsBlockIndex_)); - case TSDataType::INT32: - return std::to_string(curTsBlock_->getColumn(index)->getInt(tsBlockIndex_)); - case TSDataType::INT64: - return std::to_string(curTsBlock_->getColumn(index)->getLong(tsBlockIndex_)); - case TSDataType::TIMESTAMP: { - int64_t value = curTsBlock_->getColumn(index)->getLong(tsBlockIndex_); - return formatDatetime(timeFormat_, timePrecision_, value, timeZoneId_); - } - case TSDataType::FLOAT: - return std::to_string(curTsBlock_->getColumn(index)->getFloat(tsBlockIndex_)); - case TSDataType::DOUBLE: - return std::to_string(curTsBlock_->getColumn(index)->getDouble(tsBlockIndex_)); - case TSDataType::TEXT: - case TSDataType::STRING: - case TSDataType::BLOB: { - auto binary = curTsBlock_->getColumn(index)->getBinary(tsBlockIndex_); - return binary->getStringValue(); - } - case TSDataType::DATE: { - int32_t value = curTsBlock_->getColumn(index)->getInt(tsBlockIndex_); - auto date = parseIntToDate(value); - return boost::gregorian::to_iso_extended_string(date); - } - default: - return ""; - } -} - -int64_t IoTDBRpcDataSet::getTimestampByIndex(int32_t columnIndex) { - return getTimestamp(findColumnNameByIndex(columnIndex)); -} - -int64_t IoTDBRpcDataSet::getTimestamp(const std::string& columnName) { - return getLong(columnName); -} - -boost::gregorian::date IoTDBRpcDataSet::getDateByIndex(int32_t columnIndex) { - return getDate(findColumnNameByIndex(columnIndex)); -} - -boost::gregorian::date IoTDBRpcDataSet::getDate(const std::string& columnName) { - int32_t value = getInt(columnName); - return parseIntToDate(value); -} - -TSDataType::TSDataType IoTDBRpcDataSet::getDataTypeByIndex(int32_t columnIndex) { - return getDataType(findColumnNameByIndex(columnIndex)); -} - -TSDataType::TSDataType IoTDBRpcDataSet::getDataType(const std::string& columnName) { - if (columnName == TIMESTAMP_STR) { - return TSDataType::INT64; - } - int index = columnOrdinalMap_[columnName] - START_INDEX; - return index < 0 || index >= columnTypeDeduplicatedList_.size() - ? TSDataType::UNKNOWN - : columnTypeDeduplicatedList_[index]; -} - -int32_t IoTDBRpcDataSet::findColumn(const std::string& columnName) { - auto it = columnOrdinalMap_.find(columnName); - if (it != columnOrdinalMap_.end()) { - return it->second; - } - return -1; -} - -std::string IoTDBRpcDataSet::findColumnNameByIndex(int32_t columnIndex) { - if (columnIndex <= 0) { - throw IoTDBException("column index should start from 1"); - } - if (columnIndex > static_cast(columnNameList_.size())) { - throw IoTDBException( - "Column index " + std::to_string(columnIndex) + - " is out of range. Valid range is 0 to " + - std::to_string(columnNameList_.size() - 1) - ); - } - return columnNameList_[columnIndex - 1]; -} - -void IoTDBRpcDataSet::checkRecord() { - if (queryResultIndex_ > queryResultSize_ || - tsBlockIndex_ >= tsBlockSize_ || - queryResult_.empty() || - !curTsBlock_) { - throw IoTDBException("no record remains"); - } -} - -int32_t IoTDBRpcDataSet::getValueColumnStartIndex() const { - return ignoreTimestamp_ ? 0 : 1; -} - -int32_t IoTDBRpcDataSet::getColumnSize() const { - return static_cast(columnNameList_.size()); -} - -const std::vector& IoTDBRpcDataSet::getColumnTypeList() const { - return columnTypeList_; -} - -const std::vector& IoTDBRpcDataSet::getColumnNameList() const { - return columnNameList_; -} - -bool IoTDBRpcDataSet::isClosed() const { - return isClosed_; -} - -int32_t IoTDBRpcDataSet::getFetchSize() const { - return fetchSize_; -} - -void IoTDBRpcDataSet::setFetchSize(int32_t fetchSize) { - fetchSize_ = fetchSize; -} - -bool IoTDBRpcDataSet::hasCachedRecord() const { - return hasCachedRecord_; -} - -void IoTDBRpcDataSet::setHasCachedRecord(bool hasCachedRecord) { - hasCachedRecord_ = hasCachedRecord; -} - -bool IoTDBRpcDataSet::isLastReadWasNull() const { - return lastReadWasNull_; -} - -int64_t IoTDBRpcDataSet::getCurrentRowTime() const { - return time_; -} - -bool IoTDBRpcDataSet::isIgnoreTimestamp() const { - return ignoreTimestamp_; -} - -bool IoTDBRpcDataSet::hasCachedBlock() const { - return curTsBlock_ && tsBlockIndex_ < tsBlockSize_ - 1; -} - -bool IoTDBRpcDataSet::hasCachedByteBuffer() const { - return !queryResult_.empty() && queryResultIndex_ < queryResultSize_; -} diff --git a/iotdb-client/client-cpp/src/main/IoTDBRpcDataSet.h b/iotdb-client/client-cpp/src/main/IoTDBRpcDataSet.h deleted file mode 100644 index 8e3f0b3d628e4..0000000000000 --- a/iotdb-client/client-cpp/src/main/IoTDBRpcDataSet.h +++ /dev/null @@ -1,150 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#ifndef IOTDB_RPC_DATA_SET_H -#define IOTDB_RPC_DATA_SET_H - -#include -#include -#include -#include -#include -#include "IClientRPCService.h" -#include -#include "TsBlock.h" - -class IoTDBRpcDataSet { -public: - static const int32_t START_INDEX; - static const std::string TIMESTAMP_STR; - - static const std::string DEFAULT_TIME_FORMAT; - static const std::string TIME_PRECISION; - static const std::string MILLISECOND; - static const std::string MICROSECOND; - static const std::string NANOSECOND; - - IoTDBRpcDataSet(const std::string& sql, - const std::vector& columnNameList, - const std::vector& columnTypeList, - const std::map& columnNameIndex, - bool ignoreTimestamp, - bool moreData, - int64_t queryId, - int64_t statementId, - std::shared_ptr client, - int64_t sessionId, - const std::vector& queryResult, - int32_t fetchSize, - int64_t timeout, - const std::string& zoneId, - const std::string& timeFormat); - - ~IoTDBRpcDataSet(); - - bool next(); - void close(bool forceClose = false); - - bool hasCachedBlock() const; - bool hasCachedByteBuffer() const; - - bool isNull(int32_t index, int32_t rowNum); - bool isNullByIndex(int32_t columnIndex); - bool isNullByColumnName(const std::string& columnName); - bool getBooleanByIndex(int32_t columnIndex); - bool getBoolean(const std::string& columnName); - double getDoubleByIndex(int32_t columnIndex); - double getDouble(const std::string& columnName); - float getFloatByIndex(int32_t columnIndex); - float getFloat(const std::string& columnName); - int32_t getIntByIndex(int32_t columnIndex); - int32_t getInt(const std::string& columnName); - int64_t getLongByIndex(int32_t columnIndex); - int64_t getLong(const std::string& columnName); - std::shared_ptr getBinaryByIndex(int32_t columnIndex); - std::shared_ptr getBinary(const std::string& columnName); - std::string getStringByIndex(int32_t columnIndex); - std::string getString(const std::string& columnName); - int64_t getTimestampByIndex(int32_t columnIndex); - int64_t getTimestamp(const std::string& columnName); - boost::gregorian::date getDateByIndex(int32_t columnIndex); - boost::gregorian::date getDate(const std::string& columnName); - - TSDataType::TSDataType getDataTypeByIndex(int32_t columnIndex); - TSDataType::TSDataType getDataType(const std::string& columnName); - int32_t findColumn(const std::string& columnName); - std::string findColumnNameByIndex(int32_t columnIndex); - int32_t getValueColumnStartIndex() const; - int32_t getColumnSize() const; - const std::vector& getColumnTypeList() const; - const std::vector& getColumnNameList() const; - bool isClosed() const; - int32_t getFetchSize() const; - void setFetchSize(int32_t fetchSize); - bool hasCachedRecord() const; - void setHasCachedRecord(bool hasCachedRecord); - bool isLastReadWasNull() const; - int64_t getCurrentRowTime() const; - bool isIgnoreTimestamp() const; - -private: - bool fetchResults(); - void constructOneRow(); - void constructOneTsBlock(); - void checkRecord(); - bool getBooleanByTsBlockColumnIndex(int32_t tsBlockColumnIndex); - std::string getStringByTsBlockColumnIndexAndDataType(int32_t index, TSDataType::TSDataType tsDataType); - double getDoubleByTsBlockColumnIndex(int32_t tsBlockColumnIndex); - float getFloatByTsBlockColumnIndex(int32_t tsBlockColumnIndex); - int32_t getIntByTsBlockColumnIndex(int32_t tsBlockColumnIndex); - int64_t getLongByTsBlockColumnIndex(int32_t tsBlockColumnIndex); - std::shared_ptr getBinaryByTsBlockColumnIndex(int32_t tsBlockColumnIndex); - std::string getStringByTsBlockColumnIndex(int32_t tsBlockColumnIndex); - - std::string sql_; - bool isClosed_; - std::shared_ptr client_; - std::vector columnNameList_; - std::vector columnTypeList_; - std::map columnOrdinalMap_; - std::vector columnTypeDeduplicatedList_; - int32_t fetchSize_; - int64_t timeout_; - bool hasCachedRecord_; - bool lastReadWasNull_; - int32_t columnSize_; - int64_t sessionId_; - int64_t queryId_; - int64_t statementId_; - int64_t time_; - bool ignoreTimestamp_; - bool moreData_; - std::vector queryResult_; - std::shared_ptr curTsBlock_; - int32_t queryResultSize_; - int32_t queryResultIndex_; - int32_t tsBlockSize_; - int32_t tsBlockIndex_; - std::string timeZoneId_; - std::string timeFormat_; - int32_t timeFactor_; - std::string timePrecision_; -}; - -#endif // IOTDB_RPC_DATA_SET_H diff --git a/iotdb-client/client-cpp/src/main/NodesSupplier.cpp b/iotdb-client/client-cpp/src/main/NodesSupplier.cpp deleted file mode 100644 index 5f268f075a108..0000000000000 --- a/iotdb-client/client-cpp/src/main/NodesSupplier.cpp +++ /dev/null @@ -1,222 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#include "NodesSupplier.h" -#include "Session.h" -#include "SessionDataSet.h" -#include -#include -#include - -const std::string NodesSupplier::SHOW_DATA_NODES_COMMAND = "SHOW DATANODES"; -const std::string NodesSupplier::STATUS_COLUMN_NAME = "Status"; -const std::string NodesSupplier::IP_COLUMN_NAME = "RpcAddress"; -const std::string NodesSupplier::PORT_COLUMN_NAME = "RpcPort"; -const std::string NodesSupplier::REMOVING_STATUS = "Removing"; - -const int64_t NodesSupplier::TIMEOUT_IN_MS = 60000; -const int NodesSupplier::FETCH_SIZE = 10000; -const int NodesSupplier::THRIFT_DEFAULT_BUFFER_SIZE = 4096; -const int NodesSupplier::THRIFT_MAX_FRAME_SIZE = 1048576; -const int NodesSupplier::CONNECTION_TIMEOUT_IN_MS = 1000; - -TEndPoint RoundRobinPolicy::select(const std::vector& nodes) { - static std::atomic_uint index{0}; - - if (nodes.empty()) { - throw IoTDBException("No available nodes"); - } - - return nodes[index++ % nodes.size()]; -} - -StaticNodesSupplier::StaticNodesSupplier(const std::vector& nodes, - NodeSelectionPolicy policy) - : availableNodes_(nodes), policy_(std::move(policy)) {} - -boost::optional StaticNodesSupplier::getQueryEndPoint() { - try { - if (availableNodes_.empty()) { - return boost::none; - } - return policy_(availableNodes_); - } catch (const IoTDBException& e) { - return boost::none; - } -} - -std::vector StaticNodesSupplier::getEndPointList() { - return availableNodes_; -} - -StaticNodesSupplier::~StaticNodesSupplier() = default; - -std::shared_ptr NodesSupplier::create( - std::vector endpoints, - std::string userName, std::string password, std::string zoneId, - int32_t thriftDefaultBufferSize, int32_t thriftMaxFrameSize, - int32_t connectionTimeoutInMs, bool useSSL, bool enableRPCCompression, - std::string version, std::chrono::milliseconds refreshInterval, - NodeSelectionPolicy policy) { - if (endpoints.empty()) { - return nullptr; - } - auto supplier = std::make_shared( - userName, password, zoneId, thriftDefaultBufferSize, - thriftMaxFrameSize, connectionTimeoutInMs, useSSL, - enableRPCCompression, version, std::move(endpoints), std::move(policy) - ); - supplier->startBackgroundRefresh(refreshInterval); - return supplier; -} - -NodesSupplier::NodesSupplier( - std::string userName, std::string password, const std::string& zoneId, - int32_t thriftDefaultBufferSize, int32_t thriftMaxFrameSize, - int32_t connectionTimeoutInMs, bool useSSL, bool enableRPCCompression, - std::string version, std::vector endpoints, NodeSelectionPolicy policy) : userName_(std::move(userName)), password_(std::move(password)), zoneId_(zoneId), - thriftDefaultBufferSize_(thriftDefaultBufferSize), thriftMaxFrameSize_(thriftMaxFrameSize), - connectionTimeoutInMs_(connectionTimeoutInMs), useSSL_(useSSL), enableRPCCompression_(enableRPCCompression), version(version), endpoints_(std::move(endpoints)), - selectionPolicy_(std::move(policy)) { - deduplicateEndpoints(); -} - -std::vector NodesSupplier::getEndPointList() { - std::lock_guard lock(mutex_); - return endpoints_; -} - -TEndPoint NodesSupplier::selectQueryEndpoint() { - std::lock_guard lock(mutex_); - try { - return selectionPolicy_(endpoints_); - } catch (const std::exception& e) { - log_error("NodesSupplier::selectQueryEndpoint exception: %s", e.what()); - throw IoTDBException("NodesSupplier::selectQueryEndpoint exception, " + std::string(e.what())); - } -} - -boost::optional NodesSupplier::getQueryEndPoint() { - try { - return selectQueryEndpoint(); - } catch (const IoTDBException& e) { - return boost::none; - } -} - -NodesSupplier::~NodesSupplier() { - stopBackgroundRefresh(); - client_->close(); -} - -void NodesSupplier::deduplicateEndpoints() { - std::vector uniqueEndpoints; - uniqueEndpoints.reserve(endpoints_.size()); - for (const auto& endpoint : endpoints_) { - if (std::find(uniqueEndpoints.begin(), uniqueEndpoints.end(), endpoint) == uniqueEndpoints.end()) { - uniqueEndpoints.push_back(endpoint); - } - } - endpoints_ = std::move(uniqueEndpoints); -} - -void NodesSupplier::startBackgroundRefresh(std::chrono::milliseconds interval) { - isRunning_ = true; - refreshThread_ = std::thread([this, interval] { - while (isRunning_) { - refreshEndpointList(); - std::unique_lock cvLock(this->mutex_); - refreshCondition_.wait_for(cvLock, interval, [this]() { - return !isRunning_.load(); - }); - } - }); -} - -std::vector NodesSupplier::fetchLatestEndpoints() { - try { - if (client_ == nullptr) { - client_ = std::make_shared(selectionPolicy_(endpoints_)); - client_->init(userName_, password_, enableRPCCompression_, zoneId_, version); - } - - auto sessionDataSet = client_->executeQueryStatement(SHOW_DATA_NODES_COMMAND); - - uint32_t columnAddrIdx = -1, columnPortIdx = -1, columnStatusIdx = -1; - auto columnNames = sessionDataSet->getColumnNames(); - for (uint32_t i = 0; i < columnNames.size(); i++) { - if (columnNames[i] == IP_COLUMN_NAME) { - columnAddrIdx = i; - } else if (columnNames[i] == PORT_COLUMN_NAME) { - columnPortIdx = i; - } else if (columnNames[i] == STATUS_COLUMN_NAME) { - columnStatusIdx = i; - } - } - - if (columnAddrIdx == -1 || columnPortIdx == -1 || columnStatusIdx == -1) { - throw IoTDBException("Required columns not found in query result."); - } - - std::vector ret; - while (sessionDataSet->hasNext()) { - auto record = sessionDataSet->next(); - std::string ip = record->fields.at(columnAddrIdx).stringV; - int32_t port = record->fields.at(columnPortIdx).intV; - std::string status = record->fields.at(columnStatusIdx).stringV; - - if (ip == "0.0.0.0" || status == REMOVING_STATUS) { - log_warn("Skipping invalid node: " + ip + ":" + to_string(port)); - continue; - } - TEndPoint endpoint; - endpoint.ip = ip; - endpoint.port = port; - ret.emplace_back(endpoint); - } - - return ret; - } catch (const IoTDBException& e) { - client_.reset(); - throw IoTDBException(std::string("NodesSupplier::fetchLatestEndpoints failed: ") + e.what()); - } -} - -void NodesSupplier::refreshEndpointList() { - try { - auto newEndpoints = fetchLatestEndpoints(); - if (newEndpoints.empty()) { - return; - } - - std::lock_guard lock(mutex_); - endpoints_.swap(newEndpoints); - deduplicateEndpoints(); - } catch (const IoTDBException& e) { - log_error(std::string("NodesSupplier::refreshEndpointList failed: ") + e.what()); - } -} - -void NodesSupplier::stopBackgroundRefresh() noexcept { - if (isRunning_.exchange(false)) { - refreshCondition_.notify_all(); - if (refreshThread_.joinable()) { - refreshThread_.join(); - } - } -} \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/main/NodesSupplier.h b/iotdb-client/client-cpp/src/main/NodesSupplier.h deleted file mode 100644 index a3cda24deac69..0000000000000 --- a/iotdb-client/client-cpp/src/main/NodesSupplier.h +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#ifndef IOTDB_NODES_SUPPLIER_H -#define IOTDB_NODES_SUPPLIER_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ThriftConnection.h" - -class TEndPoint; - -class RoundRobinPolicy { -public: - static TEndPoint select(const std::vector& nodes); -}; - -class INodesSupplier { -public: - virtual ~INodesSupplier() = default; - virtual boost::optional getQueryEndPoint() = 0; - virtual std::vector getEndPointList() = 0; - using NodeSelectionPolicy = std::function&)>; -}; - -class StaticNodesSupplier : public INodesSupplier { -public: - explicit StaticNodesSupplier(const std::vector& nodes, - NodeSelectionPolicy policy = RoundRobinPolicy::select); - - boost::optional getQueryEndPoint() override; - - std::vector getEndPointList() override; - - ~StaticNodesSupplier() override; - -private: - const std::vector availableNodes_; - NodeSelectionPolicy policy_; -}; - -class NodesSupplier : public INodesSupplier { -public: - static const std::string SHOW_DATA_NODES_COMMAND; - static const std::string STATUS_COLUMN_NAME; - static const std::string IP_COLUMN_NAME; - static const std::string PORT_COLUMN_NAME; - static const std::string REMOVING_STATUS; - - static const int64_t TIMEOUT_IN_MS; - static const int FETCH_SIZE; - static const int THRIFT_DEFAULT_BUFFER_SIZE; - static const int THRIFT_MAX_FRAME_SIZE; - static const int CONNECTION_TIMEOUT_IN_MS; - - static std::shared_ptr create( - std::vector endpoints, - std::string userName, std::string password, std::string zoneId = "", - int32_t thriftDefaultBufferSize = ThriftConnection::THRIFT_DEFAULT_BUFFER_SIZE, - int32_t thriftMaxFrameSize = ThriftConnection::THRIFT_MAX_FRAME_SIZE, - int32_t connectionTimeoutInMs = ThriftConnection::CONNECTION_TIMEOUT_IN_MS, - bool useSSL = false, bool enableRPCCompression = false, - std::string version = "V_1_0", - std::chrono::milliseconds refreshInterval = std::chrono::milliseconds(TIMEOUT_IN_MS), - NodeSelectionPolicy policy = RoundRobinPolicy::select - ); - - NodesSupplier( - std::string userName, std::string password, const std::string& zoneId, - int32_t thriftDefaultBufferSize, int32_t thriftMaxFrameSize, - int32_t connectionTimeoutInMs, bool useSSL, bool enableRPCCompression, - std::string version, std::vector endpoints, NodeSelectionPolicy policy - ); - std::vector getEndPointList() override; - - boost::optional getQueryEndPoint() override; - - ~NodesSupplier() override; - -private: - std::string userName_; - std::string password_; - int32_t thriftDefaultBufferSize_; - int32_t thriftMaxFrameSize_; - int32_t connectionTimeoutInMs_; - bool useSSL_; - bool enableRPCCompression_; - std::string version; - std::string zoneId_; - - std::mutex mutex_; - std::vector endpoints_; - NodeSelectionPolicy selectionPolicy_; - - std::atomic isRunning_{false}; - std::thread refreshThread_; - std::condition_variable refreshCondition_; - - std::shared_ptr client_; - - void deduplicateEndpoints(); - - void startBackgroundRefresh(std::chrono::milliseconds interval); - - std::vector fetchLatestEndpoints(); - - void refreshEndpointList(); - - TEndPoint selectQueryEndpoint(); - - void stopBackgroundRefresh() noexcept; -}; - -#endif \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/main/Session.cpp b/iotdb-client/client-cpp/src/main/Session.cpp deleted file mode 100644 index 6e8d37e6199ea..0000000000000 --- a/iotdb-client/client-cpp/src/main/Session.cpp +++ /dev/null @@ -1,2048 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "Session.h" -#include -#include -#include -#include -#include -#include "NodesSupplier.h" -#include "SessionDataSet.h" - -using namespace std; - -/** -* Timeout of query can be set by users. -* A negative number means using the default configuration of server. -* And value 0 will disable the function of query timeout. -*/ -static const int64_t QUERY_TIMEOUT_MS = -1; - -LogLevelType LOG_LEVEL = LEVEL_DEBUG; - -TSDataType::TSDataType getTSDataTypeFromString(const string& str) { - // BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT, STRING, BLOB, TIMESTAMP, DATE, NULLTYPE - if (str == "BOOLEAN") { - return TSDataType::BOOLEAN; - } else if (str == "INT32") { - return TSDataType::INT32; - } else if (str == "INT64") { - return TSDataType::INT64; - } else if (str == "FLOAT") { - return TSDataType::FLOAT; - } else if (str == "DOUBLE") { - return TSDataType::DOUBLE; - } else if (str == "TEXT") { - return TSDataType::TEXT; - } else if (str == "TIMESTAMP") { - return TSDataType::TIMESTAMP; - } else if (str == "DATE") { - return TSDataType::DATE; - } else if (str == "BLOB") { - return TSDataType::BLOB; - } else if (str == "STRING") { - return TSDataType::STRING; - } - return TSDataType::UNKNOWN; -} - -void Tablet::createColumns() { - for (size_t i = 0; i < schemas.size(); i++) { - TSDataType::TSDataType dataType = schemas[i].second; - switch (dataType) { - case TSDataType::BOOLEAN: - values[i] = new bool[maxRowNumber]; - break; - case TSDataType::DATE: - values[i] = new boost::gregorian::date[maxRowNumber]; - break; - case TSDataType::INT32: - values[i] = new int[maxRowNumber]; - break; - case TSDataType::TIMESTAMP: - case TSDataType::INT64: - values[i] = new int64_t[maxRowNumber]; - break; - case TSDataType::FLOAT: - values[i] = new float[maxRowNumber]; - break; - case TSDataType::DOUBLE: - values[i] = new double[maxRowNumber]; - break; - case TSDataType::STRING: - case TSDataType::BLOB: - case TSDataType::TEXT: - values[i] = new string[maxRowNumber]; - break; - default: - throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); - } - } -} - -void Tablet::deleteColumns() { - for (size_t i = 0; i < schemas.size(); i++) { - if (!values[i]) continue; - TSDataType::TSDataType dataType = schemas[i].second; - switch (dataType) { - case TSDataType::BOOLEAN: { - bool* valueBuf = (bool*)(values[i]); - delete[] valueBuf; - break; - } - case TSDataType::INT32: { - int* valueBuf = (int*)(values[i]); - delete[] valueBuf; - break; - } - case TSDataType::DATE: { - boost::gregorian::date* valueBuf = (boost::gregorian::date*)(values[i]); - delete[] valueBuf; - break; - } - case TSDataType::TIMESTAMP: - case TSDataType::INT64: { - int64_t* valueBuf = (int64_t*)(values[i]); - delete[] valueBuf; - break; - } - case TSDataType::FLOAT: { - float* valueBuf = (float*)(values[i]); - delete[] valueBuf; - break; - } - case TSDataType::DOUBLE: { - double* valueBuf = (double*)(values[i]); - delete[] valueBuf; - break; - } - case TSDataType::STRING: - case TSDataType::BLOB: - case TSDataType::TEXT: { - string* valueBuf = (string*)(values[i]); - delete[] valueBuf; - break; - } - default: - throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); - } - values[i] = nullptr; - } -} - -void Tablet::deepCopyTabletColValue(void* const* srcPtr, void** destPtr, TSDataType::TSDataType type, int maxRowNumber) { - void *src = *srcPtr; - switch (type) { - case TSDataType::BOOLEAN: - *destPtr = new bool[maxRowNumber]; - memcpy(*destPtr, src, maxRowNumber * sizeof(bool)); - break; - case TSDataType::INT32: - *destPtr = new int32_t[maxRowNumber]; - memcpy(*destPtr, src, maxRowNumber * sizeof(int32_t)); - break; - case TSDataType::INT64: - case TSDataType::TIMESTAMP: - *destPtr = new int64_t[maxRowNumber]; - memcpy(*destPtr, src, maxRowNumber * sizeof(int64_t)); - break; - case TSDataType::FLOAT: - *destPtr = new float[maxRowNumber]; - memcpy(*destPtr, src, maxRowNumber * sizeof(float)); - break; - case TSDataType::DOUBLE: - *destPtr = new double[maxRowNumber]; - memcpy(*destPtr, src, maxRowNumber * sizeof(double)); - break; - case TSDataType::DATE: { - *destPtr = new boost::gregorian::date[maxRowNumber]; - boost::gregorian::date* srcDate = static_cast(src); - boost::gregorian::date* destDate = static_cast(*destPtr); - for (size_t j = 0; j < maxRowNumber; ++j) { - destDate[j] = srcDate[j]; - } - break; - } - case TSDataType::STRING: - case TSDataType::TEXT: - case TSDataType::BLOB: { - *destPtr = new std::string[maxRowNumber]; - std::string* srcStr = static_cast(src); - std::string* destStr = static_cast(*destPtr); - for (size_t j = 0; j < maxRowNumber; ++j) { - destStr[j] = srcStr[j]; - } - break; - } - default: - break; - } -} - -void Tablet::reset() { - rowSize = 0; - for (size_t i = 0; i < schemas.size(); i++) { - bitMaps[i].reset(); - } -} - -size_t Tablet::getTimeBytesSize() { - return rowSize * 8; -} - -size_t Tablet::getValueByteSize() { - size_t valueOccupation = 0; - for (size_t i = 0; i < schemas.size(); i++) { - switch (schemas[i].second) { - case TSDataType::BOOLEAN: - valueOccupation += rowSize; - break; - case TSDataType::INT32: - valueOccupation += rowSize * 4; - break; - case TSDataType::DATE: - valueOccupation += rowSize * 4; - break; - case TSDataType::TIMESTAMP: - case TSDataType::INT64: - valueOccupation += rowSize * 8; - break; - case TSDataType::FLOAT: - valueOccupation += rowSize * 4; - break; - case TSDataType::DOUBLE: - valueOccupation += rowSize * 8; - break; - case TSDataType::STRING: - case TSDataType::BLOB: - case TSDataType::TEXT: { - valueOccupation += rowSize * 4; - string* valueBuf = (string*)(values[i]); - for (size_t j = 0; j < rowSize; j++) { - valueOccupation += valueBuf[j].size(); - } - break; - } - default: - throw UnSupportedDataTypeException( - string("Data type ") + to_string(schemas[i].second) + " is not supported."); - } - } - return valueOccupation; -} - -void Tablet::setAligned(bool isAligned) { - this->isAligned = isAligned; -} - -std::shared_ptr Tablet::getDeviceID(int row) { - std::vector id_array(idColumnIndexes.size() + 1); - size_t idArrayIdx = 0; - id_array[idArrayIdx++] = this->deviceId; - for (auto idColumnIndex : idColumnIndexes) { - void* strPtr = getValue(idColumnIndex, row, TSDataType::TEXT); - id_array[idArrayIdx++] = *static_cast(strPtr); - } - return std::make_shared(id_array); -} - -string SessionUtils::getTime(const Tablet& tablet) { - MyStringBuffer timeBuffer; - unsigned int n = 8u * tablet.rowSize; - if (n > timeBuffer.str.capacity()) { - timeBuffer.reserve(n); - } - - for (size_t i = 0; i < tablet.rowSize; i++) { - timeBuffer.putInt64(tablet.timestamps[i]); - } - return timeBuffer.str; -} - -string SessionUtils::getValue(const Tablet& tablet) { - MyStringBuffer valueBuffer; - unsigned int n = 8u * tablet.schemas.size() * tablet.rowSize; - if (n > valueBuffer.str.capacity()) { - valueBuffer.reserve(n); - } - for (size_t i = 0; i < tablet.schemas.size(); i++) { - TSDataType::TSDataType dataType = tablet.schemas[i].second; - const BitMap& bitMap = tablet.bitMaps[i]; - switch (dataType) { - case TSDataType::BOOLEAN: { - bool* valueBuf = (bool*)(tablet.values[i]); - for (size_t index = 0; index < tablet.rowSize; index++) { - if (!bitMap.isMarked(index)) { - valueBuffer.putBool(valueBuf[index]); - } - else { - valueBuffer.putBool(false); - } - } - break; - } - case TSDataType::INT32: { - int* valueBuf = (int*)(tablet.values[i]); - for (size_t index = 0; index < tablet.rowSize; index++) { - if (!bitMap.isMarked(index)) { - valueBuffer.putInt(valueBuf[index]); - } - else { - valueBuffer.putInt((numeric_limits::min)()); - } - } - break; - } - case TSDataType::DATE: { - boost::gregorian::date* valueBuf = (boost::gregorian::date*)(tablet.values[i]); - for (size_t index = 0; index < tablet.rowSize; index++) { - if (!bitMap.isMarked(index)) { - valueBuffer.putDate(valueBuf[index]); - } - else { - valueBuffer.putInt(EMPTY_DATE_INT); - } - } - break; - } - case TSDataType::TIMESTAMP: - case TSDataType::INT64: { - int64_t* valueBuf = (int64_t*)(tablet.values[i]); - for (size_t index = 0; index < tablet.rowSize; index++) { - if (!bitMap.isMarked(index)) { - valueBuffer.putInt64(valueBuf[index]); - } - else { - valueBuffer.putInt64((numeric_limits::min)()); - } - } - break; - } - case TSDataType::FLOAT: { - float* valueBuf = (float*)(tablet.values[i]); - for (size_t index = 0; index < tablet.rowSize; index++) { - if (!bitMap.isMarked(index)) { - valueBuffer.putFloat(valueBuf[index]); - } - else { - valueBuffer.putFloat((numeric_limits::min)()); - } - } - break; - } - case TSDataType::DOUBLE: { - double* valueBuf = (double*)(tablet.values[i]); - for (size_t index = 0; index < tablet.rowSize; index++) { - if (!bitMap.isMarked(index)) { - valueBuffer.putDouble(valueBuf[index]); - } - else { - valueBuffer.putDouble((numeric_limits::min)()); - } - } - break; - } - case TSDataType::STRING: - case TSDataType::BLOB: - case TSDataType::TEXT: { - string* valueBuf = (string*)(tablet.values[i]); - for (size_t index = 0; index < tablet.rowSize; index++) { - if (!bitMap.isMarked(index)) { - valueBuffer.putString(valueBuf[index]); - } - else { - valueBuffer.putString(""); - } - } - break; - } - default: - throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); - } - } - for (size_t i = 0; i < tablet.schemas.size(); i++) { - const BitMap& bitMap = tablet.bitMaps[i]; - bool columnHasNull = !bitMap.isAllUnmarked(); - valueBuffer.putChar(columnHasNull ? (char)1 : (char)0); - if (columnHasNull) { - const vector& bytes = bitMap.getByteArray(); - for (size_t index = 0; index < tablet.rowSize / 8 + 1; index++) { - valueBuffer.putChar(bytes[index]); - } - } - } - return valueBuffer.str; -} - -bool SessionUtils::isTabletContainsSingleDevice(Tablet tablet) { - if (tablet.rowSize == 1) { - return true; - } - auto firstDeviceId = tablet.getDeviceID(0); - for (int i = 1; i < tablet.rowSize; ++i) { - if (*firstDeviceId != *tablet.getDeviceID(i)) { - return false; - } - } - return true; -} - -string MeasurementNode::serialize() const { - MyStringBuffer buffer; - buffer.putString(getName()); - buffer.putChar(getDataType()); - buffer.putChar(getEncoding()); - buffer.putChar(getCompressionType()); - return buffer.str; -} - -string Template::serialize() const { - MyStringBuffer buffer; - stack>> stack; - unordered_set alignedPrefix; - buffer.putString(getName()); - buffer.putBool(isAligned()); - if (isAligned()) { - alignedPrefix.emplace(""); - } - - for (const auto& child : children_) { - stack.push(make_pair("", child.second)); - } - - while (!stack.empty()) { - auto cur = stack.top(); - stack.pop(); - - string prefix = cur.first; - shared_ptr cur_node_ptr = cur.second; - string fullPath(prefix); - - if (!cur_node_ptr->isMeasurement()) { - if (!prefix.empty()) { - fullPath.append("."); - } - fullPath.append(cur_node_ptr->getName()); - if (cur_node_ptr->isAligned()) { - alignedPrefix.emplace(fullPath); - } - for (const auto& child : cur_node_ptr->getChildren()) { - stack.push(make_pair(fullPath, child.second)); - } - } - else { - buffer.putString(prefix); - buffer.putBool(alignedPrefix.find(prefix) != alignedPrefix.end()); - buffer.concat(cur_node_ptr->serialize()); - } - } - - return buffer.str; -} - -/** - * When delete variable, make sure release all resource. - */ -Session::~Session() { - try { - close(); - } - catch (const exception& e) { - log_debug(e.what()); - } -} - -void Session::removeBrokenSessionConnection(shared_ptr sessionConnection) { - if (enableRedirection_) { - this->endPointToSessionConnection.erase(sessionConnection->getEndPoint()); - } - - auto it1 = deviceIdToEndpoint.begin(); - while (it1 != deviceIdToEndpoint.end()) { - if (it1->second == sessionConnection->getEndPoint()) { - it1 = deviceIdToEndpoint.erase(it1); - } - else { - ++it1; - } - } - - auto it2 = tableModelDeviceIdToEndpoint.begin(); - while (it2 != tableModelDeviceIdToEndpoint.end()) { - if (it2->second == sessionConnection->getEndPoint()) { - it2 = tableModelDeviceIdToEndpoint.erase(it2); - } - else { - ++it2; - } - } -} - -/** - * check whether the batch has been sorted - * - * @return whether the batch has been sorted - */ -bool Session::checkSorted(const Tablet& tablet) { - for (size_t i = 1; i < tablet.rowSize; i++) { - if (tablet.timestamps[i] < tablet.timestamps[i - 1]) { - return false; - } - } - return true; -} - -bool Session::checkSorted(const vector& times) { - for (size_t i = 1; i < times.size(); i++) { - if (times[i] < times[i - 1]) { - return false; - } - } - return true; -} - -template -std::vector sortList(const std::vector& valueList, const int* index, int indexLength) { - std::vector sortedValues(valueList.size()); - for (int i = 0; i < indexLength; i++) { - sortedValues[i] = valueList[index[i]]; - } - return sortedValues; -} - -template -void sortValuesList(T* valueList, const int* index, size_t indexLength) { - T* sortedValues = new T[indexLength]; - for (int i = 0; i < indexLength; i++) { - sortedValues[i] = valueList[index[i]]; - } - for (int i = 0; i < indexLength; i++) { - valueList[i] = sortedValues[i]; - } - delete[] sortedValues; -} - -void Session::sortTablet(Tablet& tablet) { - /* - * following part of code sort the batch data by time, - * so we can insert continuous data in value list to get a better performance - */ - // sort to get index, and use index to sort value list - int* index = new int[tablet.rowSize]; - for (size_t i = 0; i < tablet.rowSize; i++) { - index[i] = i; - } - - sortIndexByTimestamp(index, tablet.timestamps, tablet.rowSize); - tablet.timestamps = sortList(tablet.timestamps, index, tablet.rowSize); - for (size_t i = 0; i < tablet.schemas.size(); i++) { - TSDataType::TSDataType dataType = tablet.schemas[i].second; - switch (dataType) { - case TSDataType::BOOLEAN: { - sortValuesList((bool*)(tablet.values[i]), index, tablet.rowSize); - break; - } - case TSDataType::INT32: { - sortValuesList((int*)(tablet.values[i]), index, tablet.rowSize); - break; - } - case TSDataType::DATE: { - sortValuesList((boost::gregorian::date*)(tablet.values[i]), index, tablet.rowSize); - break; - } - case TSDataType::TIMESTAMP: - case TSDataType::INT64: { - sortValuesList((int64_t*)(tablet.values[i]), index, tablet.rowSize); - break; - } - case TSDataType::FLOAT: { - sortValuesList((float*)(tablet.values[i]), index, tablet.rowSize); - break; - } - case TSDataType::DOUBLE: { - sortValuesList((double*)(tablet.values[i]), index, tablet.rowSize); - break; - } - case TSDataType::STRING: - case TSDataType::BLOB: - case TSDataType::TEXT: { - sortValuesList((string*)(tablet.values[i]), index, tablet.rowSize); - break; - } - default: - throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); - } - } - - delete[] index; -} - -void Session::sortIndexByTimestamp(int* index, std::vector& timestamps, int length) { - if (length <= 1) { - return; - } - - TsCompare tsCompareObj(timestamps); - std::sort(&index[0], &index[length], tsCompareObj); -} - -/** - * Append value into buffer in Big Endian order to comply with IoTDB server - */ -void Session::appendValues(string& buffer, const char* value, int size) { - static bool hasCheckedEndianFlag = false; - static bool localCpuIsBigEndian = false; - if (!hasCheckedEndianFlag) { - hasCheckedEndianFlag = true; - int chk = 0x0201; //used to distinguish CPU's type (BigEndian or LittleEndian) - localCpuIsBigEndian = (0x01 != *(char*)(&chk)); - } - - if (localCpuIsBigEndian) { - buffer.append(value, size); - } - else { - for (int i = size - 1; i >= 0; i--) { - buffer.append(value + i, 1); - } - } -} - -void -Session::putValuesIntoBuffer(const vector& types, const vector& values, string& buf) { - int32_t date; - for (size_t i = 0; i < values.size(); i++) { - int8_t typeNum = getDataTypeNumber(types[i]); - buf.append((char*)(&typeNum), sizeof(int8_t)); - switch (types[i]) { - case TSDataType::BOOLEAN: - buf.append(values[i], 1); - break; - case TSDataType::INT32: - appendValues(buf, values[i], sizeof(int32_t)); - break; - case TSDataType::DATE: - date = parseDateExpressionToInt(*(boost::gregorian::date*)values[i]); - appendValues(buf, (char*)&date, sizeof(int32_t)); - break; - case TSDataType::TIMESTAMP: - case TSDataType::INT64: - appendValues(buf, values[i], sizeof(int64_t)); - break; - case TSDataType::FLOAT: - appendValues(buf, values[i], sizeof(float)); - break; - case TSDataType::DOUBLE: - appendValues(buf, values[i], sizeof(double)); - break; - case TSDataType::STRING: - case TSDataType::BLOB: - case TSDataType::TEXT: { - int32_t len = (uint32_t)strlen(values[i]); - appendValues(buf, (char*)(&len), sizeof(uint32_t)); - // no need to change the byte order of string value - buf.append(values[i], len); - break; - } - default: - break; - } - } -} - -int8_t Session::getDataTypeNumber(TSDataType::TSDataType type) { - switch (type) { - case TSDataType::BOOLEAN: - return 0; - case TSDataType::INT32: - return 1; - case TSDataType::INT64: - return 2; - case TSDataType::FLOAT: - return 3; - case TSDataType::DOUBLE: - return 4; - case TSDataType::TEXT: - return 5; - case TSDataType::TIMESTAMP: - return 8; - case TSDataType::DATE: - return 9; - case TSDataType::BLOB: - return 10; - case TSDataType::STRING: - return 11; - default: - return -1; - } -} - -string Session::getVersionString(Version::Version version) { - switch (version) { - case Version::V_0_12: - return "V_0_12"; - case Version::V_0_13: - return "V_0_13"; - case Version::V_1_0: - return "V_1_0"; - default: - return "V_0_12"; - } -} - -void Session::initZoneId() { - if (!zoneId_.empty()) { - return; - } - - time_t ts = 0; - struct tm tmv; -#if defined(_WIN64) || defined (WIN32) || defined (_WIN32) - localtime_s(&tmv, &ts); -#else - localtime_r(&ts, &tmv); -#endif - - char zoneStr[32]; - strftime(zoneStr, sizeof(zoneStr), "%z", &tmv); - zoneId_ = zoneStr; -} - -void Session::initNodesSupplier() { - std::vector endPoints; - TEndPoint endPoint; - endPoint.__set_ip(host_); - endPoint.__set_port(rpcPort_); - endPoints.emplace_back(endPoint); - if (enableAutoFetch_) { - nodesSupplier_ = NodesSupplier::create(endPoints, username_, password_); - } - else { - nodesSupplier_ = make_shared(endPoints); - } -} - -void Session::initDefaultSessionConnection() { - defaultEndPoint_.__set_ip(host_); - defaultEndPoint_.__set_port(rpcPort_); - defaultSessionConnection_ = make_shared(this, defaultEndPoint_, zoneId_, nodesSupplier_, fetchSize_, - 60, 500, - sqlDialect_, database_); -} - -void Session::insertStringRecordsWithLeaderCache(vector deviceIds, vector times, - vector> measurementsList, - vector> valuesList, bool isAligned) { - std::unordered_map, TSInsertStringRecordsReq> recordsGroup; - for (int i = 0; i < deviceIds.size(); i++) { - auto connection = getSessionConnection(deviceIds[i]); - if (recordsGroup.find(connection) == recordsGroup.end()) { - TSInsertStringRecordsReq request; - std::vector emptyPrefixPaths; - std::vector> emptyMeasurementsList; - vector> emptyValuesList; - std::vector emptyTimestamps; - request.__set_isAligned(isAligned); - request.__set_prefixPaths(emptyPrefixPaths); - request.__set_timestamps(emptyTimestamps); - request.__set_measurementsList(emptyMeasurementsList); - request.__set_valuesList(emptyValuesList); - recordsGroup.insert(make_pair(connection, request)); - } - TSInsertStringRecordsReq& existingReq = recordsGroup[connection]; - existingReq.prefixPaths.emplace_back(deviceIds[i]); - existingReq.timestamps.emplace_back(times[i]); - existingReq.measurementsList.emplace_back(measurementsList[i]); - existingReq.valuesList.emplace_back(valuesList[i]); - } - std::function, const TSInsertStringRecordsReq&)> consumer = - [](const std::shared_ptr& c, const TSInsertStringRecordsReq& r) { - c->insertStringRecords(r); - }; - if (recordsGroup.size() == 1) { - insertOnce(recordsGroup, consumer); - } - else { - insertByGroup(recordsGroup, consumer); - } -} - -void Session::insertRecordsWithLeaderCache(vector deviceIds, vector times, - vector> measurementsList, - const vector>& typesList, - vector> valuesList, bool isAligned) { - std::unordered_map, TSInsertRecordsReq> recordsGroup; - for (int i = 0; i < deviceIds.size(); i++) { - auto connection = getSessionConnection(deviceIds[i]); - if (recordsGroup.find(connection) == recordsGroup.end()) { - TSInsertRecordsReq request; - std::vector emptyPrefixPaths; - std::vector> emptyMeasurementsList; - std::vector emptyValuesList; - std::vector emptyTimestamps; - request.__set_isAligned(isAligned); - request.__set_prefixPaths(emptyPrefixPaths); - request.__set_timestamps(emptyTimestamps); - request.__set_measurementsList(emptyMeasurementsList); - request.__set_valuesList(emptyValuesList); - recordsGroup.insert(make_pair(connection, request)); - } - TSInsertRecordsReq& existingReq = recordsGroup[connection]; - existingReq.prefixPaths.emplace_back(deviceIds[i]); - existingReq.timestamps.emplace_back(times[i]); - existingReq.measurementsList.emplace_back(measurementsList[i]); - vector bufferList; - string buffer; - putValuesIntoBuffer(typesList[i], valuesList[i], buffer); - existingReq.valuesList.emplace_back(buffer); - recordsGroup[connection] = existingReq; - } - std::function, const TSInsertRecordsReq&)> consumer = - [](const std::shared_ptr& c, const TSInsertRecordsReq& r) { - c->insertRecords(r); - }; - if (recordsGroup.size() == 1) { - insertOnce(recordsGroup, consumer); - } - else { - insertByGroup(recordsGroup, consumer); - } -} - -void Session::insertTabletsWithLeaderCache(unordered_map tablets, bool sorted, bool isAligned) { - std::unordered_map, TSInsertTabletsReq> tabletsGroup; - if (tablets.empty()) { - throw BatchExecutionException("No tablet is inserting!"); - } - for (const auto& item : tablets) { - if (isAligned != item.second->isAligned) { - throw BatchExecutionException("The tablets should be all aligned or non-aligned!"); - } - if (!checkSorted(*(item.second))) { - sortTablet(*(item.second)); - } - auto deviceId = item.first; - auto tablet = item.second; - auto connection = getSessionConnection(deviceId); - auto it = tabletsGroup.find(connection); - if (it == tabletsGroup.end()) { - TSInsertTabletsReq request; - tabletsGroup[connection] = request; - } - TSInsertTabletsReq& existingReq = tabletsGroup[connection]; - existingReq.prefixPaths.emplace_back(tablet->deviceId); - existingReq.timestampsList.emplace_back(move(SessionUtils::getTime(*tablet))); - existingReq.valuesList.emplace_back(move(SessionUtils::getValue(*tablet))); - existingReq.sizeList.emplace_back(tablet->rowSize); - vector dataTypes; - vector measurements; - for (pair schema : tablet->schemas) { - measurements.push_back(schema.first); - dataTypes.push_back(schema.second); - } - existingReq.measurementsList.emplace_back(measurements); - existingReq.typesList.emplace_back(dataTypes); - } - - std::function, const TSInsertTabletsReq&)> consumer = - [](const std::shared_ptr& c, const TSInsertTabletsReq& r) { - c->insertTablets(r); - }; - if (tabletsGroup.size() == 1) { - insertOnce(tabletsGroup, consumer); - } - else { - insertByGroup(tabletsGroup, consumer); - } -} - -void Session::open() { - open(false, DEFAULT_TIMEOUT_MS); -} - -void Session::open(bool enableRPCCompression) { - open(enableRPCCompression, DEFAULT_TIMEOUT_MS); -} - -void Session::open(bool enableRPCCompression, int connectionTimeoutInMs) { - if (!isClosed_) { - return; - } - - try { - initDefaultSessionConnection(); - } - catch (const exception& e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } - zoneId_ = defaultSessionConnection_->zoneId; - - if (enableRedirection_) { - endPointToSessionConnection.insert(make_pair(defaultEndPoint_, defaultSessionConnection_)); - } - - isClosed_ = false; -} - - -void Session::close() { - if (isClosed_) { - return; - } - isClosed_ = true; -} - - -void Session::insertRecord(const string& deviceId, int64_t time, - const vector& measurements, - const vector& values) { - TSInsertStringRecordReq req; - req.__set_prefixPath(deviceId); - req.__set_timestamp(time); - req.__set_measurements(measurements); - req.__set_values(values); - req.__set_isAligned(false); - try { - getSessionConnection(deviceId)->insertStringRecord(req); - } - catch (RedirectException& e) { - handleRedirection(deviceId, e.endPoint); - } catch (const IoTDBConnectionException& e) { - if (enableRedirection_ && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { - deviceIdToEndpoint.erase(deviceId); - try { - defaultSessionConnection_->insertStringRecord(req); - } - catch (RedirectException& e) { - } - } - else { - throw e; - } - } -} - -void Session::insertRecord(const string& deviceId, int64_t time, - const vector& measurements, - const vector& types, - const vector& values) { - TSInsertRecordReq req; - req.__set_prefixPath(deviceId); - req.__set_timestamp(time); - req.__set_measurements(measurements); - string buffer; - putValuesIntoBuffer(types, values, buffer); - req.__set_values(buffer); - req.__set_isAligned(false); - try { - getSessionConnection(deviceId)->insertRecord(req); - } - catch (RedirectException& e) { - handleRedirection(deviceId, e.endPoint); - } catch (const IoTDBConnectionException& e) { - if (enableRedirection_ && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { - deviceIdToEndpoint.erase(deviceId); - try { - defaultSessionConnection_->insertRecord(req); - } - catch (RedirectException& e) { - } - } - else { - throw e; - } - } -} - -void Session::insertAlignedRecord(const string& deviceId, int64_t time, - const vector& measurements, - const vector& values) { - TSInsertStringRecordReq req; - req.__set_prefixPath(deviceId); - req.__set_timestamp(time); - req.__set_measurements(measurements); - req.__set_values(values); - req.__set_isAligned(true); - try { - getSessionConnection(deviceId)->insertStringRecord(req); - } - catch (RedirectException& e) { - handleRedirection(deviceId, e.endPoint); - } catch (const IoTDBConnectionException& e) { - if (enableRedirection_ && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { - deviceIdToEndpoint.erase(deviceId); - try { - defaultSessionConnection_->insertStringRecord(req); - } - catch (RedirectException& e) { - } - } - else { - throw e; - } - } -} - -void Session::insertAlignedRecord(const string& deviceId, int64_t time, - const vector& measurements, - const vector& types, - const vector& values) { - TSInsertRecordReq req; - req.__set_prefixPath(deviceId); - req.__set_timestamp(time); - req.__set_measurements(measurements); - string buffer; - putValuesIntoBuffer(types, values, buffer); - req.__set_values(buffer); - req.__set_isAligned(false); - try { - getSessionConnection(deviceId)->insertRecord(req); - } - catch (RedirectException& e) { - handleRedirection(deviceId, e.endPoint); - } catch (const IoTDBConnectionException& e) { - if (enableRedirection_ && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { - deviceIdToEndpoint.erase(deviceId); - try { - defaultSessionConnection_->insertRecord(req); - } - catch (RedirectException& e) { - } - } - else { - throw e; - } - } -} - -void Session::insertRecords(const vector& deviceIds, - const vector& times, - const vector>& measurementsList, - const vector>& valuesList) { - size_t len = deviceIds.size(); - if (len != times.size() || len != measurementsList.size() || len != valuesList.size()) { - logic_error e("deviceIds, times, measurementsList and valuesList's size should be equal"); - throw exception(e); - } - - if (enableRedirection_) { - insertStringRecordsWithLeaderCache(deviceIds, times, measurementsList, valuesList, false); - } - else { - TSInsertStringRecordsReq request; - request.__set_prefixPaths(deviceIds); - request.__set_timestamps(times); - request.__set_measurementsList(measurementsList); - request.__set_valuesList(valuesList); - request.__set_isAligned(false); - try { - defaultSessionConnection_->insertStringRecords(request); - } - catch (RedirectException& e) { - } - } -} - -void Session::insertRecords(const vector& deviceIds, - const vector& times, - const vector>& measurementsList, - const vector>& typesList, - const vector>& valuesList) { - size_t len = deviceIds.size(); - if (len != times.size() || len != measurementsList.size() || len != valuesList.size()) { - logic_error e("deviceIds, times, measurementsList and valuesList's size should be equal"); - throw exception(e); - } - - if (enableRedirection_) { - insertRecordsWithLeaderCache(deviceIds, times, measurementsList, typesList, valuesList, false); - } - else { - TSInsertRecordsReq request; - request.__set_prefixPaths(deviceIds); - request.__set_timestamps(times); - request.__set_measurementsList(measurementsList); - vector bufferList; - for (size_t i = 0; i < valuesList.size(); i++) { - string buffer; - putValuesIntoBuffer(typesList[i], valuesList[i], buffer); - bufferList.push_back(buffer); - } - request.__set_valuesList(bufferList); - request.__set_isAligned(false); - try { - defaultSessionConnection_->insertRecords(request); - } - catch (RedirectException& e) { - } - } -} - -void Session::insertAlignedRecords(const vector& deviceIds, - const vector& times, - const vector>& measurementsList, - const vector>& valuesList) { - size_t len = deviceIds.size(); - if (len != times.size() || len != measurementsList.size() || len != valuesList.size()) { - logic_error e("deviceIds, times, measurementsList and valuesList's size should be equal"); - throw exception(e); - } - - if (enableRedirection_) { - insertStringRecordsWithLeaderCache(deviceIds, times, measurementsList, valuesList, true); - } - else { - TSInsertStringRecordsReq request; - request.__set_prefixPaths(deviceIds); - request.__set_timestamps(times); - request.__set_measurementsList(measurementsList); - request.__set_valuesList(valuesList); - request.__set_isAligned(true); - try { - defaultSessionConnection_->insertStringRecords(request); - } - catch (RedirectException& e) { - } - } -} - -void Session::insertAlignedRecords(const vector& deviceIds, - const vector& times, - const vector>& measurementsList, - const vector>& typesList, - const vector>& valuesList) { - size_t len = deviceIds.size(); - if (len != times.size() || len != measurementsList.size() || len != valuesList.size()) { - logic_error e("deviceIds, times, measurementsList and valuesList's size should be equal"); - throw exception(e); - } - - if (enableRedirection_) { - insertRecordsWithLeaderCache(deviceIds, times, measurementsList, typesList, valuesList, true); - } - else { - TSInsertRecordsReq request; - request.__set_prefixPaths(deviceIds); - request.__set_timestamps(times); - request.__set_measurementsList(measurementsList); - vector bufferList; - for (size_t i = 0; i < valuesList.size(); i++) { - string buffer; - putValuesIntoBuffer(typesList[i], valuesList[i], buffer); - bufferList.push_back(buffer); - } - request.__set_valuesList(bufferList); - request.__set_isAligned(false); - try { - defaultSessionConnection_->insertRecords(request); - } - catch (RedirectException& e) { - } - } -} - -void Session::insertRecordsOfOneDevice(const string& deviceId, - vector& times, - vector>& measurementsList, - vector>& typesList, - vector>& valuesList) { - insertRecordsOfOneDevice(deviceId, times, measurementsList, typesList, valuesList, false); -} - -void Session::insertRecordsOfOneDevice(const string& deviceId, - vector& times, - vector>& measurementsList, - vector>& typesList, - vector>& valuesList, - bool sorted) { - if (!checkSorted(times)) { - int* index = new int[times.size()]; - for (size_t i = 0; i < times.size(); i++) { - index[i] = (int)i; - } - - sortIndexByTimestamp(index, times, (int)(times.size())); - times = sortList(times, index, (int)(times.size())); - measurementsList = sortList(measurementsList, index, (int)(times.size())); - typesList = sortList(typesList, index, (int)(times.size())); - valuesList = sortList(valuesList, index, (int)(times.size())); - delete[] index; - } - TSInsertRecordsOfOneDeviceReq request; - request.__set_prefixPath(deviceId); - request.__set_timestamps(times); - request.__set_measurementsList(measurementsList); - vector bufferList; - for (size_t i = 0; i < valuesList.size(); i++) { - string buffer; - putValuesIntoBuffer(typesList[i], valuesList[i], buffer); - bufferList.push_back(buffer); - } - request.__set_valuesList(bufferList); - request.__set_isAligned(false); - TSStatus respStatus; - try { - getSessionConnection(deviceId)->insertRecordsOfOneDevice(request); - } - catch (RedirectException& e) { - handleRedirection(deviceId, e.endPoint); - } catch (const IoTDBConnectionException& e) { - if (enableRedirection_ && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { - deviceIdToEndpoint.erase(deviceId); - try { - defaultSessionConnection_->insertRecordsOfOneDevice(request); - } - catch (RedirectException& e) { - } - } - else { - throw e; - } - } -} - -void Session::insertAlignedRecordsOfOneDevice(const string& deviceId, - vector& times, - vector>& measurementsList, - vector>& typesList, - vector>& valuesList) { - insertAlignedRecordsOfOneDevice(deviceId, times, measurementsList, typesList, valuesList, false); -} - -void Session::insertAlignedRecordsOfOneDevice(const string& deviceId, - vector& times, - vector>& measurementsList, - vector>& typesList, - vector>& valuesList, - bool sorted) { - if (!checkSorted(times)) { - int* index = new int[times.size()]; - for (size_t i = 0; i < times.size(); i++) { - index[i] = (int)i; - } - - sortIndexByTimestamp(index, times, (int)(times.size())); - times = sortList(times, index, (int)(times.size())); - measurementsList = sortList(measurementsList, index, (int)(times.size())); - typesList = sortList(typesList, index, (int)(times.size())); - valuesList = sortList(valuesList, index, (int)(times.size())); - delete[] index; - } - TSInsertRecordsOfOneDeviceReq request; - request.__set_prefixPath(deviceId); - request.__set_timestamps(times); - request.__set_measurementsList(measurementsList); - vector bufferList; - for (size_t i = 0; i < valuesList.size(); i++) { - string buffer; - putValuesIntoBuffer(typesList[i], valuesList[i], buffer); - bufferList.push_back(buffer); - } - request.__set_valuesList(bufferList); - request.__set_isAligned(true); - TSStatus respStatus; - try { - getSessionConnection(deviceId)->insertRecordsOfOneDevice(request); - } - catch (RedirectException& e) { - handleRedirection(deviceId, e.endPoint); - } catch (const IoTDBConnectionException& e) { - if (enableRedirection_ && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { - deviceIdToEndpoint.erase(deviceId); - try { - defaultSessionConnection_->insertRecordsOfOneDevice(request); - } - catch (RedirectException& e) { - } - } - else { - throw e; - } - } -} - -void Session::insertTablet(Tablet& tablet) { - try { - insertTablet(tablet, false); - } - catch (const exception& e) { - log_debug(e.what()); - logic_error error(e.what()); - throw exception(error); - } -} - -void Session::buildInsertTabletReq(TSInsertTabletReq& request, Tablet& tablet, bool sorted) { - if ((!sorted) && !checkSorted(tablet)) { - sortTablet(tablet); - } - - request.prefixPath = tablet.deviceId; - - request.measurements.reserve(tablet.schemas.size()); - request.types.reserve(tablet.schemas.size()); - for (pair schema : tablet.schemas) { - request.measurements.push_back(schema.first); - request.types.push_back(schema.second); - } - request.values = move(SessionUtils::getValue(tablet)); - request.timestamps = move(SessionUtils::getTime(tablet)); - request.__set_size(tablet.rowSize); - request.__set_isAligned(tablet.isAligned); -} - -void Session::insertTablet(TSInsertTabletReq request) { - auto deviceId = request.prefixPath; - try { - getSessionConnection(deviceId)->insertTablet(request); - } - catch (RedirectException& e) { - handleRedirection(deviceId, e.endPoint); - } catch (const IoTDBConnectionException& e) { - if (enableRedirection_ && deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { - deviceIdToEndpoint.erase(deviceId); - try { - defaultSessionConnection_->insertTablet(request); - } - catch (RedirectException& e) { - } - } - else { - throw e; - } - } -} - -void Session::insertTablet(Tablet& tablet, bool sorted) { - TSInsertTabletReq request; - buildInsertTabletReq(request, tablet, sorted); - insertTablet(request); -} - -void Session::insertAlignedTablet(Tablet& tablet) { - insertAlignedTablet(tablet, false); -} - -void Session::insertAlignedTablet(Tablet& tablet, bool sorted) { - tablet.setAligned(true); - try { - insertTablet(tablet, sorted); - } - catch (const exception& e) { - log_debug(e.what()); - logic_error error(e.what()); - throw exception(error); - } -} - -void Session::insertTablets(unordered_map& tablets) { - try { - insertTablets(tablets, false); - } - catch (const exception& e) { - log_debug(e.what()); - logic_error error(e.what()); - throw exception(error); - } -} - -void Session::insertTablets(unordered_map& tablets, bool sorted) { - if (tablets.empty()) { - throw BatchExecutionException("No tablet is inserting!"); - } - auto beginIter = tablets.begin(); - bool isAligned = ((*beginIter).second)->isAligned; - if (enableRedirection_) { - insertTabletsWithLeaderCache(tablets, sorted, isAligned); - } - else { - TSInsertTabletsReq request; - for (const auto& item : tablets) { - if (isAligned != item.second->isAligned) { - throw BatchExecutionException("The tablets should be all aligned or non-aligned!"); - } - if (!checkSorted(*(item.second))) { - sortTablet(*(item.second)); - } - request.prefixPaths.push_back(item.second->deviceId); - vector measurements; - vector dataTypes; - for (pair schema : item.second->schemas) { - measurements.push_back(schema.first); - dataTypes.push_back(schema.second); - } - request.measurementsList.push_back(measurements); - request.typesList.push_back(dataTypes); - request.timestampsList.push_back(move(SessionUtils::getTime(*(item.second)))); - request.valuesList.push_back(move(SessionUtils::getValue(*(item.second)))); - request.sizeList.push_back(item.second->rowSize); - } - request.__set_isAligned(isAligned); - try { - TSStatus respStatus; - defaultSessionConnection_->insertTablets(request); - RpcUtils::verifySuccess(respStatus); - } - catch (RedirectException& e) { - } - } -} - - -void Session::insertAlignedTablets(unordered_map& tablets, bool sorted) { - for (auto iter = tablets.begin(); iter != tablets.end(); iter++) { - iter->second->setAligned(true); - } - try { - insertTablets(tablets, sorted); - } - catch (const exception& e) { - log_debug(e.what()); - logic_error error(e.what()); - throw exception(error); - } -} - -void Session::testInsertRecord(const string& deviceId, int64_t time, const vector& measurements, - const vector& values) { - TSInsertStringRecordReq req; - req.__set_prefixPath(deviceId); - req.__set_timestamp(time); - req.__set_measurements(measurements); - req.__set_values(values); - TSStatus tsStatus; - try { - defaultSessionConnection_->testInsertStringRecord(req); - RpcUtils::verifySuccess(tsStatus); - } - catch (const TTransportException& e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException& e) { - log_debug(e.what()); - throw; - } catch (const exception& e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } -} - -void Session::testInsertTablet(const Tablet& tablet) { - TSInsertTabletReq request; - request.prefixPath = tablet.deviceId; - for (pair schema : tablet.schemas) { - request.measurements.push_back(schema.first); - request.types.push_back(schema.second); - } - request.__set_timestamps(move(SessionUtils::getTime(tablet))); - request.__set_values(move(SessionUtils::getValue(tablet))); - request.__set_size(tablet.rowSize); - try { - TSStatus tsStatus; - defaultSessionConnection_->testInsertTablet(request); - RpcUtils::verifySuccess(tsStatus); - } - catch (const TTransportException& e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException& e) { - log_debug(e.what()); - throw; - } catch (const exception& e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } -} - -void Session::testInsertRecords(const vector& deviceIds, - const vector& times, - const vector>& measurementsList, - const vector>& valuesList) { - size_t len = deviceIds.size(); - if (len != times.size() || len != measurementsList.size() || len != valuesList.size()) { - logic_error error("deviceIds, times, measurementsList and valuesList's size should be equal"); - throw exception(error); - } - TSInsertStringRecordsReq request; - request.__set_prefixPaths(deviceIds); - request.__set_timestamps(times); - request.__set_measurementsList(measurementsList); - request.__set_valuesList(valuesList); - - try { - TSStatus tsStatus; - defaultSessionConnection_->getSessionClient()->insertStringRecords(tsStatus, request); - RpcUtils::verifySuccess(tsStatus); - } - catch (const TTransportException& e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException& e) { - log_debug(e.what()); - throw; - } catch (const exception& e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } -} - -void Session::deleteTimeseries(const string& path) { - vector paths; - paths.push_back(path); - deleteTimeseries(paths); -} - -void Session::deleteTimeseries(const vector& paths) { - defaultSessionConnection_->deleteTimeseries(paths); -} - -void Session::deleteData(const string& path, int64_t endTime) { - vector paths; - paths.push_back(path); - deleteData(paths, LONG_LONG_MIN, endTime); -} - -void Session::deleteData(const vector& paths, int64_t endTime) { - deleteData(paths, LONG_LONG_MIN, endTime); -} - -void Session::deleteData(const vector& paths, int64_t startTime, int64_t endTime) { - TSDeleteDataReq req; - req.__set_paths(paths); - req.__set_startTime(startTime); - req.__set_endTime(endTime); - defaultSessionConnection_->deleteData(req); -} - -void Session::setStorageGroup(const string& storageGroupId) { - defaultSessionConnection_->setStorageGroup(storageGroupId); -} - -void Session::deleteStorageGroup(const string& storageGroup) { - vector storageGroups; - storageGroups.push_back(storageGroup); - deleteStorageGroups(storageGroups); -} - -void Session::deleteStorageGroups(const vector& storageGroups) { - defaultSessionConnection_->deleteStorageGroups(storageGroups); -} - -void Session::createDatabase(const string& database) { - this->setStorageGroup(database); -} - -void Session::deleteDatabase(const string& database) { - this->deleteStorageGroups(vector{database}); -} - -void Session::deleteDatabases(const vector& databases) { - this->deleteStorageGroups(databases); -} - -void Session::createTimeseries(const string& path, - TSDataType::TSDataType dataType, - TSEncoding::TSEncoding encoding, - CompressionType::CompressionType compressor) { - try { - createTimeseries(path, dataType, encoding, compressor, nullptr, nullptr, nullptr, ""); - } - catch (const exception& e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } -} - -void Session::createTimeseries(const string& path, - TSDataType::TSDataType dataType, - TSEncoding::TSEncoding encoding, - CompressionType::CompressionType compressor, - map* props, - map* tags, - map* attributes, - const string& measurementAlias) { - TSCreateTimeseriesReq req; - req.__set_path(path); - req.__set_dataType(dataType); - req.__set_encoding(encoding); - req.__set_compressor(compressor); - if (props != nullptr) { - req.__set_props(*props); - } - - if (tags != nullptr) { - req.__set_tags(*tags); - } - if (attributes != nullptr) { - req.__set_attributes(*attributes); - } - if (!measurementAlias.empty()) { - req.__set_measurementAlias(measurementAlias); - } - defaultSessionConnection_->createTimeseries(req); -} - -void Session::createMultiTimeseries(const vector& paths, - const vector& dataTypes, - const vector& encodings, - const vector& compressors, - vector>* propsList, - vector>* tagsList, - vector>* attributesList, - vector* measurementAliasList) { - TSCreateMultiTimeseriesReq request; - request.__set_paths(paths); - - vector dataTypesOrdinal; - dataTypesOrdinal.reserve(dataTypes.size()); - for (TSDataType::TSDataType dataType : dataTypes) { - dataTypesOrdinal.push_back(dataType); - } - request.__set_dataTypes(dataTypesOrdinal); - - vector encodingsOrdinal; - encodingsOrdinal.reserve(encodings.size()); - for (TSEncoding::TSEncoding encoding : encodings) { - encodingsOrdinal.push_back(encoding); - } - request.__set_encodings(encodingsOrdinal); - - vector compressorsOrdinal; - compressorsOrdinal.reserve(compressors.size()); - for (CompressionType::CompressionType compressor : compressors) { - compressorsOrdinal.push_back(compressor); - } - request.__set_compressors(compressorsOrdinal); - - if (propsList != nullptr) { - request.__set_propsList(*propsList); - } - - if (tagsList != nullptr) { - request.__set_tagsList(*tagsList); - } - if (attributesList != nullptr) { - request.__set_attributesList(*attributesList); - } - if (measurementAliasList != nullptr) { - request.__set_measurementAliasList(*measurementAliasList); - } - - defaultSessionConnection_->createMultiTimeseries(request); -} - -void Session::createAlignedTimeseries(const std::string& deviceId, - const std::vector& measurements, - const std::vector& dataTypes, - const std::vector& encodings, - const std::vector& compressors) { - TSCreateAlignedTimeseriesReq request; - request.__set_prefixPath(deviceId); - request.__set_measurements(measurements); - - vector dataTypesOrdinal; - dataTypesOrdinal.reserve(dataTypes.size()); - for (TSDataType::TSDataType dataType : dataTypes) { - dataTypesOrdinal.push_back(dataType); - } - request.__set_dataTypes(dataTypesOrdinal); - - vector encodingsOrdinal; - encodingsOrdinal.reserve(encodings.size()); - for (TSEncoding::TSEncoding encoding : encodings) { - encodingsOrdinal.push_back(encoding); - } - request.__set_encodings(encodingsOrdinal); - - vector compressorsOrdinal; - compressorsOrdinal.reserve(compressors.size()); - for (CompressionType::CompressionType compressor : compressors) { - compressorsOrdinal.push_back(compressor); - } - request.__set_compressors(compressorsOrdinal); - - defaultSessionConnection_->createAlignedTimeseries(request); -} - -bool Session::checkTimeseriesExists(const string& path) { - try { - std::unique_ptr dataset = executeQueryStatement("SHOW TIMESERIES " + path); - bool isExisted = dataset->hasNext(); - dataset->closeOperationHandle(); - return isExisted; - } - catch (const exception& e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } -} - -shared_ptr Session::getQuerySessionConnection() { - auto endPoint = nodesSupplier_->getQueryEndPoint(); - if (!endPoint.is_initialized() || endPointToSessionConnection.empty()) { - return defaultSessionConnection_; - } - - auto it = endPointToSessionConnection.find(endPoint.value()); - if (it != endPointToSessionConnection.end()) { - return it->second; - } - - shared_ptr newConnection; - try { - newConnection = make_shared(this, endPoint.value(), zoneId_, nodesSupplier_, - fetchSize_, 60, 500, sqlDialect_, database_); - endPointToSessionConnection.emplace(endPoint.value(), newConnection); - return newConnection; - } - catch (exception& e) { - log_debug("Session::getQuerySessionConnection() exception: " + e.what()); - return newConnection; - } -} - -shared_ptr Session::getSessionConnection(std::string deviceId) { - if (!enableRedirection_ || - deviceIdToEndpoint.find(deviceId) == deviceIdToEndpoint.end() || - endPointToSessionConnection.find(deviceIdToEndpoint[deviceId]) == endPointToSessionConnection.end()) { - return defaultSessionConnection_; - } - return endPointToSessionConnection.find(deviceIdToEndpoint[deviceId])->second; -} - -shared_ptr Session::getSessionConnection(std::shared_ptr deviceId) { - if (!enableRedirection_ || - tableModelDeviceIdToEndpoint.find(deviceId) == tableModelDeviceIdToEndpoint.end() || - endPointToSessionConnection.find(tableModelDeviceIdToEndpoint[deviceId]) == endPointToSessionConnection.end()) { - return defaultSessionConnection_; - } - return endPointToSessionConnection.find(tableModelDeviceIdToEndpoint[deviceId])->second; -} - -string Session::getTimeZone() { - auto ret = defaultSessionConnection_->getTimeZone(); - return ret.timeZone; -} - -void Session::setTimeZone(const string& zoneId) { - TSSetTimeZoneReq req; - req.__set_sessionId(defaultSessionConnection_->sessionId); - req.__set_timeZone(zoneId); - defaultSessionConnection_->setTimeZone(req); -} - -unique_ptr Session::executeQueryStatement(const string& sql) { - return executeQueryStatementMayRedirect(sql, QUERY_TIMEOUT_MS); -} - -unique_ptr Session::executeQueryStatement(const string& sql, int64_t timeoutInMs) { - return executeQueryStatementMayRedirect(sql, timeoutInMs); -} - -void Session::handleQueryRedirection(TEndPoint endPoint) { - if (!enableRedirection_) return; - shared_ptr newConnection; - auto it = endPointToSessionConnection.find(endPoint); - if (it != endPointToSessionConnection.end()) { - newConnection = it->second; - } - else { - try { - newConnection = make_shared(this, endPoint, zoneId_, nodesSupplier_, - fetchSize_, 60, 500, sqlDialect_, database_); - - endPointToSessionConnection.emplace(endPoint, newConnection); - } - catch (exception& e) { - throw IoTDBConnectionException(e.what()); - } - } - defaultSessionConnection_ = newConnection; -} - -void Session::handleRedirection(const std::string& deviceId, TEndPoint endPoint) { - if (!enableRedirection_) return; - if (endPoint.ip == "127.0.0.1") return; - deviceIdToEndpoint[deviceId] = endPoint; - - shared_ptr newConnection; - auto it = endPointToSessionConnection.find(endPoint); - if (it != endPointToSessionConnection.end()) { - newConnection = it->second; - } - else { - try { - newConnection = make_shared(this, endPoint, zoneId_, nodesSupplier_, - fetchSize_, 60, 500, sqlDialect_, database_); - endPointToSessionConnection.emplace(endPoint, newConnection); - } - catch (exception& e) { - deviceIdToEndpoint.erase(deviceId); - throw IoTDBConnectionException(e.what()); - } - } -} - -void Session::handleRedirection(const std::shared_ptr& deviceId, TEndPoint endPoint) { - if (!enableRedirection_) return; - if (endPoint.ip == "127.0.0.1") return; - tableModelDeviceIdToEndpoint[deviceId] = endPoint; - - shared_ptr newConnection; - auto it = endPointToSessionConnection.find(endPoint); - if (it != endPointToSessionConnection.end()) { - newConnection = it->second; - } - else { - try { - newConnection = make_shared(this, endPoint, zoneId_, nodesSupplier_, - fetchSize_, 60, 500, sqlDialect_, database_); - endPointToSessionConnection.emplace(endPoint, newConnection); - } - catch (exception& e) { - tableModelDeviceIdToEndpoint.erase(deviceId); - throw IoTDBConnectionException(e.what()); - } - } -} - -std::unique_ptr Session::executeQueryStatementMayRedirect(const std::string& sql, int64_t timeoutInMs) { - auto sessionConnection = getQuerySessionConnection(); - if (!sessionConnection) { - log_warn("Session connection not found"); - return nullptr; - } - try { - return sessionConnection->executeQueryStatement(sql, timeoutInMs); - } - catch (RedirectException& e) { - log_warn("Session connection redirect exception: " + e.what()); - handleQueryRedirection(e.endPoint); - try { - return defaultSessionConnection_->executeQueryStatement(sql, timeoutInMs); - } - catch (exception& e) { - log_error("Exception while executing redirected query statement: %s", e.what()); - throw ExecutionException(e.what()); - } - } catch (exception& e) { - log_error("Exception while executing query statement: %s", e.what()); - throw e; - } -} - -void Session::executeNonQueryStatement(const string& sql) { - try { - defaultSessionConnection_->executeNonQueryStatement(sql); - } - catch (const exception& e) { - throw IoTDBException(e.what()); - } -} - -unique_ptr -Session::executeRawDataQuery(const vector& paths, int64_t startTime, int64_t endTime) { - return defaultSessionConnection_->executeRawDataQuery(paths, startTime, endTime); -} - - -unique_ptr Session::executeLastDataQuery(const vector& paths) { - return executeLastDataQuery(paths, LONG_LONG_MIN); -} - -unique_ptr Session::executeLastDataQuery(const vector& paths, int64_t lastTime) { - return defaultSessionConnection_->executeLastDataQuery(paths, lastTime); -} - -void Session::createSchemaTemplate(const Template& templ) { - TSCreateSchemaTemplateReq req; - req.__set_name(templ.getName()); - req.__set_serializedTemplate(templ.serialize()); - defaultSessionConnection_->createSchemaTemplate(req); -} - -void Session::setSchemaTemplate(const string& template_name, const string& prefix_path) { - TSSetSchemaTemplateReq req; - req.__set_templateName(template_name); - req.__set_prefixPath(prefix_path); - defaultSessionConnection_->setSchemaTemplate(req); -} - -void Session::unsetSchemaTemplate(const string& prefix_path, const string& template_name) { - TSUnsetSchemaTemplateReq req; - req.__set_templateName(template_name); - req.__set_prefixPath(prefix_path); - defaultSessionConnection_->unsetSchemaTemplate(req); -} - -void Session::addAlignedMeasurementsInTemplate(const string& template_name, const vector& measurements, - const vector& dataTypes, - const vector& encodings, - const vector& compressors) { - TSAppendSchemaTemplateReq req; - req.__set_name(template_name); - req.__set_measurements(measurements); - req.__set_isAligned(true); - - vector dataTypesOrdinal; - dataTypesOrdinal.reserve(dataTypes.size()); - for (TSDataType::TSDataType dataType : dataTypes) { - dataTypesOrdinal.push_back(dataType); - } - req.__set_dataTypes(dataTypesOrdinal); - - vector encodingsOrdinal; - encodingsOrdinal.reserve(encodings.size()); - for (TSEncoding::TSEncoding encoding : encodings) { - encodingsOrdinal.push_back(encoding); - } - req.__set_encodings(encodingsOrdinal); - - vector compressorsOrdinal; - compressorsOrdinal.reserve(compressors.size()); - for (CompressionType::CompressionType compressor : compressors) { - compressorsOrdinal.push_back(compressor); - } - req.__set_compressors(compressorsOrdinal); - - defaultSessionConnection_->appendSchemaTemplate(req); -} - -void Session::addAlignedMeasurementsInTemplate(const string& template_name, const string& measurement, - TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, - CompressionType::CompressionType compressor) { - vector measurements(1, measurement); - vector dataTypes(1, dataType); - vector encodings(1, encoding); - vector compressors(1, compressor); - addAlignedMeasurementsInTemplate(template_name, measurements, dataTypes, encodings, compressors); -} - -void Session::addUnalignedMeasurementsInTemplate(const string& template_name, const vector& measurements, - const vector& dataTypes, - const vector& encodings, - const vector& compressors) { - TSAppendSchemaTemplateReq req; - req.__set_name(template_name); - req.__set_measurements(measurements); - req.__set_isAligned(false); - - vector dataTypesOrdinal; - dataTypesOrdinal.reserve(dataTypes.size()); - for (TSDataType::TSDataType dataType : dataTypes) { - dataTypesOrdinal.push_back(dataType); - } - req.__set_dataTypes(dataTypesOrdinal); - - vector encodingsOrdinal; - encodingsOrdinal.reserve(encodings.size()); - for (TSEncoding::TSEncoding encoding : encodings) { - encodingsOrdinal.push_back(encoding); - } - req.__set_encodings(encodingsOrdinal); - - vector compressorsOrdinal; - compressorsOrdinal.reserve(compressors.size()); - for (CompressionType::CompressionType compressor : compressors) { - compressorsOrdinal.push_back(compressor); - } - req.__set_compressors(compressorsOrdinal); - - defaultSessionConnection_->appendSchemaTemplate(req); -} - -void Session::addUnalignedMeasurementsInTemplate(const string& template_name, const string& measurement, - TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, - CompressionType::CompressionType compressor) { - vector measurements(1, measurement); - vector dataTypes(1, dataType); - vector encodings(1, encoding); - vector compressors(1, compressor); - addUnalignedMeasurementsInTemplate(template_name, measurements, dataTypes, encodings, compressors); -} - -void Session::deleteNodeInTemplate(const string& template_name, const string& path) { - TSPruneSchemaTemplateReq req; - req.__set_name(template_name); - req.__set_path(path); - defaultSessionConnection_->pruneSchemaTemplate(req); -} - -int Session::countMeasurementsInTemplate(const string& template_name) { - TSQueryTemplateReq req; - req.__set_name(template_name); - req.__set_queryType(TemplateQueryType::COUNT_MEASUREMENTS); - TSQueryTemplateResp resp = defaultSessionConnection_->querySchemaTemplate(req); - return resp.count; -} - -bool Session::isMeasurementInTemplate(const string& template_name, const string& path) { - TSQueryTemplateReq req; - req.__set_name(template_name); - req.__set_measurement(path); - req.__set_queryType(TemplateQueryType::IS_MEASUREMENT); - TSQueryTemplateResp resp = defaultSessionConnection_->querySchemaTemplate(req); - return resp.result; -} - -bool Session::isPathExistInTemplate(const string& template_name, const string& path) { - TSQueryTemplateReq req; - req.__set_name(template_name); - req.__set_measurement(path); - req.__set_queryType(TemplateQueryType::PATH_EXIST); - TSQueryTemplateResp resp = defaultSessionConnection_->querySchemaTemplate(req); - return resp.result; -} - -std::vector Session::showMeasurementsInTemplate(const string& template_name) { - TSQueryTemplateReq req; - req.__set_name(template_name); - req.__set_measurement(""); - req.__set_queryType(TemplateQueryType::SHOW_MEASUREMENTS); - TSQueryTemplateResp resp = defaultSessionConnection_->querySchemaTemplate(req); - return resp.measurements; -} - -std::vector Session::showMeasurementsInTemplate(const string& template_name, const string& pattern) { - TSQueryTemplateReq req; - req.__set_name(template_name); - req.__set_measurement(pattern); - req.__set_queryType(TemplateQueryType::SHOW_MEASUREMENTS); - TSQueryTemplateResp resp = defaultSessionConnection_->querySchemaTemplate(req); - return resp.measurements; -} - -bool Session::checkTemplateExists(const string& template_name) { - try { - std::unique_ptr dataset = executeQueryStatement( - "SHOW NODES IN DEVICE TEMPLATE " + template_name); - bool isExisted = dataset->hasNext(); - dataset->closeOperationHandle(); - return isExisted; - } - catch (const exception& e) { - if (strstr(e.what(), "does not exist") != NULL) { - return false; - } - log_debug(e.what()); - throw IoTDBException(e.what()); - } -} diff --git a/iotdb-client/client-cpp/src/main/Session.h b/iotdb-client/client-cpp/src/main/Session.h deleted file mode 100644 index a1176e42bbe2b..0000000000000 --- a/iotdb-client/client-cpp/src/main/Session.h +++ /dev/null @@ -1,1014 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#ifndef IOTDB_SESSION_H -#define IOTDB_SESSION_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "IClientRPCService.h" -#include "NodesSupplier.h" -#include "AbstractSessionBuilder.h" -#include "SessionConnection.h" -#include "SessionDataSet.h" -#include "DeviceID.h" -#include "Common.h" - -//== For compatible with Windows OS == -#ifndef LONG_LONG_MIN -#define LONG_LONG_MIN 0x8000000000000000 -#endif - -using namespace std; - -using ::apache::thrift::protocol::TBinaryProtocol; -using ::apache::thrift::protocol::TCompactProtocol; -using ::apache::thrift::transport::TSocket; -using ::apache::thrift::transport::TTransport; -using ::apache::thrift::transport::TTransportException; -using ::apache::thrift::transport::TBufferedTransport; -using ::apache::thrift::transport::TFramedTransport; -using ::apache::thrift::TException; - - -template -void safe_cast(const T& value, Target& target) { - /* - Target Allowed Source Types - BOOLEAN BOOLEAN - INT32 INT32 - INT64 INT32 INT64 - FLOAT INT32 FLOAT - DOUBLE INT32 INT64 FLOAT DOUBLE - TEXT TEXT - */ - if (std::is_same::value) { - target = *(Target*)&value; - } - else if (std::is_same::value && std::is_array::value && std::is_same< - char, typename std::remove_extent::type>::value) { - string tmp((const char*)&value); - target = *(Target*)&tmp; - } - else if (std::is_same::value && std::is_same::value) { - int64_t tmp = *(int32_t*)&value; - target = *(Target*)&tmp; - } - else if (std::is_same::value && std::is_same::value) { - float tmp = *(int32_t*)&value; - target = *(Target*)&tmp; - } - else if (std::is_same::value && std::is_same::value) { - double tmp = *(int32_t*)&value; - target = *(Target*)&tmp; - } - else if (std::is_same::value && std::is_same::value) { - double tmp = *(int64_t*)&value; - target = *(Target*)&tmp; - } - else if (std::is_same::value && std::is_same::value) { - double tmp = *(float*)&value; - target = *(Target*)&tmp; - } - else { - throw UnSupportedDataTypeException("Error: Parameter type " + - std::string(typeid(T).name()) + " cannot be converted to DataType" + - std::string(typeid(Target).name())); - } -} - -/* - * A tablet data of one device, the tablet contains multiple measurements of this device that share - * the same time column. - * - * for example: device root.sg1.d1 - * - * time, m1, m2, m3 - * 1, 1, 2, 3 - * 2, 1, 2, 3 - * 3, 1, 2, 3 - * - * Notice: The tablet should not have empty cell - * - */ -class Tablet { -private: - static const int DEFAULT_ROW_SIZE = 1024; - - void createColumns(); - void deleteColumns(); - -public: - std::string deviceId; // deviceId of this tablet - std::vector> schemas; - // the list of measurement schemas for creating the tablet - std::map schemaNameIndex; // the map of schema name to index - std::vector columnTypes; // the list of column types (used in table model) - std::vector timestamps; // timestamps in this tablet - std::vector values; // each object is a primitive type array, which represents values of one measurement - std::vector bitMaps; // each bitmap represents the existence of each value in the current column - size_t rowSize; //the number of rows to include in this tablet - size_t maxRowNumber; // the maximum number of rows for this tablet - bool isAligned; // whether this tablet store data of aligned timeseries or not - std::vector idColumnIndexes; - - Tablet() = default; - - /** - * Return a tablet with default specified row number. This is the standard - * constructor (all Tablet should be the same size). - * - * @param deviceId the name of the device specified to be written in - * @param timeseries the list of measurement schemas for creating the tablet - */ - Tablet(const std::string& deviceId, - const std::vector>& timeseries) - : Tablet(deviceId, timeseries, DEFAULT_ROW_SIZE) { - } - - Tablet(const std::string& deviceId, - const std::vector>& timeseries, - const std::vector& columnTypes) - : Tablet(deviceId, timeseries, columnTypes, DEFAULT_ROW_SIZE) { - } - - /** - * Return a tablet with the specified number of rows (maxBatchSize). Only - * call this constructor directly for testing purposes. Tablet should normally - * always be default size. - * - * @param deviceId the name of the device specified to be written in - * @param schemas the list of measurement schemas for creating the row - * batch - * @param columnTypes the list of column types (used in table model) - * @param maxRowNumber the maximum number of rows for this tablet - */ - Tablet(const std::string& deviceId, - const std::vector>& schemas, - int maxRowNumber) - : Tablet(deviceId, schemas, std::vector(schemas.size(), ColumnCategory::FIELD), maxRowNumber) { - } - - Tablet(const std::string& deviceId, const std::vector>& schemas, - const std::vector columnTypes, - size_t maxRowNumber, bool _isAligned = false) : deviceId(deviceId), schemas(schemas), - columnTypes(columnTypes), - maxRowNumber(maxRowNumber), isAligned(_isAligned) { - // create timestamp column - timestamps.resize(maxRowNumber); - // create value columns - values.resize(schemas.size()); - createColumns(); - // init idColumnIndexs - for (size_t i = 0; i < this->columnTypes.size(); i++) { - if (this->columnTypes[i] == ColumnCategory::TAG) { - idColumnIndexes.push_back(i); - } - } - // create bitMaps - bitMaps.resize(schemas.size()); - for (size_t i = 0; i < schemas.size(); i++) { - bitMaps[i].resize(maxRowNumber); - } - // create schemaNameIndex - for (size_t i = 0; i < schemas.size(); i++) { - schemaNameIndex[schemas[i].first] = i; - } - this->rowSize = 0; - } - - Tablet(const Tablet& other) - : deviceId(other.deviceId), - schemas(other.schemas), - schemaNameIndex(other.schemaNameIndex), - columnTypes(other.columnTypes), - timestamps(other.timestamps), - maxRowNumber(other.maxRowNumber), - bitMaps(other.bitMaps), - rowSize(other.rowSize), - isAligned(other.isAligned), - idColumnIndexes(other.idColumnIndexes) { - values.resize(other.values.size()); - for (size_t i = 0; i < other.values.size(); ++i) { - if (!other.values[i]) continue; - TSDataType::TSDataType type = schemas[i].second; - deepCopyTabletColValue(&(other.values[i]), &values[i], type, maxRowNumber); - } - } - - Tablet& operator=(const Tablet& other) { - if (this != &other) { - deleteColumns(); - deviceId = other.deviceId; - schemas = other.schemas; - schemaNameIndex = other.schemaNameIndex; - columnTypes = other.columnTypes; - timestamps = other.timestamps; - maxRowNumber = other.maxRowNumber; - rowSize = other.rowSize; - isAligned = other.isAligned; - idColumnIndexes = other.idColumnIndexes; - bitMaps = other.bitMaps; - values.resize(other.values.size()); - for (size_t i = 0; i < other.values.size(); ++i) { - if (!other.values[i]) continue; - TSDataType::TSDataType type = schemas[i].second; - deepCopyTabletColValue(&(other.values[i]), &values[i], type, maxRowNumber); - } - } - return *this; - } - - ~Tablet() { - try { - deleteColumns(); - } - catch (exception& e) { - log_debug(string("Tablet::~Tablet(), ") + e.what()); - } - } - - void addTimestamp(size_t rowIndex, int64_t timestamp) { - timestamps[rowIndex] = timestamp; - rowSize = max(rowSize, rowIndex + 1); - } - - static void deepCopyTabletColValue(void* const* srcPtr, void** destPtr, - TSDataType::TSDataType type, int maxRowNumber); - - template - void addValue(size_t schemaId, size_t rowIndex, const T& value) { - if (schemaId >= schemas.size()) { - char tmpStr[100]; - sprintf(tmpStr, "Tablet::addValue(), schemaId >= schemas.size(). schemaId=%ld, schemas.size()=%ld.", - schemaId, schemas.size()); - throw std::out_of_range(tmpStr); - } - - if (rowIndex >= rowSize) { - char tmpStr[100]; - sprintf(tmpStr, "Tablet::addValue(), rowIndex >= rowSize. rowIndex=%ld, rowSize.size()=%ld.", rowIndex, - rowSize); - throw std::out_of_range(tmpStr); - } - - TSDataType::TSDataType dataType = schemas[schemaId].second; - switch (dataType) { - case TSDataType::BOOLEAN: { - safe_cast(value, ((bool*)values[schemaId])[rowIndex]); - break; - } - case TSDataType::INT32: { - safe_cast(value, ((int*)values[schemaId])[rowIndex]); - break; - } - case TSDataType::DATE: { - safe_cast(value, ((boost::gregorian::date*)values[schemaId])[rowIndex]); - break; - } - case TSDataType::TIMESTAMP: - case TSDataType::INT64: { - safe_cast(value, ((int64_t*)values[schemaId])[rowIndex]); - break; - } - case TSDataType::FLOAT: { - safe_cast(value, ((float*)values[schemaId])[rowIndex]); - break; - } - case TSDataType::DOUBLE: { - safe_cast(value, ((double*)values[schemaId])[rowIndex]); - break; - } - case TSDataType::BLOB: - case TSDataType::STRING: - case TSDataType::TEXT: { - safe_cast(value, ((string*)values[schemaId])[rowIndex]); - break; - } - default: - throw UnSupportedDataTypeException(string("Data type ") + to_string(dataType) + " is not supported."); - } - } - - template - void addValue(const string& schemaName, size_t rowIndex, const T& value) { - if (schemaNameIndex.find(schemaName) == schemaNameIndex.end()) { - throw SchemaNotFoundException(string("Schema ") + schemaName + " not found."); - } - size_t schemaId = schemaNameIndex[schemaName]; - addValue(schemaId, rowIndex, value); - } - - - void* getValue(size_t schemaId, size_t rowIndex, TSDataType::TSDataType dataType) { - if (schemaId >= schemas.size()) { - throw std::out_of_range("Tablet::getValue schemaId out of range: " - + std::to_string(schemaId)); - } - if (rowIndex >= rowSize) { - throw std::out_of_range("Tablet::getValue rowIndex out of range: " - + std::to_string(rowIndex)); - } - - switch (dataType) { - case TSDataType::BOOLEAN: - return &(reinterpret_cast(values[schemaId])[rowIndex]); - case TSDataType::INT32: - return &(reinterpret_cast(values[schemaId])[rowIndex]); - case TSDataType::DATE: - return &(reinterpret_cast(values[schemaId])[rowIndex]); - case TSDataType::TIMESTAMP: - case TSDataType::INT64: - return &(reinterpret_cast(values[schemaId])[rowIndex]); - case TSDataType::FLOAT: - return &(reinterpret_cast(values[schemaId])[rowIndex]); - case TSDataType::DOUBLE: - return &(reinterpret_cast(values[schemaId])[rowIndex]); - case TSDataType::BLOB: - case TSDataType::STRING: - case TSDataType::TEXT: - return &(reinterpret_cast(values[schemaId])[rowIndex]); - default: - throw UnSupportedDataTypeException("Unsupported data type: " - + std::to_string(dataType)); - } - } - - std::shared_ptr getDeviceID(int i); - - std::vector> getSchemas() const { - return schemas; - } - - void reset(); // Reset Tablet to the default state - set the rowSize to 0 - - size_t getTimeBytesSize(); - - size_t getValueByteSize(); // total byte size that values occupies - - void setAligned(bool isAligned); -}; - -class SessionUtils { -public: - static std::string getTime(const Tablet& tablet); - - static std::string getValue(const Tablet& tablet); - - static bool isTabletContainsSingleDevice(Tablet tablet); -}; - -class TemplateNode { -public: - explicit TemplateNode(const std::string& name) : name_(name) { - } - - const std::string& getName() const { - return name_; - } - - virtual const std::unordered_map>& getChildren() const { - throw BatchExecutionException("Should call exact sub class!"); - } - - virtual bool isMeasurement() = 0; - - virtual bool isAligned() { - throw BatchExecutionException("Should call exact sub class!"); - } - - virtual std::string serialize() const { - throw BatchExecutionException("Should call exact sub class!"); - } - -private: - std::string name_; -}; - -class MeasurementNode : public TemplateNode { -public: - MeasurementNode(const std::string& name_, TSDataType::TSDataType data_type_, TSEncoding::TSEncoding encoding_, - CompressionType::CompressionType compression_type_) : TemplateNode(name_) { - this->data_type_ = data_type_; - this->encoding_ = encoding_; - this->compression_type_ = compression_type_; - } - - TSDataType::TSDataType getDataType() const { - return data_type_; - } - - TSEncoding::TSEncoding getEncoding() const { - return encoding_; - } - - CompressionType::CompressionType getCompressionType() const { - return compression_type_; - } - - bool isMeasurement() override { - return true; - } - - std::string serialize() const override; - -private: - TSDataType::TSDataType data_type_; - TSEncoding::TSEncoding encoding_; - CompressionType::CompressionType compression_type_; -}; - -class InternalNode : public TemplateNode { -public: - InternalNode(const std::string& name, bool is_aligned) : TemplateNode(name), is_aligned_(is_aligned) { - } - - void addChild(const InternalNode& node) { - if (this->children_.count(node.getName())) { - throw BatchExecutionException("Duplicated child of node in template."); - } - this->children_[node.getName()] = std::make_shared(node); - } - - void addChild(const MeasurementNode& node) { - if (this->children_.count(node.getName())) { - throw BatchExecutionException("Duplicated child of node in template."); - } - this->children_[node.getName()] = std::make_shared(node); - } - - void deleteChild(const TemplateNode& node) { - this->children_.erase(node.getName()); - } - - const std::unordered_map>& getChildren() const override { - return children_; - } - - bool isMeasurement() override { - return false; - } - - bool isAligned() override { - return is_aligned_; - } - -private: - std::unordered_map> children_; - bool is_aligned_; -}; - -namespace TemplateQueryType { -enum TemplateQueryType { - COUNT_MEASUREMENTS, IS_MEASUREMENT, PATH_EXIST, SHOW_MEASUREMENTS -}; -} - -class Template { -public: - Template(const std::string& name, bool is_aligned) : name_(name), is_aligned_(is_aligned) { - } - - const std::string& getName() const { - return name_; - } - - bool isAligned() const { - return is_aligned_; - } - - void addToTemplate(const InternalNode& child) { - if (this->children_.count(child.getName())) { - throw BatchExecutionException("Duplicated child of node in template."); - } - this->children_[child.getName()] = std::make_shared(child); - } - - void addToTemplate(const MeasurementNode& child) { - if (this->children_.count(child.getName())) { - throw BatchExecutionException("Duplicated child of node in template."); - } - this->children_[child.getName()] = std::make_shared(child); - } - - std::string serialize() const; - -private: - std::string name_; - std::unordered_map> children_; - bool is_aligned_; -}; - -class Session { -private: - std::string host_; - int rpcPort_; - std::string username_; - std::string password_; - const TSProtocolVersion::type protocolVersion_ = TSProtocolVersion::IOTDB_SERVICE_PROTOCOL_V3; - bool isClosed_ = true; - std::string zoneId_; - int fetchSize_; - const static int DEFAULT_FETCH_SIZE = 10000; - const static int DEFAULT_TIMEOUT_MS = 0; - Version::Version version; - std::string sqlDialect_ = "tree"; // default sql dialect - std::string database_; - bool enableAutoFetch_ = true; - bool enableRedirection_ = true; - std::shared_ptr nodesSupplier_; - friend class SessionConnection; - friend class TableSession; - std::shared_ptr defaultSessionConnection_; - - TEndPoint defaultEndPoint_; - - struct TEndPointHash { - size_t operator()(const TEndPoint& endpoint) const { - return std::hash()(endpoint.ip) ^ std::hash()(endpoint.port); - } - }; - - struct TEndPointEqual { - bool operator()(const TEndPoint& lhs, const TEndPoint& rhs) const { - return lhs.ip == rhs.ip && lhs.port == rhs.port; - } - }; - - using EndPointSessionMap = std::unordered_map< - TEndPoint, shared_ptr, TEndPointHash, TEndPointEqual>; - EndPointSessionMap endPointToSessionConnection; - std::unordered_map deviceIdToEndpoint; - std::unordered_map, TEndPoint> tableModelDeviceIdToEndpoint; - -private: - void removeBrokenSessionConnection(shared_ptr sessionConnection); - - static bool checkSorted(const Tablet& tablet); - - static bool checkSorted(const std::vector& times); - - static void sortTablet(Tablet& tablet); - - static void sortIndexByTimestamp(int* index, std::vector& timestamps, int length); - - void appendValues(std::string& buffer, const char* value, int size); - - void - putValuesIntoBuffer(const std::vector& types, const std::vector& values, - std::string& buf); - - int8_t getDataTypeNumber(TSDataType::TSDataType type); - - struct TsCompare { - std::vector& timestamps; - - explicit TsCompare(std::vector& inTimestamps) : timestamps(inTimestamps) { - }; - - bool operator()(int i, int j) { return (timestamps[i] < timestamps[j]); }; - }; - - std::string getVersionString(Version::Version version); - - void initZoneId(); - - void initNodesSupplier(); - - void initDefaultSessionConnection(); - - template - void insertByGroup(std::unordered_map, T>& insertGroup, - InsertConsumer insertConsumer); - - template - void insertOnce(std::unordered_map, T>& insertGroup, - InsertConsumer insertConsumer); - - void insertStringRecordsWithLeaderCache(vector deviceIds, vector times, - vector> measurementsList, vector> valuesList, - bool isAligned); - - void insertRecordsWithLeaderCache(vector deviceIds, vector times, - vector> measurementsList, - const vector>& typesList, - vector> valuesList, bool isAligned); - - void insertTabletsWithLeaderCache(unordered_map tablets, bool sorted, bool isAligned); - - shared_ptr getQuerySessionConnection(); - - shared_ptr getSessionConnection(std::string deviceId); - - shared_ptr getSessionConnection(std::shared_ptr deviceId); - - void handleQueryRedirection(TEndPoint endPoint); - - void handleRedirection(const std::string& deviceId, TEndPoint endPoint); - - void handleRedirection(const std::shared_ptr& deviceId, TEndPoint endPoint); - - void setSqlDialect(const std::string& dialect) { - this->sqlDialect_ = dialect; - } - - void setDatabase(const std::string& database) { - this->database_ = database; - } - - string getDatabase() { - return database_; - } - - void changeDatabase(string database) { - this->database_ = database; - } - -public: - Session(const std::string& host, int rpcPort) : username_("root"), password_("root"), version(Version::V_1_0) { - this->host_ = host; - this->rpcPort_ = rpcPort; - initZoneId(); - initNodesSupplier(); - } - - Session(const std::string& host, int rpcPort, const std::string& username, const std::string& password) - : fetchSize_(DEFAULT_FETCH_SIZE) { - this->host_ = host; - this->rpcPort_ = rpcPort; - this->username_ = username; - this->password_ = password; - this->version = Version::V_1_0; - initZoneId(); - initNodesSupplier(); - } - - Session(const std::string& host, int rpcPort, const std::string& username, const std::string& password, - const std::string& zoneId, int fetchSize = DEFAULT_FETCH_SIZE) { - this->host_ = host; - this->rpcPort_ = rpcPort; - this->username_ = username; - this->password_ = password; - this->zoneId_ = zoneId; - this->fetchSize_ = fetchSize; - this->version = Version::V_1_0; - initZoneId(); - initNodesSupplier(); - } - - Session(const std::string& host, const std::string& rpcPort, const std::string& username = "user", - const std::string& password = "password", const std::string& zoneId = "", - int fetchSize = DEFAULT_FETCH_SIZE) { - this->host_ = host; - this->rpcPort_ = stoi(rpcPort); - this->username_ = username; - this->password_ = password; - this->zoneId_ = zoneId; - this->fetchSize_ = fetchSize; - this->version = Version::V_1_0; - initZoneId(); - initNodesSupplier(); - } - - Session(AbstractSessionBuilder* builder) { - this->host_ = builder->host; - this->rpcPort_ = builder->rpcPort; - this->username_ = builder->username; - this->password_ = builder->password; - this->zoneId_ = builder->zoneId; - this->fetchSize_ = builder->fetchSize; - this->version = Version::V_1_0; - this->sqlDialect_ = builder->sqlDialect; - this->database_ = builder->database; - this->enableAutoFetch_ = builder->enableAutoFetch; - this->enableRedirection_ = builder->enableRedirections; - initZoneId(); - initNodesSupplier(); - } - - ~Session(); - - void open(); - - void open(bool enableRPCCompression); - - void open(bool enableRPCCompression, int connectionTimeoutInMs); - - void close(); - - void setTimeZone(const std::string& zoneId); - - std::string getTimeZone(); - - void insertRecord(const std::string& deviceId, int64_t time, const std::vector& measurements, - const std::vector& values); - - void insertRecord(const std::string& deviceId, int64_t time, const std::vector& measurements, - const std::vector& types, const std::vector& values); - - void insertAlignedRecord(const std::string& deviceId, int64_t time, const std::vector& measurements, - const std::vector& values); - - void insertAlignedRecord(const std::string& deviceId, int64_t time, const std::vector& measurements, - const std::vector& types, const std::vector& values); - - void insertRecords(const std::vector& deviceIds, - const std::vector& times, - const std::vector>& measurementsList, - const std::vector>& valuesList); - - void insertRecords(const std::vector& deviceIds, - const std::vector& times, - const std::vector>& measurementsList, - const std::vector>& typesList, - const std::vector>& valuesList); - - void insertAlignedRecords(const std::vector& deviceIds, - const std::vector& times, - const std::vector>& measurementsList, - const std::vector>& valuesList); - - void insertAlignedRecords(const std::vector& deviceIds, - const std::vector& times, - const std::vector>& measurementsList, - const std::vector>& typesList, - const std::vector>& valuesList); - - void insertRecordsOfOneDevice(const std::string& deviceId, - std::vector& times, - std::vector>& measurementsList, - std::vector>& typesList, - std::vector>& valuesList); - - void insertRecordsOfOneDevice(const std::string& deviceId, - std::vector& times, - std::vector>& measurementsList, - std::vector>& typesList, - std::vector>& valuesList, - bool sorted); - - void insertAlignedRecordsOfOneDevice(const std::string& deviceId, - std::vector& times, - std::vector>& measurementsList, - std::vector>& typesList, - std::vector>& valuesList); - - void insertAlignedRecordsOfOneDevice(const std::string& deviceId, - std::vector& times, - std::vector>& measurementsList, - std::vector>& typesList, - std::vector>& valuesList, - bool sorted); - - void insertTablet(Tablet& tablet); - - void insertTablet(Tablet& tablet, bool sorted); - - static void buildInsertTabletReq(TSInsertTabletReq& request, Tablet& tablet, bool sorted); - - void insertTablet(TSInsertTabletReq request); - - void insertAlignedTablet(Tablet& tablet); - - void insertAlignedTablet(Tablet& tablet, bool sorted); - - void insertTablets(std::unordered_map& tablets); - - void insertTablets(std::unordered_map& tablets, bool sorted); - - void insertAlignedTablets(std::unordered_map& tablets, bool sorted = false); - - void testInsertRecord(const std::string& deviceId, int64_t time, - const std::vector& measurements, - const std::vector& values); - - void testInsertTablet(const Tablet& tablet); - - void testInsertRecords(const std::vector& deviceIds, - const std::vector& times, - const std::vector>& measurementsList, - const std::vector>& valuesList); - - void deleteTimeseries(const std::string& path); - - void deleteTimeseries(const std::vector& paths); - - void deleteData(const std::string& path, int64_t endTime); - - void deleteData(const std::vector& paths, int64_t endTime); - - void deleteData(const std::vector& paths, int64_t startTime, int64_t endTime); - - void setStorageGroup(const std::string& storageGroupId); - - void deleteStorageGroup(const std::string& storageGroup); - - void deleteStorageGroups(const std::vector& storageGroups); - - void createDatabase(const std::string& database); - - void deleteDatabase(const std::string& database); - - void deleteDatabases(const std::vector& databases); - - void createTimeseries(const std::string& path, TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, - CompressionType::CompressionType compressor); - - void createTimeseries(const std::string& path, TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, - CompressionType::CompressionType compressor, - std::map* props, std::map* tags, - std::map* attributes, - const std::string& measurementAlias); - - void createMultiTimeseries(const std::vector& paths, - const std::vector& dataTypes, - const std::vector& encodings, - const std::vector& compressors, - std::vector>* propsList, - std::vector>* tagsList, - std::vector>* attributesList, - std::vector* measurementAliasList); - - void createAlignedTimeseries(const std::string& deviceId, - const std::vector& measurements, - const std::vector& dataTypes, - const std::vector& encodings, - const std::vector& compressors); - - bool checkTimeseriesExists(const std::string& path); - - std::unique_ptr executeQueryStatement(const std::string& sql); - - std::unique_ptr executeQueryStatement(const std::string& sql, int64_t timeoutInMs); - - std::unique_ptr executeQueryStatementMayRedirect(const std::string& sql, int64_t timeoutInMs); - - void executeNonQueryStatement(const std::string& sql); - - std::unique_ptr executeRawDataQuery(const std::vector& paths, int64_t startTime, - int64_t endTime); - - std::unique_ptr executeLastDataQuery(const std::vector& paths); - - std::unique_ptr executeLastDataQuery(const std::vector& paths, int64_t lastTime); - - void createSchemaTemplate(const Template& templ); - - void setSchemaTemplate(const std::string& template_name, const std::string& prefix_path); - - void unsetSchemaTemplate(const std::string& prefix_path, const std::string& template_name); - - void addAlignedMeasurementsInTemplate(const std::string& template_name, - const std::vector& measurements, - const std::vector& dataTypes, - const std::vector& encodings, - const std::vector& compressors); - - void addAlignedMeasurementsInTemplate(const std::string& template_name, - const std::string& measurement, - TSDataType::TSDataType dataType, - TSEncoding::TSEncoding encoding, - CompressionType::CompressionType compressor); - - void addUnalignedMeasurementsInTemplate(const std::string& template_name, - const std::vector& measurements, - const std::vector& dataTypes, - const std::vector& encodings, - const std::vector& compressors); - - void addUnalignedMeasurementsInTemplate(const std::string& template_name, - const std::string& measurement, - TSDataType::TSDataType dataType, - TSEncoding::TSEncoding encoding, - CompressionType::CompressionType compressor); - - void deleteNodeInTemplate(const std::string& template_name, const std::string& path); - - int countMeasurementsInTemplate(const std::string& template_name); - - bool isMeasurementInTemplate(const std::string& template_name, const std::string& path); - - bool isPathExistInTemplate(const std::string& template_name, const std::string& path); - - std::vector showMeasurementsInTemplate(const std::string& template_name); - - std::vector showMeasurementsInTemplate(const std::string& template_name, const std::string& pattern); - - bool checkTemplateExists(const std::string& template_name); -}; - -template -void Session::insertByGroup(std::unordered_map, T>& insertGroup, - InsertConsumer insertConsumer) { - std::vector> futures; - - for (auto& entry : insertGroup) { - auto connection = entry.first; - auto& req = entry.second; - futures.emplace_back(std::async(std::launch::async, [=, &req]() mutable { - try { - insertConsumer(connection, req); - } - catch (const RedirectException& e) { - for (const auto& deviceEndPoint : e.deviceEndPointMap) { - handleRedirection(deviceEndPoint.first, deviceEndPoint.second); - } - } catch (const IoTDBConnectionException& e) { - if (endPointToSessionConnection.size() > 1) { - removeBrokenSessionConnection(connection); - try { - insertConsumer(defaultSessionConnection_, req); - } - catch (const RedirectException&) { - } - } - else { - throw; - } - } catch (const std::exception& e) { - log_debug(e.what()); - throw IoTDBException(e.what()); - } - })); - } - - std::string errorMessages; - for (auto& f : futures) { - try { - f.get(); - } - catch (const IoTDBConnectionException& e) { - throw; - } catch (const std::exception& e) { - if (!errorMessages.empty()) { - errorMessages += ";"; - } - errorMessages += e.what(); - } - } - - if (!errorMessages.empty()) { - throw StatementExecutionException(errorMessages); - } -} - -template -void Session::insertOnce(std::unordered_map, T>& insertGroup, - InsertConsumer insertConsumer) { - auto connection = insertGroup.begin()->first; - auto req = insertGroup.begin()->second; - try { - insertConsumer(connection, req); - } - catch (RedirectException e) { - for (const auto& deviceEndPoint : e.deviceEndPointMap) { - handleRedirection(deviceEndPoint.first, deviceEndPoint.second); - } - } catch (IoTDBConnectionException e) { - if (endPointToSessionConnection.size() > 1) { - removeBrokenSessionConnection(connection); - try { - insertConsumer(defaultSessionConnection_, req); - } - catch (RedirectException e) { - } - } - else { - throw e; - } - } -} - -#endif // IOTDB_SESSION_H diff --git a/iotdb-client/client-cpp/src/main/SessionConnection.cpp b/iotdb-client/client-cpp/src/main/SessionConnection.cpp deleted file mode 100644 index 14032ae24c04a..0000000000000 --- a/iotdb-client/client-cpp/src/main/SessionConnection.cpp +++ /dev/null @@ -1,673 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#include "SessionConnection.h" -#include "Session.h" -#include "common_types.h" -#include - -#include - -#include "SessionDataSet.h" - -using namespace apache::thrift; -using namespace apache::thrift::protocol; -using namespace apache::thrift::transport; - -SessionConnection::SessionConnection(Session* session_ptr, const TEndPoint& endpoint, - const std::string& zoneId, - std::shared_ptr nodeSupplier, - int fetchSize, - int maxRetries, - int64_t retryInterval, - std::string dialect, - std::string db) - : session(session_ptr), - zoneId(zoneId), - endPoint(endpoint), - availableNodes(std::move(nodeSupplier)), - fetchSize(fetchSize), - maxRetryCount(maxRetries), - retryIntervalMs(retryInterval), - sqlDialect(std::move(dialect)), - database(std::move(db)) { - this->zoneId = zoneId.empty() ? getSystemDefaultZoneId() : zoneId; - endPointList.push_back(endpoint); - init(endPoint); -} - -void SessionConnection::close() { - bool needThrowException = false; - string errMsg; - session = nullptr; - try { - TSCloseSessionReq req; - req.__set_sessionId(sessionId); - TSStatus tsStatus; - client->closeSession(tsStatus, req); - } - catch (const TTransportException& e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const exception& e) { - log_debug(e.what()); - errMsg = errMsg + "Session::close() client->closeSession() error, maybe remote server is down. " + e.what() + - "\n"; - needThrowException = true; - } - - try { - if (transport->isOpen()) { - transport->close(); - } - } - catch (const exception& e) { - log_debug(e.what()); - errMsg = errMsg + "Session::close() transport->close() error. " + e.what() + "\n"; - needThrowException = true; - } - - if (needThrowException) { - throw IoTDBException(errMsg); - } -} - -SessionConnection::~SessionConnection() { - try { - close(); - } - catch (const exception& e) { - log_debug(e.what()); - } -} - -void SessionConnection::init(const TEndPoint& endpoint) { - shared_ptr socket(new TSocket(endpoint.ip, endpoint.port)); - transport = std::make_shared(socket); - socket->setConnTimeout(connectionTimeoutInMs); - if (!transport->isOpen()) { - try { - transport->open(); - } - catch (TTransportException& e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } - } - if (enableRPCCompression) { - shared_ptr protocol(new TCompactProtocol(transport)); - client = std::make_shared(protocol); - } - else { - shared_ptr protocol(new TBinaryProtocol(transport)); - client = std::make_shared(protocol); - } - - std::map configuration; - configuration["version"] = session->getVersionString(session->version); - configuration["sql_dialect"] = sqlDialect; - if (database != "") { - configuration["db"] = database; - } - TSOpenSessionReq openReq; - openReq.__set_username(session->username_); - openReq.__set_password(session->password_); - openReq.__set_zoneId(zoneId); - openReq.__set_configuration(configuration); - try { - TSOpenSessionResp openResp; - client->openSession(openResp, openReq); - RpcUtils::verifySuccess(openResp.status); - if (session->protocolVersion_ != openResp.serverProtocolVersion) { - if (openResp.serverProtocolVersion == 0) { - // less than 0.10 - throw logic_error(string("Protocol not supported, Client version is ") + - to_string(session->protocolVersion_) + - ", but Server version is " + to_string(openResp.serverProtocolVersion)); - } - } - - sessionId = openResp.sessionId; - statementId = client->requestStatementId(sessionId); - - if (!zoneId.empty()) { - setTimeZone(zoneId); - } - } - catch (const TTransportException& e) { - log_debug(e.what()); - transport->close(); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException& e) { - log_debug(e.what()); - transport->close(); - throw; - } catch (const exception& e) { - log_debug(e.what()); - transport->close(); - throw; - } -} - -std::unique_ptr SessionConnection::executeQueryStatement(const std::string& sql, int64_t timeoutInMs) { - TSExecuteStatementReq req; - req.__set_sessionId(sessionId); - req.__set_statementId(statementId); - req.__set_statement(sql); - req.__set_timeout(timeoutInMs); - req.__set_enableRedirectQuery(true); - - auto result = callWithRetryAndReconnect( - [this, &req]() { - TSExecuteStatementResp resp; - client->executeQueryStatementV2(resp, req); - return resp; - }, - [](const TSExecuteStatementResp& resp) { - return resp.status; - } - ); - TSExecuteStatementResp resp = result.getResult(); - if (result.getRetryAttempts() == 0) { - RpcUtils::verifySuccessWithRedirection(resp.status); - } - else { - RpcUtils::verifySuccess(resp.status); - } - - return std::unique_ptr(new SessionDataSet(sql, resp.columns, resp.dataTypeList, - resp.columnNameIndexMap, resp.queryId, statementId, - client, sessionId, resp.queryResult, resp.ignoreTimeStamp, - timeoutInMs, resp.moreData, fetchSize, zoneId)); -} - -std::unique_ptr SessionConnection::executeRawDataQuery(const std::vector& paths, - int64_t startTime, int64_t endTime) { - TSRawDataQueryReq req; - req.__set_sessionId(sessionId); - req.__set_statementId(statementId); - req.__set_fetchSize(fetchSize); - req.__set_paths(paths); - req.__set_startTime(startTime); - req.__set_endTime(endTime); - auto result = callWithRetryAndReconnect( - [this, &req]() { - TSExecuteStatementResp resp; - client->executeRawDataQueryV2(resp, req); - return resp; - }, - [](const TSExecuteStatementResp& resp) { - return resp.status; - } - ); - TSExecuteStatementResp resp = result.getResult(); - if (result.getRetryAttempts() == 0) { - RpcUtils::verifySuccessWithRedirection(resp.status); - } - else { - RpcUtils::verifySuccess(resp.status); - } - return std::unique_ptr(new SessionDataSet("", resp.columns, resp.dataTypeList, - resp.columnNameIndexMap, resp.queryId, statementId, - client, sessionId, resp.queryResult, resp.ignoreTimeStamp, - connectionTimeoutInMs, resp.moreData, fetchSize, zoneId)); -} - -std::unique_ptr SessionConnection::executeLastDataQuery(const std::vector& paths, - int64_t lastTime) { - TSLastDataQueryReq req; - req.__set_sessionId(sessionId); - req.__set_statementId(statementId); - req.__set_fetchSize(fetchSize); - req.__set_paths(paths); - req.__set_time(lastTime); - - auto result = callWithRetryAndReconnect( - [this, &req]() { - TSExecuteStatementResp resp; - client->executeLastDataQuery(resp, req); - return resp; - }, - [](const TSExecuteStatementResp& resp) { - return resp.status; - } - ); - TSExecuteStatementResp resp = result.getResult(); - if (result.getRetryAttempts() == 0) { - RpcUtils::verifySuccessWithRedirection(resp.status); - } - else { - RpcUtils::verifySuccess(resp.status); - } - return std::unique_ptr(new SessionDataSet("", resp.columns, resp.dataTypeList, - resp.columnNameIndexMap, resp.queryId, statementId, - client, sessionId, resp.queryResult, resp.ignoreTimeStamp, - connectionTimeoutInMs, resp.moreData, fetchSize, zoneId)); -} - -void SessionConnection::executeNonQueryStatement(const string& sql) { - TSExecuteStatementReq req; - req.__set_sessionId(sessionId); - req.__set_statementId(statementId); - req.__set_statement(sql); - req.__set_timeout(0); //0 means no timeout. This value keep consistent to JAVA SDK. - TSExecuteStatementResp resp; - try { - client->executeUpdateStatementV2(resp, req); - RpcUtils::verifySuccess(resp.status); - } - catch (const TTransportException& e) { - log_debug(e.what()); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException& e) { - log_debug(e.what()); - throw; - } catch (const exception& e) { - throw IoTDBException(e.what()); - } -} - -const TEndPoint& SessionConnection::getEndPoint() { - return endPoint; -} - -void SessionConnection::setTimeZone(const std::string& newZoneId) { - TSSetTimeZoneReq req; - req.__set_sessionId(sessionId); - req.__set_timeZone(newZoneId); - - try { - TSStatus tsStatus; - client->setTimeZone(tsStatus, req); - zoneId = newZoneId; - } - catch (const TException& e) { - throw IoTDBConnectionException(e.what()); - } -} - -std::string SessionConnection::getSystemDefaultZoneId() { - time_t ts = 0; - struct tm tmv{}; -#if defined(_WIN64) || defined (WIN32) || defined (_WIN32) - localtime_s(&tmv, &ts); -#else - localtime_r(&ts, &tmv); -#endif - char zoneStr[32]; - strftime(zoneStr, sizeof(zoneStr), "%z", &tmv); - return zoneStr; -} - -bool SessionConnection::reconnect() { - bool reconnect = false; - for (int i = 1; i <= 3; i++) { - if (transport != nullptr) { - transport->close(); - endPointList = std::move(availableNodes->getEndPointList()); - int currHostIndex = rand() % endPointList.size(); - int tryHostNum = 0; - for (int j = currHostIndex; j < endPointList.size(); j++) { - if (tryHostNum == endPointList.size()) { - break; - } - this->endPoint = endPointList[j]; - if (j == endPointList.size() - 1) { - j = -1; - } - tryHostNum++; - try { - init(this->endPoint); - reconnect = true; - } - catch (const IoTDBConnectionException& e) { - log_warn("The current node may have been down, connection exception: %s", e.what()); - continue; - } catch (exception& e) { - log_warn("login in failed, because %s", e.what()); - } - break; - } - } - if (reconnect) { - session->removeBrokenSessionConnection(shared_from_this()); - session->defaultEndPoint_ = this->endPoint; - session->defaultSessionConnection_ = shared_from_this(); - session->endPointToSessionConnection.insert(make_pair(this->endPoint, shared_from_this())); - } - } - return reconnect; -} - -void SessionConnection::insertStringRecord(const TSInsertStringRecordReq& request) { - auto rpc = [this, request]() { - return this->insertStringRecordInternal(request); - }; - callWithRetryAndVerifyWithRedirection(rpc); -} - -void SessionConnection::insertRecord(const TSInsertRecordReq& request) { - auto rpc = [this, request]() { - return this->insertRecordInternal(request); - }; - callWithRetryAndVerifyWithRedirection(rpc); -} - -void SessionConnection::insertStringRecords(const TSInsertStringRecordsReq& request) { - auto rpc = [this, request]() { - return this->insertStringRecordsInternal(request); - }; - callWithRetryAndVerifyWithRedirection(rpc); -} - -void SessionConnection::insertRecords(const TSInsertRecordsReq& request) { - auto rpc = [this, request]() { - return this->insertRecordsInternal(request); - }; - callWithRetryAndVerifyWithRedirection(rpc); -} - -void SessionConnection::insertRecordsOfOneDevice(TSInsertRecordsOfOneDeviceReq request) { - auto rpc = [this, request]() { - return this->insertRecordsOfOneDeviceInternal(request); - }; - callWithRetryAndVerifyWithRedirection(rpc); -} - -void SessionConnection::insertStringRecordsOfOneDevice(TSInsertStringRecordsOfOneDeviceReq request) { - auto rpc = [this, request]() { - return this->insertStringRecordsOfOneDeviceInternal(request); - }; - callWithRetryAndVerifyWithRedirection(rpc); -} - -void SessionConnection::insertTablet(TSInsertTabletReq request) { - auto rpc = [this, request]() { - return this->insertTabletInternal(request); - }; - callWithRetryAndVerifyWithRedirection(rpc); -} - -void SessionConnection::insertTablets(TSInsertTabletsReq request) { - auto rpc = [this, request]() { - return this->insertTabletsInternal(request); - }; - callWithRetryAndVerifyWithRedirection(rpc); -} - -void SessionConnection::testInsertStringRecord(TSInsertStringRecordReq& request) { - auto rpc = [this, &request]() { - request.sessionId = sessionId; - TSStatus ret; - client->testInsertStringRecord(ret, request); - return ret; - }; - auto status = callWithRetryAndReconnect(rpc).getResult(); - RpcUtils::verifySuccess(status); -} - -void SessionConnection::testInsertTablet(TSInsertTabletReq& request) { - auto rpc = [this, &request]() { - request.sessionId = sessionId; - TSStatus ret; - client->testInsertTablet(ret, request); - return ret; - }; - auto status = callWithRetryAndReconnect(rpc).getResult(); - RpcUtils::verifySuccess(status); -} - -void SessionConnection::testInsertRecords(TSInsertRecordsReq& request) { - auto rpc = [this, &request]() { - request.sessionId = sessionId; - TSStatus ret; - client->testInsertRecords(ret, request); - return ret; - }; - auto status = callWithRetryAndReconnect(rpc).getResult(); - RpcUtils::verifySuccess(status); -} - -void SessionConnection::deleteTimeseries(const vector& paths) { - auto rpc = [this, &paths]() { - TSStatus ret; - client->deleteTimeseries(ret, sessionId, paths); - return ret; - }; - callWithRetryAndVerify(rpc); -} - -void SessionConnection::deleteData(const TSDeleteDataReq& request) { - auto rpc = [this, request]() { - return this->deleteDataInternal(request); - }; - callWithRetryAndVerify(rpc); -} - -void SessionConnection::setStorageGroup(const string& storageGroupId) { - auto rpc = [this, &storageGroupId]() { - TSStatus ret; - client->setStorageGroup(ret, sessionId, storageGroupId); - return ret; - }; - auto ret = callWithRetryAndReconnect(rpc); - RpcUtils::verifySuccess(ret.getResult()); -} - -void SessionConnection::deleteStorageGroups(const vector& storageGroups) { - auto rpc = [this, &storageGroups]() { - TSStatus ret; - client->deleteStorageGroups(ret, sessionId, storageGroups); - return ret; - }; - auto ret = callWithRetryAndReconnect(rpc); - RpcUtils::verifySuccess(ret.getResult()); -} - -void SessionConnection::createTimeseries(TSCreateTimeseriesReq& req) { - auto rpc = [this, &req]() { - TSStatus ret; - req.sessionId = sessionId; - client->createTimeseries(ret, req); - return ret; - }; - auto ret = callWithRetryAndReconnect(rpc); - RpcUtils::verifySuccess(ret.getResult()); -} - -void SessionConnection::createMultiTimeseries(TSCreateMultiTimeseriesReq& req) { - auto rpc = [this, &req]() { - TSStatus ret; - req.sessionId = sessionId; - client->createMultiTimeseries(ret, req); - return ret; - }; - auto ret = callWithRetryAndReconnect(rpc); - RpcUtils::verifySuccess(ret.getResult()); -} - -void SessionConnection::createAlignedTimeseries(TSCreateAlignedTimeseriesReq& req) { - auto rpc = [this, &req]() { - TSStatus ret; - req.sessionId = sessionId; - client->createAlignedTimeseries(ret, req); - return ret; - }; - auto ret = callWithRetryAndReconnect(rpc); - RpcUtils::verifySuccess(ret.getResult()); -} - -TSGetTimeZoneResp SessionConnection::getTimeZone() { - auto rpc = [this]() { - TSGetTimeZoneResp resp; - client->getTimeZone(resp, sessionId); - zoneId = resp.timeZone; - return resp; - }; - auto ret = callWithRetryAndReconnect(rpc, - [](const TSGetTimeZoneResp& resp) { - return resp.status; - }); - RpcUtils::verifySuccess(ret.getResult().status); - return ret.result; -} - -void SessionConnection::setTimeZone(TSSetTimeZoneReq& req) { - auto rpc = [this, &req]() { - TSStatus ret; - req.sessionId = sessionId; - client->setTimeZone(ret, req); - zoneId = req.timeZone; - return ret; - }; - auto ret = callWithRetryAndReconnect(rpc); - RpcUtils::verifySuccess(ret.getResult()); -} - -void SessionConnection::createSchemaTemplate(TSCreateSchemaTemplateReq req) { - auto rpc = [this, &req]() { - TSStatus ret; - req.sessionId = sessionId; - client->createSchemaTemplate(ret, req); - return ret; - }; - auto ret = callWithRetryAndReconnect(rpc); - RpcUtils::verifySuccess(ret.getResult()); -} - -void SessionConnection::setSchemaTemplate(TSSetSchemaTemplateReq req) { - auto rpc = [this, &req]() { - TSStatus ret; - req.sessionId = sessionId; - client->setSchemaTemplate(ret, req); - return ret; - }; - auto ret = callWithRetryAndReconnect(rpc); - RpcUtils::verifySuccess(ret.getResult()); -} - -void SessionConnection::unsetSchemaTemplate(TSUnsetSchemaTemplateReq req) { - auto rpc = [this, &req]() { - TSStatus ret; - req.sessionId = sessionId; - client->unsetSchemaTemplate(ret, req); - return ret; - }; - auto ret = callWithRetryAndReconnect(rpc); - RpcUtils::verifySuccess(ret.getResult()); -} - -void SessionConnection::appendSchemaTemplate(TSAppendSchemaTemplateReq req) { - auto rpc = [this, &req]() { - TSStatus ret; - req.sessionId = sessionId; - client->appendSchemaTemplate(ret, req); - return ret; - }; - auto ret = callWithRetryAndReconnect(rpc); - RpcUtils::verifySuccess(ret.getResult()); -} - -void SessionConnection::pruneSchemaTemplate(TSPruneSchemaTemplateReq req) { - auto rpc = [this, &req]() { - TSStatus ret; - req.sessionId = sessionId; - client->pruneSchemaTemplate(ret, req); - return ret; - }; - auto ret = callWithRetryAndReconnect(rpc); - RpcUtils::verifySuccess(ret.getResult()); -} - -TSQueryTemplateResp SessionConnection::querySchemaTemplate(TSQueryTemplateReq req) { - auto rpc = [this, &req]() { - TSQueryTemplateResp ret; - req.sessionId = sessionId; - client->querySchemaTemplate(ret, req); - return ret; - }; - auto ret = callWithRetryAndReconnect(rpc, - [](const TSQueryTemplateResp& resp) { - return resp.status; - }); - RpcUtils::verifySuccess(ret.getResult().status); - return ret.getResult(); -} - -TSStatus SessionConnection::insertStringRecordInternal(TSInsertStringRecordReq request) { - request.sessionId = sessionId; - TSStatus ret; - client->insertStringRecord(ret, request); - return ret; -} - -TSStatus SessionConnection::insertRecordInternal(TSInsertRecordReq request) { - request.sessionId = sessionId; - TSStatus ret; - client->insertRecord(ret, request); - return ret; -} - -TSStatus SessionConnection::insertStringRecordsInternal(TSInsertStringRecordsReq request) { - request.sessionId = sessionId; - TSStatus ret; - client->insertStringRecords(ret, request); - return ret; -} - -TSStatus SessionConnection::insertRecordsInternal(TSInsertRecordsReq request) { - request.sessionId = sessionId; - TSStatus ret; - client->insertRecords(ret, request); - return ret; -} - -TSStatus SessionConnection::insertRecordsOfOneDeviceInternal(TSInsertRecordsOfOneDeviceReq request) { - request.sessionId = sessionId; - TSStatus ret; - client->insertRecordsOfOneDevice(ret, request); - return ret; -} - -TSStatus SessionConnection::insertStringRecordsOfOneDeviceInternal(TSInsertStringRecordsOfOneDeviceReq request) { - request.sessionId = sessionId; - TSStatus ret; - client->insertStringRecordsOfOneDevice(ret, request); - return ret; -} - -TSStatus SessionConnection::insertTabletInternal(TSInsertTabletReq request) { - request.sessionId = sessionId; - TSStatus ret; - client->insertTablet(ret, request); - return ret; -} - -TSStatus SessionConnection::insertTabletsInternal(TSInsertTabletsReq request) { - request.sessionId = sessionId; - TSStatus ret; - client->insertTablets(ret, request); - return ret; -} - -TSStatus SessionConnection::deleteDataInternal(TSDeleteDataReq request) { - request.sessionId = sessionId; - TSStatus ret; - client->deleteData(ret, request); - return ret; -} diff --git a/iotdb-client/client-cpp/src/main/SessionConnection.h b/iotdb-client/client-cpp/src/main/SessionConnection.h deleted file mode 100644 index 093c48aae452d..0000000000000 --- a/iotdb-client/client-cpp/src/main/SessionConnection.h +++ /dev/null @@ -1,364 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#ifndef IOTDB_SESSIONCONNECTION_H -#define IOTDB_SESSIONCONNECTION_H - -#include -#include -#include -#include -#include "IClientRPCService.h" -#include "common_types.h" -#include "NodesSupplier.h" -#include "Common.h" - -class SessionDataSet; -class Session; - -class SessionConnection : public std::enable_shared_from_this { -public: - SessionConnection(Session* session_ptr, const TEndPoint& endpoint, - const std::string& zoneId, - std::shared_ptr nodeSupplier, - int fetchSize = 10000, - int maxRetries = 60, - int64_t retryInterval = 500, - std::string dialect = "tree", - std::string db = ""); - - ~SessionConnection(); - - void setTimeZone(const std::string& newZoneId); - - - const TEndPoint& getEndPoint(); - - void init(const TEndPoint& endpoint); - - void insertStringRecord(const TSInsertStringRecordReq& request); - - void insertRecord(const TSInsertRecordReq& request); - - void insertStringRecords(const TSInsertStringRecordsReq& request); - - void insertRecords(const TSInsertRecordsReq& request); - - void insertRecordsOfOneDevice(TSInsertRecordsOfOneDeviceReq request); - - void insertStringRecordsOfOneDevice(TSInsertStringRecordsOfOneDeviceReq request); - - void insertTablet(TSInsertTabletReq request); - - void insertTablets(TSInsertTabletsReq request); - - void testInsertStringRecord(TSInsertStringRecordReq& request); - - void testInsertTablet(TSInsertTabletReq& request); - - void testInsertRecords(TSInsertRecordsReq& request); - - void deleteTimeseries(const vector& paths); - - void deleteData(const TSDeleteDataReq& request); - - void setStorageGroup(const string& storageGroupId); - - void deleteStorageGroups(const vector& storageGroups); - - void createTimeseries(TSCreateTimeseriesReq& req); - - void createMultiTimeseries(TSCreateMultiTimeseriesReq& req); - - void createAlignedTimeseries(TSCreateAlignedTimeseriesReq& req); - - TSGetTimeZoneResp getTimeZone(); - - void setTimeZone(TSSetTimeZoneReq& req); - - void createSchemaTemplate(TSCreateSchemaTemplateReq req); - - void setSchemaTemplate(TSSetSchemaTemplateReq req); - - void unsetSchemaTemplate(TSUnsetSchemaTemplateReq req); - - void appendSchemaTemplate(TSAppendSchemaTemplateReq req); - - void pruneSchemaTemplate(TSPruneSchemaTemplateReq req); - - TSQueryTemplateResp querySchemaTemplate(TSQueryTemplateReq req); - - std::unique_ptr executeRawDataQuery(const std::vector& paths, int64_t startTime, - int64_t endTime); - - std::unique_ptr executeLastDataQuery(const std::vector& paths, int64_t lastTime); - - void executeNonQueryStatement(const std::string& sql); - - std::unique_ptr executeQueryStatement(const std::string& sql, int64_t timeoutInMs = -1); - - std::shared_ptr getSessionClient() { - return client; - } - - friend class Session; - -private: - void close(); - std::string getSystemDefaultZoneId(); - bool reconnect(); - - template - struct RetryResult { - T result; - std::exception_ptr exception; - int retryAttempts; - - RetryResult(T r, std::exception_ptr e, int a) - : result(r), exception(e), retryAttempts(a) { - } - - int getRetryAttempts() const { return retryAttempts; } - T getResult() const { return result; } - std::exception_ptr getException() const { return exception; } - }; - - template - void callWithRetryAndVerifyWithRedirection(std::function rpc); - - template - void callWithRetryAndVerifyWithRedirectionForMultipleDevices( - std::function rpc, const vector& deviceIds); - - template - RetryResult callWithRetryAndVerify(std::function rpc); - - template - RetryResult callWithRetry(std::function rpc); - - template - RetryResult callWithRetryAndReconnect(RpcFunc rpc); - - template - RetryResult callWithRetryAndReconnect(RpcFunc rpc, StatusGetter statusGetter); - - template - RetryResult callWithRetryAndReconnect(RpcFunc rpc, ShouldRetry shouldRetry, ForceReconnect forceReconnect); - - TSStatus insertStringRecordInternal(TSInsertStringRecordReq request); - - TSStatus insertRecordInternal(TSInsertRecordReq request); - - TSStatus insertStringRecordsInternal(TSInsertStringRecordsReq request); - - TSStatus insertRecordsInternal(TSInsertRecordsReq request); - - TSStatus insertRecordsOfOneDeviceInternal(TSInsertRecordsOfOneDeviceReq request); - - TSStatus insertStringRecordsOfOneDeviceInternal(TSInsertStringRecordsOfOneDeviceReq request); - - TSStatus insertTabletInternal(TSInsertTabletReq request); - - TSStatus insertTabletsInternal(TSInsertTabletsReq request); - - TSStatus deleteDataInternal(TSDeleteDataReq request); - - std::shared_ptr transport; - std::shared_ptr client; - Session* session; - int64_t sessionId{}; - int64_t statementId{}; - int64_t connectionTimeoutInMs{}; - bool enableRPCCompression = false; - std::string zoneId; - TEndPoint endPoint; - std::vector endPointList; - std::shared_ptr availableNodes; - int fetchSize; - int maxRetryCount; - int64_t retryIntervalMs; - std::string sqlDialect; - std::string database; - int timeFactor = 1000; -}; - -template -SessionConnection::RetryResult SessionConnection::callWithRetry(std::function rpc) { - std::exception_ptr lastException = nullptr; - TSStatus status; - int i; - for (i = 0; i <= maxRetryCount; i++) { - if (i > 0) { - lastException = nullptr; - status = TSStatus(); - try { - std::this_thread::sleep_for( - std::chrono::milliseconds(retryIntervalMs)); - } - catch (const std::exception& e) { - break; - } - if (!reconnect()) { - continue; - } - } - - try { - status = rpc(); - if (status.__isset.needRetry && status.needRetry) { - continue; - } - break; - } - catch (...) { - lastException = std::current_exception(); - } - } - return {status, lastException, i}; -} - -template -void SessionConnection::callWithRetryAndVerifyWithRedirection(std::function rpc) { - auto result = callWithRetry(rpc); - - auto status = result.getResult(); - if (result.getRetryAttempts() == 0) { - RpcUtils::verifySuccessWithRedirection(status); - } - else { - RpcUtils::verifySuccess(status); - } - - if (result.getException()) { - try { - std::rethrow_exception(result.getException()); - } - catch (const std::exception& e) { - throw IoTDBConnectionException(e.what()); - } - } -} - -template -void SessionConnection::callWithRetryAndVerifyWithRedirectionForMultipleDevices( - std::function rpc, const vector& deviceIds) { - auto result = callWithRetry(rpc); - auto status = result.getResult(); - if (result.getRetryAttempts() == 0) { - RpcUtils::verifySuccessWithRedirectionForMultiDevices(status, deviceIds); - } - else { - RpcUtils::verifySuccess(status); - } - if (result.getException()) { - try { - std::rethrow_exception(result.getException()); - } - catch (const std::exception& e) { - throw IoTDBConnectionException(e.what()); - } - } - result.exception = nullptr; -} - -template -SessionConnection::RetryResult SessionConnection::callWithRetryAndVerify(std::function rpc) { - auto result = callWithRetry(rpc); - RpcUtils::verifySuccess(result.getResult()); - if (result.getException()) { - try { - std::rethrow_exception(result.getException()); - } - catch (const std::exception& e) { - throw IoTDBConnectionException(e.what()); - } - } - return result; -} - -template -SessionConnection::RetryResult SessionConnection::callWithRetryAndReconnect(RpcFunc rpc) { - return callWithRetryAndReconnect(rpc, - [](const TSStatus& status) { - return status.__isset.needRetry && status.needRetry; - }, - [](const TSStatus& status) { - return status.code == TSStatusCode::PLAN_FAILED_NETWORK_PARTITION; - } - ); -} - -template -SessionConnection::RetryResult SessionConnection::callWithRetryAndReconnect(RpcFunc rpc, StatusGetter statusGetter) { - auto shouldRetry = [&statusGetter](const T& t) { - auto status = statusGetter(t); - return status.__isset.needRetry && status.needRetry; - }; - auto forceReconnect = [&statusGetter](const T& t) { - auto status = statusGetter(t); - return status.code == TSStatusCode::PLAN_FAILED_NETWORK_PARTITION;; - }; - return callWithRetryAndReconnect(rpc, shouldRetry, forceReconnect); -} - - -template -SessionConnection::RetryResult SessionConnection::callWithRetryAndReconnect(RpcFunc rpc, - ShouldRetry shouldRetry, - ForceReconnect forceReconnect) { - std::exception_ptr lastException = nullptr; - T result; - int retryAttempt; - for (retryAttempt = 0; retryAttempt <= maxRetryCount; retryAttempt++) { - try { - result = rpc(); - lastException = nullptr; - } - catch (...) { - result = T(); - lastException = std::current_exception(); - } - - if (!shouldRetry(result)) { - return {result, lastException, retryAttempt}; - } - - if (lastException != nullptr || - std::find(availableNodes->getEndPointList().begin(), availableNodes->getEndPointList().end(), - this->endPoint) == availableNodes->getEndPointList().end() || - forceReconnect(result)) { - reconnect(); - } - - try { - std::this_thread::sleep_for(std::chrono::milliseconds(retryIntervalMs)); - } - catch (const std::exception& e) { - log_debug("Thread was interrupted during retry " + - std::to_string(retryAttempt) + - " with wait time " + - std::to_string(retryIntervalMs) + - " ms. Exiting retry loop."); - break; - } - } - - return {result, lastException, retryAttempt}; -} - -#endif diff --git a/iotdb-client/client-cpp/src/main/SessionDataSet.cpp b/iotdb-client/client-cpp/src/main/SessionDataSet.cpp deleted file mode 100644 index 30e11fa1efb22..0000000000000 --- a/iotdb-client/client-cpp/src/main/SessionDataSet.cpp +++ /dev/null @@ -1,252 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "SessionDataSet.h" -#include -#include - -RowRecord::RowRecord(int64_t timestamp) { - this->timestamp = timestamp; -} - -RowRecord::RowRecord(int64_t timestamp, const std::vector& fields) - : timestamp(timestamp), fields(fields) { -} - -RowRecord::RowRecord(const std::vector& fields) - : timestamp(-1), fields(fields) { -} - -RowRecord::RowRecord() { - this->timestamp = -1; -} - -void RowRecord::addField(const Field& f) { - this->fields.push_back(f); -} - -std::string RowRecord::toString() { - std::string ret; - if (this->timestamp != -1) { - ret.append(std::to_string(timestamp)); - ret.append("\t"); - } - for (size_t i = 0; i < fields.size(); i++) { - if (i != 0) { - ret.append("\t"); - } - TSDataType::TSDataType dataType = fields[i].dataType; - switch (dataType) { - case TSDataType::BOOLEAN: - ret.append(fields[i].boolV ? "true" : "false"); - break; - case TSDataType::INT32: - ret.append(std::to_string(fields[i].intV)); - break; - case TSDataType::DATE: - ret.append(boost::gregorian::to_iso_extended_string(fields[i].dateV)); - break; - case TSDataType::TIMESTAMP: - case TSDataType::INT64: - ret.append(std::to_string(fields[i].longV)); - break; - case TSDataType::FLOAT: - ret.append(std::to_string(fields[i].floatV)); - break; - case TSDataType::DOUBLE: - ret.append(std::to_string(fields[i].doubleV)); - break; - case TSDataType::BLOB: - case TSDataType::STRING: - case TSDataType::TEXT: - ret.append(fields[i].stringV); - break; - default: - break; - } - } - ret.append("\n"); - return ret; -} - -bool SessionDataSet::hasNext() { - if (iotdbRpcDataSet_->hasCachedRecord()) { - return true; - } - return iotdbRpcDataSet_->next(); -} - -shared_ptr SessionDataSet::next() { - if (!iotdbRpcDataSet_->hasCachedRecord() && !hasNext()) { - return nullptr; - } - iotdbRpcDataSet_->setHasCachedRecord(false); - - return constructRowRecordFromValueArray(); -} - -int SessionDataSet::getFetchSize() { - return iotdbRpcDataSet_->getFetchSize(); -} - -void SessionDataSet::setFetchSize(int fetchSize) { - return iotdbRpcDataSet_->setFetchSize(fetchSize); -} - -const std::vector& SessionDataSet::getColumnNames() const { - return iotdbRpcDataSet_->getColumnNameList(); -} - -const std::vector& SessionDataSet::getColumnTypeList() const { - return iotdbRpcDataSet_->getColumnTypeList(); -} - -void SessionDataSet::closeOperationHandle(bool forceClose) { - iotdbRpcDataSet_->close(forceClose); -} - -bool SessionDataSet::DataIterator::next() { - return iotdbRpcDataSet_->next(); -} - -bool SessionDataSet::DataIterator::isNull(const std::string& columnName) { - return iotdbRpcDataSet_->isNullByColumnName(columnName); -} - -bool SessionDataSet::DataIterator::isNullByIndex(int32_t columnIndex) { - return iotdbRpcDataSet_->isNullByIndex(columnIndex); -} - -bool SessionDataSet::DataIterator::getBooleanByIndex(int32_t columnIndex) { - return iotdbRpcDataSet_->getBooleanByIndex(columnIndex); -} - -bool SessionDataSet::DataIterator::getBoolean(const std::string& columnName) { - return iotdbRpcDataSet_->getBoolean(columnName); -} - -double SessionDataSet::DataIterator::getDoubleByIndex(int32_t columnIndex) { - return iotdbRpcDataSet_->getDoubleByIndex(columnIndex); -} - -double SessionDataSet::DataIterator::getDouble(const std::string& columnName) { - return iotdbRpcDataSet_->getDouble(columnName); -} - -float SessionDataSet::DataIterator::getFloatByIndex(int32_t columnIndex) { - return iotdbRpcDataSet_->getFloatByIndex(columnIndex); -} - -float SessionDataSet::DataIterator::getFloat(const std::string& columnName) { - return iotdbRpcDataSet_->getFloat(columnName); -} - -int32_t SessionDataSet::DataIterator::getIntByIndex(int32_t columnIndex) { - return iotdbRpcDataSet_->getIntByIndex(columnIndex); -} - -int32_t SessionDataSet::DataIterator::getInt(const std::string& columnName) { - return iotdbRpcDataSet_->getInt(columnName); -} - -int64_t SessionDataSet::DataIterator::getLongByIndex(int32_t columnIndex) { - return iotdbRpcDataSet_->getLongByIndex(columnIndex); -} - -int64_t SessionDataSet::DataIterator::getLong(const std::string& columnName) { - return iotdbRpcDataSet_->getLong(columnName); -} - -std::string SessionDataSet::DataIterator::getStringByIndex(int32_t columnIndex) { - return iotdbRpcDataSet_->getStringByIndex(columnIndex); -} - -std::string SessionDataSet::DataIterator::getString(const std::string& columnName) { - return iotdbRpcDataSet_->getString(columnName); -} - -int64_t SessionDataSet::DataIterator::getTimestampByIndex(int32_t columnIndex) { - return iotdbRpcDataSet_->getTimestampByIndex(columnIndex); -} - -int64_t SessionDataSet::DataIterator::getTimestamp(const std::string& columnName) { - return iotdbRpcDataSet_->getTimestamp(columnName); -} - -boost::gregorian::date SessionDataSet::DataIterator::getDateByIndex(int32_t columnIndex) { - return iotdbRpcDataSet_->getDateByIndex(columnIndex); -} - -boost::gregorian::date SessionDataSet::DataIterator::getDate(const std::string& columnName) { - return iotdbRpcDataSet_->getDate(columnName); -} - -int32_t SessionDataSet::DataIterator::findColumn(const std::string& columnName) { - return iotdbRpcDataSet_->findColumn(columnName); -} - -const std::vector& SessionDataSet::DataIterator::getColumnNames() const { - return iotdbRpcDataSet_->getColumnNameList(); -} - -const std::vector& SessionDataSet::DataIterator::getColumnTypeList() const { - return iotdbRpcDataSet_->getColumnTypeList(); -} - -shared_ptr SessionDataSet::constructRowRecordFromValueArray() { - std::vector outFields; - for (int i = iotdbRpcDataSet_->getValueColumnStartIndex(); i < iotdbRpcDataSet_->getColumnSize(); i++) { - Field field; - std::string columnName = iotdbRpcDataSet_->getColumnNameList().at(i); - if (!iotdbRpcDataSet_->isNullByColumnName(columnName)) { - TSDataType::TSDataType dataType = iotdbRpcDataSet_->getDataType(columnName); - field.dataType = dataType; - switch (dataType) { - case TSDataType::BOOLEAN: - field.boolV = iotdbRpcDataSet_->getBoolean(columnName); - break; - case TSDataType::INT32: - field.intV = iotdbRpcDataSet_->getInt(columnName); - break; - case TSDataType::DATE: - field.dateV = parseIntToDate(iotdbRpcDataSet_->getInt(columnName)); - break; - case TSDataType::INT64: - case TSDataType::TIMESTAMP: - field.longV = iotdbRpcDataSet_->getLong(columnName); - break; - case TSDataType::FLOAT: - field.floatV = iotdbRpcDataSet_->getFloat(columnName); - break; - case TSDataType::DOUBLE: - field.doubleV = iotdbRpcDataSet_->getDouble(columnName); - break; - case TSDataType::TEXT: - case TSDataType::BLOB: - case TSDataType::STRING: - field.stringV = iotdbRpcDataSet_->getBinary(columnName)->getStringValue(); - break; - default: - throw new UnSupportedDataTypeException("Data type %s is not supported." + dataType); - } - } - outFields.emplace_back(field); - } - return std::make_shared(iotdbRpcDataSet_->getCurrentRowTime(), outFields); -} diff --git a/iotdb-client/client-cpp/src/main/SessionDataSet.h b/iotdb-client/client-cpp/src/main/SessionDataSet.h deleted file mode 100644 index d6178d97fbafa..0000000000000 --- a/iotdb-client/client-cpp/src/main/SessionDataSet.h +++ /dev/null @@ -1,142 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#ifndef IOTDB_SESSION_DATA_SET_H -#define IOTDB_SESSION_DATA_SET_H - -#include -#include -#include -#include -#include -#include "IoTDBRpcDataSet.h" -#include "Column.h" - -class RowRecord { -public: - int64_t timestamp; - std::vector fields; - - explicit RowRecord(int64_t timestamp); - RowRecord(int64_t timestamp, const std::vector& fields); - explicit RowRecord(const std::vector& fields); - RowRecord(); - - void addField(const Field& f); - std::string toString(); -}; - -class SessionDataSet { -public: - SessionDataSet(const std::string& sql, - const std::vector& columnNameList, - const std::vector& columnTypeList, - const std::map& columnNameIndex, - int64_t queryId, - int64_t statementId, - std::shared_ptr client, - int64_t sessionId, - const std::vector& queryResult, - bool ignoreTimestamp, - int64_t timeout, - bool moreData, - int32_t fetchSize, - const std::string& zoneId) { - iotdbRpcDataSet_ = std::make_shared(sql, - columnNameList, - columnTypeList, - columnNameIndex, - ignoreTimestamp, - moreData, - queryId, - statementId, - client, - sessionId, - queryResult, - fetchSize, - timeout, - zoneId, - IoTDBRpcDataSet::DEFAULT_TIME_FORMAT); - } - - ~SessionDataSet() = default; - - bool hasNext(); - shared_ptr next(); - - int getFetchSize(); - void setFetchSize(int fetchSize); - - const std::vector& getColumnNames() const; - const std::vector& getColumnTypeList() const; - void closeOperationHandle(bool forceClose = false); - - class DataIterator { - std::shared_ptr iotdbRpcDataSet_; - - public: - DataIterator(std::shared_ptr iotdbRpcDataSet) : - iotdbRpcDataSet_(iotdbRpcDataSet) { - } - - bool next(); - - bool isNull(const std::string& columnName); - bool isNullByIndex(int32_t columnIndex); - - bool getBooleanByIndex(int32_t columnIndex); - bool getBoolean(const std::string& columnName); - - double getDoubleByIndex(int32_t columnIndex); - double getDouble(const std::string& columnName); - - float getFloatByIndex(int32_t columnIndex); - float getFloat(const std::string& columnName); - - int32_t getIntByIndex(int32_t columnIndex); - int32_t getInt(const std::string& columnName); - - int64_t getLongByIndex(int32_t columnIndex); - int64_t getLong(const std::string& columnName); - - std::string getStringByIndex(int32_t columnIndex); - std::string getString(const std::string& columnName); - - int64_t getTimestampByIndex(int32_t columnIndex); - int64_t getTimestamp(const std::string& columnName); - - boost::gregorian::date getDateByIndex(int32_t columnIndex); - boost::gregorian::date getDate(const std::string& columnName); - - int32_t findColumn(const std::string& columnName); - const std::vector& getColumnNames() const; - const std::vector& getColumnTypeList() const; - }; - - DataIterator getIterator() { - return {iotdbRpcDataSet_}; - }; - -private: - std::shared_ptr constructRowRecordFromValueArray(); - - std::shared_ptr iotdbRpcDataSet_; -}; - -#endif diff --git a/iotdb-client/client-cpp/src/main/ThriftConnection.cpp b/iotdb-client/client-cpp/src/main/ThriftConnection.cpp deleted file mode 100644 index 8cee0c6f0140d..0000000000000 --- a/iotdb-client/client-cpp/src/main/ThriftConnection.cpp +++ /dev/null @@ -1,170 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#include "ThriftConnection.h" -#include -#include -#include - -#include "Session.h" -#include "SessionDataSet.h" - -const int ThriftConnection::THRIFT_DEFAULT_BUFFER_SIZE = 4096; -const int ThriftConnection::THRIFT_MAX_FRAME_SIZE = 1048576; -const int ThriftConnection::CONNECTION_TIMEOUT_IN_MS = 1000; -const int ThriftConnection::DEFAULT_FETCH_SIZE = 10000; - -ThriftConnection::ThriftConnection(const TEndPoint& endPoint, - int thriftDefaultBufferSize, - int thriftMaxFrameSize, - int connectionTimeoutInMs, - int fetchSize) - : endPoint_(endPoint), - thriftDefaultBufferSize_(thriftDefaultBufferSize), - thriftMaxFrameSize_(thriftMaxFrameSize), - connectionTimeoutInMs_(connectionTimeoutInMs), - fetchSize_(fetchSize){ -} - -ThriftConnection::~ThriftConnection() = default; - -void ThriftConnection::initZoneId() { - if (!zoneId_.empty()) { - return; - } - - time_t ts = 0; - struct tm tmv{}; -#if defined(_WIN64) || defined (WIN32) || defined (_WIN32) - localtime_s(&tmv, &ts); -#else - localtime_r(&ts, &tmv); -#endif - - char zoneStr[32]; - strftime(zoneStr, sizeof(zoneStr), "%z", &tmv); - zoneId_ = zoneStr; -} - -void ThriftConnection::init(const std::string& username, - const std::string& password, - bool enableRPCCompression, - const std::string& zoneId, - const std::string& version) { - std::shared_ptr socket(new TSocket(endPoint_.ip, endPoint_.port)); - socket->setConnTimeout(connectionTimeoutInMs_); - transport_ = std::make_shared(socket); - if (!transport_->isOpen()) { - try { - transport_->open(); - } - catch (TTransportException& e) { - throw IoTDBConnectionException(e.what()); - } - } - if (zoneId.empty()) { - initZoneId(); - } - else { - this->zoneId_ = zoneId; - } - - if (enableRPCCompression) { - std::shared_ptr protocol(new TCompactProtocol(transport_)); - client_ = std::make_shared(protocol); - } - else { - std::shared_ptr protocol(new TBinaryProtocol(transport_)); - client_ = std::make_shared(protocol); - } - - std::map configuration; - configuration["version"] = version; - TSOpenSessionReq openReq; - openReq.__set_username(username); - openReq.__set_password(password); - openReq.__set_zoneId(this->zoneId_); - openReq.__set_configuration(configuration); - try { - TSOpenSessionResp openResp; - client_->openSession(openResp, openReq); - RpcUtils::verifySuccess(openResp.status); - sessionId_ = openResp.sessionId; - statementId_ = client_->requestStatementId(sessionId_); - } - catch (const TTransportException& e) { - transport_->close(); - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException& e) { - transport_->close(); - throw IoTDBException(e.what()); - } catch (const std::exception& e) { - transport_->close(); - throw IoTDBException(e.what()); - } -} - -std::unique_ptr ThriftConnection::executeQueryStatement(const std::string& sql, int64_t timeoutInMs) { - TSExecuteStatementReq req; - req.__set_sessionId(sessionId_); - req.__set_statementId(statementId_); - req.__set_statement(sql); - req.__set_timeout(timeoutInMs); - TSExecuteStatementResp resp; - try { - client_->executeQueryStatementV2(resp, req); - RpcUtils::verifySuccess(resp.status); - } - catch (const TTransportException& e) { - throw IoTDBConnectionException(e.what()); - } catch (const IoTDBException& e) { - throw IoTDBConnectionException(e.what()); - } catch (const std::exception& e) { - throw IoTDBException(e.what()); - } - std::shared_ptr queryDataSet(new TSQueryDataSet(resp.queryDataSet)); - return std::unique_ptr(new SessionDataSet("", resp.columns, resp.dataTypeList, - resp.columnNameIndexMap, resp.queryId, statementId_, - client_, sessionId_, resp.queryResult, resp.ignoreTimeStamp, - connectionTimeoutInMs_, resp.moreData, fetchSize_, zoneId_)); -} - -void ThriftConnection::close() { - try { - if (client_) { - TSCloseSessionReq req; - req.__set_sessionId(sessionId_); - TSStatus tsStatus; - client_->closeSession(tsStatus, req); - } - } - catch (const TTransportException& e) { - throw IoTDBConnectionException(e.what()); - } catch (const std::exception& e) { - throw IoTDBConnectionException(e.what()); - } - - try { - if (transport_->isOpen()) { - transport_->close(); - } - } - catch (const std::exception& e) { - throw IoTDBConnectionException(e.what()); - } -} diff --git a/iotdb-client/client-cpp/src/main/ThriftConnection.h b/iotdb-client/client-cpp/src/main/ThriftConnection.h deleted file mode 100644 index d578e37b1b705..0000000000000 --- a/iotdb-client/client-cpp/src/main/ThriftConnection.h +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#ifndef IOTDB_THRIFTCONNECTION_H -#define IOTDB_THRIFTCONNECTION_H - -#include -#include -#include "IClientRPCService.h" - -class SessionDataSet; - -class ThriftConnection { -public: - static const int THRIFT_DEFAULT_BUFFER_SIZE; - static const int THRIFT_MAX_FRAME_SIZE; - static const int CONNECTION_TIMEOUT_IN_MS; - static const int DEFAULT_FETCH_SIZE; - - explicit ThriftConnection(const TEndPoint& endPoint, - int thriftDefaultBufferSize = THRIFT_DEFAULT_BUFFER_SIZE, - int thriftMaxFrameSize = THRIFT_MAX_FRAME_SIZE, - int connectionTimeoutInMs = CONNECTION_TIMEOUT_IN_MS, - int fetchSize = DEFAULT_FETCH_SIZE); - - ~ThriftConnection(); - - void init(const std::string& username, - const std::string& password, - bool enableRPCCompression = false, - const std::string& zoneId = std::string(), - const std::string& version = "V_1_0"); - - std::unique_ptr executeQueryStatement(const std::string& sql, int64_t timeoutInMs = -1); - - void close(); - -private: - TEndPoint endPoint_; - - int thriftDefaultBufferSize_; - int thriftMaxFrameSize_; - int connectionTimeoutInMs_; - int fetchSize_; - - std::shared_ptr transport_; - std::shared_ptr client_; - int64_t sessionId_{}; - int64_t statementId_{}; - std::string zoneId_; - int timeFactor_{}; - - void initZoneId(); -}; - -#endif diff --git a/iotdb-client/client-cpp/src/main/TsBlock.cpp b/iotdb-client/client-cpp/src/main/TsBlock.cpp deleted file mode 100644 index 7c2bac272a601..0000000000000 --- a/iotdb-client/client-cpp/src/main/TsBlock.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -#include -#include -#include "TsBlock.h" - -std::shared_ptr TsBlock::create(int32_t positionCount, - std::shared_ptr timeColumn, - std::vector> valueColumns) { - if (valueColumns.empty()) { - throw std::invalid_argument("valueColumns cannot be empty"); - } - return std::shared_ptr(new TsBlock(positionCount, std::move(timeColumn), std::move(valueColumns))); -} - -std::shared_ptr TsBlock::deserialize(const std::string& data) { - MyStringBuffer buffer(data); - - // Read value column count - int32_t valueColumnCount = buffer.getInt(); - - // Read value column data types - std::vector valueColumnDataTypes(valueColumnCount); - for (int32_t i = 0; i < valueColumnCount; i++) { - valueColumnDataTypes[i] = static_cast(buffer.getChar()); - } - - // Read position count - int32_t positionCount = buffer.getInt(); - - // Read column encodings - std::vector columnEncodings(valueColumnCount + 1); - for (int32_t i = 0; i < valueColumnCount + 1; i++) { - columnEncodings[i] = static_cast(buffer.getChar()); - } - - // Read time column - auto timeColumnDecoder = getColumnDecoder(columnEncodings[0]); - auto timeColumn = timeColumnDecoder->readColumn(buffer, TSDataType::INT64, positionCount); - - // Read value columns - std::vector> valueColumns(valueColumnCount); - for (int32_t i = 0; i < valueColumnCount; i++) { - auto valueColumnDecoder = getColumnDecoder(columnEncodings[i + 1]); - valueColumns[i] = valueColumnDecoder->readColumn(buffer, valueColumnDataTypes[i], positionCount); - } - - return create(positionCount, std::move(timeColumn), std::move(valueColumns)); -} - -TsBlock::TsBlock(int32_t positionCount, - std::shared_ptr timeColumn, - std::vector> valueColumns) - : positionCount_(positionCount), - timeColumn_(std::move(timeColumn)), - valueColumns_(std::move(valueColumns)) { -} - -int32_t TsBlock::getPositionCount() const { - return positionCount_; -} - -int64_t TsBlock::getStartTime() const { - return timeColumn_->getLong(0); -} - -int64_t TsBlock::getEndTime() const { - return timeColumn_->getLong(positionCount_ - 1); -} - -bool TsBlock::isEmpty() const { - return positionCount_ == 0; -} - -int64_t TsBlock::getTimeByIndex(int32_t index) const { - return timeColumn_->getLong(index); -} - -int32_t TsBlock::getValueColumnCount() const { - return static_cast(valueColumns_.size()); -} - -const std::shared_ptr TsBlock::getTimeColumn() const { - return timeColumn_; -} - -const std::vector>& TsBlock::getValueColumns() const { - return valueColumns_; -} - -const std::shared_ptr TsBlock::getColumn(int32_t columnIndex) const { - return valueColumns_[columnIndex]; -} diff --git a/iotdb-client/client-cpp/src/main/TsBlock.h b/iotdb-client/client-cpp/src/main/TsBlock.h deleted file mode 100644 index 2ca6b0197445d..0000000000000 --- a/iotdb-client/client-cpp/src/main/TsBlock.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#ifndef IOTDB_TS_BLOCK_H -#define IOTDB_TS_BLOCK_H - -#include -#include -#include "Column.h" - -class TsBlock { -public: - static std::shared_ptr create(int32_t positionCount, - std::shared_ptr timeColumn, - std::vector> valueColumns); - - static std::shared_ptr deserialize(const std::string& data); - - int32_t getPositionCount() const; - int64_t getStartTime() const; - int64_t getEndTime() const; - bool isEmpty() const; - int64_t getTimeByIndex(int32_t index) const; - int32_t getValueColumnCount() const; - const std::shared_ptr getTimeColumn() const; - const std::vector>& getValueColumns() const; - const std::shared_ptr getColumn(int32_t columnIndex) const; - -private: - TsBlock(int32_t positionCount, - std::shared_ptr timeColumn, - std::vector> valueColumns); - - std::shared_ptr timeColumn_; - std::vector> valueColumns_; - int32_t positionCount_; -}; - -#endif diff --git a/iotdb-client/client-cpp/src/rpc/IoTDBRpcDataSet.cpp b/iotdb-client/client-cpp/src/rpc/IoTDBRpcDataSet.cpp new file mode 100644 index 0000000000000..27ea6bca24877 --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/IoTDBRpcDataSet.cpp @@ -0,0 +1,734 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include + +#include "IoTDBRpcDataSet.h" + +#include + +#include "Column.h" +#include "Date.h" +#include "Optional.h" +#include "RpcCommon.h" + +using apache::thrift::transport::TTransportException; + +const int32_t IoTDBRpcDataSet::startIndex = 2; +const std::string IoTDBRpcDataSet::TimestampColumnName = "Time"; +const std::string IoTDBRpcDataSet::DEFAULT_TIME_FORMAT = "default"; +const std::string IoTDBRpcDataSet::TIME_PRECISION = "timestamp_precision"; +const std::string IoTDBRpcDataSet::MILLISECOND = "ms"; +const std::string IoTDBRpcDataSet::MICROSECOND = "us"; +const std::string IoTDBRpcDataSet::NANOSECOND = "ns"; + +IoTDBRpcDataSet::IoTDBRpcDataSet( + const std::string &sql, const std::vector &columnNameList, + const std::vector &columnTypeList, + const std::map &columnNameIndex, bool ignoreTimestamp, + bool moreData, int64_t queryId, int64_t statementId, + std::shared_ptr client, int64_t sessionId, + const std::vector &queryResult, int32_t fetchSize, + const int64_t timeout, const std::string &zoneId, + const std::string &timeFormat) + : sql_(sql), isClosed_(false), client_(client), fetchSize_(fetchSize), + timeout_(timeout), hasCachedRecord_(false), lastReadWasNull_(false), + columnSize_(static_cast(columnNameList.size())), + sessionId_(sessionId), queryId_(queryId), statementId_(statementId), + time_(0), ignoreTimestamp_(ignoreTimestamp), moreData_(moreData), + queryResult_(queryResult), curTsBlock_(nullptr), + queryResultSize_(static_cast(queryResult.size())), + queryResultIndex_(0), tsBlockSize_(0), tsBlockIndex_(-1), + timeZoneId_(zoneId), timeFormat_(timeFormat) { + std::vector columnIndex2TsBlockColumnIndexList; + int columnStartIndex = 1; + int resultSetColumnSize = columnNameList_.size(); + // newly generated or updated columnIndex2TsBlockColumnIndexList.size() may + // not be equal to columnNameList.size() so we need + // startIndexForColumnIndex2TsBlockColumnIndexList to adjust the mapping + // relation + int startIndexForColumnIndex2TsBlockColumnIndexList = 0; + const bool serverIncludesTime = + !columnNameList.empty() && columnNameList[0] == TimestampColumnName; + // for Time Column in tree model which should always be the first column and + // its index for TsBlockColumn is -1 + if (!ignoreTimestamp && !serverIncludesTime) { + columnNameList_.push_back(TimestampColumnName); + columnTypeList_.push_back("INT64"); + columnName2TsBlockColumnIndexMap_[TimestampColumnName] = -1; + columnOrdinalMap_[TimestampColumnName] = 1; + if (!columnIndex2TsBlockColumnIndexList.empty()) { + columnIndex2TsBlockColumnIndexList.insert( + columnIndex2TsBlockColumnIndexList.begin(), -1); + startIndexForColumnIndex2TsBlockColumnIndexList = 1; + } + columnStartIndex++; + resultSetColumnSize++; + } + + if (columnIndex2TsBlockColumnIndexList.empty()) { + columnIndex2TsBlockColumnIndexList.reserve(resultSetColumnSize); + if (!ignoreTimestamp && !serverIncludesTime) { + startIndexForColumnIndex2TsBlockColumnIndexList = 1; + columnIndex2TsBlockColumnIndexList.push_back(-1); + } + for (size_t i = 0; i < columnNameList.size(); ++i) { + if (!columnNameIndex.empty()) { + auto it = columnNameIndex.find(columnNameList[i]); + columnIndex2TsBlockColumnIndexList.push_back( + it != columnNameIndex.end() ? it->second : static_cast(i)); + } else { + columnIndex2TsBlockColumnIndexList.push_back(static_cast(i)); + } + } + } + + columnNameList_.insert(columnNameList_.end(), columnNameList.begin(), + columnNameList.end()); + columnTypeList_.insert(columnTypeList_.end(), columnTypeList.begin(), + columnTypeList.end()); + + // Initialize data types for TsBlock columns + int32_t tsBlockColumnSize = 0; + for (auto value : columnIndex2TsBlockColumnIndexList) { + if (value > tsBlockColumnSize) { + tsBlockColumnSize = value; + } + } + tsBlockColumnSize += 1; + dataTypeForTsBlockColumn_.resize(tsBlockColumnSize); + + // Populate data types and maps + for (size_t i = 0; i < columnNameList.size(); i++) { + auto columnName = columnNameList[i]; + int32_t tsBlockColumnIndex; + if (!columnNameIndex.empty()) { + auto it = columnNameIndex.find(columnName); + tsBlockColumnIndex = + it != columnNameIndex.end() ? it->second : static_cast(i); + } else { + tsBlockColumnIndex = columnIndex2TsBlockColumnIndexList + [i + startIndexForColumnIndex2TsBlockColumnIndexList]; + } + if (tsBlockColumnIndex != -1) { + if (static_cast(tsBlockColumnIndex) >= + dataTypeForTsBlockColumn_.size()) { + dataTypeForTsBlockColumn_.resize(tsBlockColumnIndex + 1); + } + dataTypeForTsBlockColumn_[tsBlockColumnIndex] = + getDataTypeByStr(columnTypeList[i]); + } + + if (columnName2TsBlockColumnIndexMap_.find(columnName) == + columnName2TsBlockColumnIndexMap_.end()) { + columnOrdinalMap_[columnName] = i + columnStartIndex; + columnName2TsBlockColumnIndexMap_[columnName] = tsBlockColumnIndex; + } + } + + timePrecision_ = getTimePrecision(timeFactor_); + columnIndex2TsBlockColumnIndexList_ = columnIndex2TsBlockColumnIndexList; + + if (serverIncludesTime) { + columnSize_--; + } +} + +IoTDBRpcDataSet::~IoTDBRpcDataSet() { + if (!isClosed_) { + close(); + } +} + +bool IoTDBRpcDataSet::next() { + if (hasCachedBlock()) { + lastReadWasNull_ = false; + constructOneRow(); + return true; + } + + if (hasCachedByteBuffer()) { + constructOneTsBlock(); + constructOneRow(); + return true; + } + + if (moreData_) { + bool hasResultSet = fetchResults(); + if (hasResultSet && hasCachedByteBuffer()) { + constructOneTsBlock(); + constructOneRow(); + return true; + } + } + + close(); + return false; +} + +void IoTDBRpcDataSet::close(bool forceClose) { + if ((!forceClose) && isClosed_) { + return; + } + TSCloseOperationReq closeReq; + closeReq.__set_sessionId(sessionId_); + closeReq.__set_statementId(statementId_); + closeReq.__set_queryId(queryId_); + TSStatus tsStatus; + try { + client_->closeOperation(tsStatus, closeReq); + RpcUtils::verifySuccess(tsStatus); + } catch (const TTransportException &e) { + log_debug(e.what()); + throw IoTDBConnectionException(e.what()); + } catch (const IoTDBException &e) { + log_debug(e.what()); + throw; + } catch (std::exception &e) { + log_debug(e.what()); + throw IoTDBException(e.what()); + } + isClosed_ = true; + client_ = nullptr; +} + +bool IoTDBRpcDataSet::fetchResults() { + if (isClosed_) { + throw IoTDBException("This data set is already closed"); + } + + TSFetchResultsReq req; + req.__set_sessionId(sessionId_); + req.__set_statement(sql_); + req.__set_fetchSize(fetchSize_); + req.__set_queryId(queryId_); + req.__set_isAlign(true); + req.__set_timeout(timeout_); + TSFetchResultsResp resp; + client_->fetchResultsV2(resp, req); + RpcUtils::verifySuccess(resp.status); + moreData_ = resp.moreData; + if (!resp.hasResultSet) { + close(); + } else { + queryResult_ = resp.queryResult; + queryResultIndex_ = 0; + if (!queryResult_.empty()) { + queryResultSize_ = queryResult_.size(); + } else { + queryResultSize_ = 0; + } + tsBlockIndex_ = -1; + tsBlockSize_ = 0; + } + return resp.hasResultSet; +} + +void IoTDBRpcDataSet::constructOneRow() { + tsBlockIndex_++; + hasCachedRecord_ = true; + time_ = curTsBlock_->getTimeColumn()->getLong(tsBlockIndex_); +} + +void IoTDBRpcDataSet::constructOneTsBlock() { + lastReadWasNull_ = false; + const auto &curTsBlockBytes = queryResult_[queryResultIndex_]; + queryResultIndex_++; + curTsBlock_ = TsBlock::deserialize(curTsBlockBytes); + tsBlockIndex_ = -1; + tsBlockSize_ = curTsBlock_->getPositionCount(); +} + +bool IoTDBRpcDataSet::isNullByIndex(int32_t columnIndex) { + int32_t index = getTsBlockColumnIndexForColumnIndex(columnIndex); + return isNull(index, tsBlockIndex_); +} + +bool IoTDBRpcDataSet::isNullByColumnName(const std::string &columnName) { + int32_t index = getTsBlockColumnIndexForColumnName(columnName); + return isNull(index, tsBlockIndex_); +} + +bool IoTDBRpcDataSet::isNull(int32_t index, int32_t rowNum) { + return index >= 0 && curTsBlock_->getColumn(index)->isNull(rowNum); +} + +Optional IoTDBRpcDataSet::getBooleanByIndex(int32_t columnIndex) { + int32_t index = getTsBlockColumnIndexForColumnIndex(columnIndex); + return getBooleanByTsBlockColumnIndex(index); +} + +Optional IoTDBRpcDataSet::getBoolean(const std::string &columnName) { + int32_t index = getTsBlockColumnIndexForColumnName(columnName); + return getBooleanByTsBlockColumnIndex(index); +} + +// Note: tsBlockColumnIndex < 0 indicates the time pseudo-column in tree model. +// Only getLong and getString support reading the time column directly. +// All other typed getters throw IoTDBException to prevent undefined behavior +// from accessing valueColumns_ with a negative index. +Optional +IoTDBRpcDataSet::getBooleanByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { + checkRecord(); + if (tsBlockColumnIndex < 0) { + throw IoTDBException("Cannot read boolean from time column"); + } + if (!isNull(tsBlockColumnIndex, tsBlockIndex_)) { + lastReadWasNull_ = false; + return curTsBlock_->getColumn(tsBlockColumnIndex) + ->getBoolean(tsBlockIndex_); + } else { + lastReadWasNull_ = true; + return Optional::none(); + } +} + +Optional IoTDBRpcDataSet::getDoubleByIndex(int32_t columnIndex) { + int32_t index = getTsBlockColumnIndexForColumnIndex(columnIndex); + return getDoubleByTsBlockColumnIndex(index); +} + +Optional IoTDBRpcDataSet::getDouble(const std::string &columnName) { + int32_t index = getTsBlockColumnIndexForColumnName(columnName); + return getDoubleByTsBlockColumnIndex(index); +} + +Optional +IoTDBRpcDataSet::getDoubleByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { + checkRecord(); + if (tsBlockColumnIndex < 0) { + throw IoTDBException("Cannot read double from time column"); + } + if (!isNull(tsBlockColumnIndex, tsBlockIndex_)) { + lastReadWasNull_ = false; + return curTsBlock_->getColumn(tsBlockColumnIndex)->getDouble(tsBlockIndex_); + } else { + lastReadWasNull_ = true; + return Optional::none(); + } +} + +Optional IoTDBRpcDataSet::getFloatByIndex(int32_t columnIndex) { + int32_t index = getTsBlockColumnIndexForColumnIndex(columnIndex); + return getFloatByTsBlockColumnIndex(index); +} + +Optional IoTDBRpcDataSet::getFloat(const std::string &columnName) { + int32_t index = getTsBlockColumnIndexForColumnName(columnName); + return getFloatByTsBlockColumnIndex(index); +} + +Optional +IoTDBRpcDataSet::getFloatByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { + checkRecord(); + if (tsBlockColumnIndex < 0) { + throw IoTDBException("Cannot read float from time column"); + } + if (!isNull(tsBlockColumnIndex, tsBlockIndex_)) { + lastReadWasNull_ = false; + return curTsBlock_->getColumn(tsBlockColumnIndex)->getFloat(tsBlockIndex_); + } else { + lastReadWasNull_ = true; + return Optional::none(); + } +} + +Optional IoTDBRpcDataSet::getIntByIndex(int32_t columnIndex) { + int32_t index = getTsBlockColumnIndexForColumnIndex(columnIndex); + return getIntByTsBlockColumnIndex(index); +} + +Optional IoTDBRpcDataSet::getInt(const std::string &columnName) { + int32_t index = getTsBlockColumnIndexForColumnName(columnName); + return getIntByTsBlockColumnIndex(index); +} + +Optional +IoTDBRpcDataSet::getIntByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { + checkRecord(); + if (tsBlockColumnIndex < 0) { + throw IoTDBException("Cannot read int32 from time column"); + } + if (!isNull(tsBlockColumnIndex, tsBlockIndex_)) { + lastReadWasNull_ = false; + const auto column = curTsBlock_->getColumn(tsBlockColumnIndex); + switch (column->getDataType()) { + case TSDataType::INT64: + return static_cast(column->getLong(tsBlockIndex_)); + case TSDataType::DOUBLE: + return static_cast(column->getDouble(tsBlockIndex_)); + case TSDataType::FLOAT: + return static_cast(column->getFloat(tsBlockIndex_)); + case TSDataType::INT32: + return column->getInt(tsBlockIndex_); + default: + return column->getInt(tsBlockIndex_); + } + } else { + lastReadWasNull_ = true; + return Optional::none(); + } +} + +Optional IoTDBRpcDataSet::getLongByIndex(int32_t columnIndex) { + int32_t index = getTsBlockColumnIndexForColumnIndex(columnIndex); + return getLongByTsBlockColumnIndex(index); +} + +Optional IoTDBRpcDataSet::getLong(const std::string &columnName) { + int32_t index = getTsBlockColumnIndexForColumnName(columnName); + return getLongByTsBlockColumnIndex(index); +} + +Optional +IoTDBRpcDataSet::getLongByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { + checkRecord(); + if (tsBlockColumnIndex < 0) { + lastReadWasNull_ = false; + return curTsBlock_->getTimeByIndex(tsBlockIndex_); + } + if (!isNull(tsBlockColumnIndex, tsBlockIndex_)) { + lastReadWasNull_ = false; + const auto column = curTsBlock_->getColumn(tsBlockColumnIndex); + switch (column->getDataType()) { + case TSDataType::INT32: + return static_cast(column->getInt(tsBlockIndex_)); + case TSDataType::FLOAT: + return static_cast(column->getFloat(tsBlockIndex_)); + case TSDataType::DOUBLE: + return static_cast(column->getDouble(tsBlockIndex_)); + case TSDataType::INT64: + return column->getLong(tsBlockIndex_); + default: + return column->getLong(tsBlockIndex_); + } + } else { + lastReadWasNull_ = true; + return Optional::none(); + } +} + +std::shared_ptr IoTDBRpcDataSet::getBinaryByIndex(int32_t columnIndex) { + int32_t index = getTsBlockColumnIndexForColumnIndex(columnIndex); + return getBinaryByTsBlockColumnIndex(index); +} + +std::shared_ptr +IoTDBRpcDataSet::getBinary(const std::string &columnName) { + int32_t index = getTsBlockColumnIndexForColumnName(columnName); + return getBinaryByTsBlockColumnIndex(index); +} + +std::shared_ptr +IoTDBRpcDataSet::getBinaryByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { + checkRecord(); + if (tsBlockColumnIndex < 0) { + throw IoTDBException("Cannot read binary from time column"); + } + if (!isNull(tsBlockColumnIndex, tsBlockIndex_)) { + lastReadWasNull_ = false; + return curTsBlock_->getColumn(tsBlockColumnIndex)->getBinary(tsBlockIndex_); + } else { + lastReadWasNull_ = true; + return nullptr; + } +} + +Optional IoTDBRpcDataSet::getStringByIndex(int32_t columnIndex) { + int32_t index = getTsBlockColumnIndexForColumnIndex(columnIndex); + return getStringByTsBlockColumnIndex(index); +} + +Optional +IoTDBRpcDataSet::getString(const std::string &columnName) { + int32_t index = getTsBlockColumnIndexForColumnName(columnName); + return getStringByTsBlockColumnIndex(index); +} + +Optional +IoTDBRpcDataSet::getStringByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { + checkRecord(); + if (tsBlockColumnIndex < 0) { + int64_t timestamp = curTsBlock_->getTimeByIndex(tsBlockIndex_); + return std::to_string(timestamp); + } + if (isNull(tsBlockColumnIndex, tsBlockIndex_)) { + lastReadWasNull_ = true; + return Optional::none(); + } + lastReadWasNull_ = false; + return getStringByTsBlockColumnIndexAndDataType( + tsBlockColumnIndex, getDataTypeByTsBlockColumnIndex(tsBlockColumnIndex)); +} + +std::string IoTDBRpcDataSet::getStringByTsBlockColumnIndexAndDataType( + int32_t index, TSDataType::TSDataType tsDataType) { + const auto column = curTsBlock_->getColumn(index); + const TSDataType::TSDataType physicalType = column->getDataType(); + switch (physicalType) { + case TSDataType::BOOLEAN: + return std::to_string(column->getBoolean(tsBlockIndex_)); + case TSDataType::INT32: + return std::to_string(column->getInt(tsBlockIndex_)); + case TSDataType::INT64: + return std::to_string(column->getLong(tsBlockIndex_)); + case TSDataType::TIMESTAMP: { + int64_t value = column->getLong(tsBlockIndex_); + return formatDatetime(timeFormat_, timePrecision_, value, timeZoneId_); + } + case TSDataType::FLOAT: + return std::to_string(column->getFloat(tsBlockIndex_)); + case TSDataType::DOUBLE: + return std::to_string(column->getDouble(tsBlockIndex_)); + case TSDataType::TEXT: + case TSDataType::STRING: + case TSDataType::OBJECT: + case TSDataType::BLOB: + return column->getBinary(tsBlockIndex_)->getStringValue(); + case TSDataType::DATE: { + int32_t value = column->getInt(tsBlockIndex_); + return parseIntToDate(value).toIsoExtendedString(); + } + default: { + if (tsDataType == TSDataType::DATE) { + auto date = getDateByTsBlockColumnIndex(index); + if (date.is_initialized()) { + return date.value().toIsoExtendedString(); + } + } + return ""; + } + } +} + +Optional IoTDBRpcDataSet::getTimestampByIndex(int32_t columnIndex) { + int32_t index = getTsBlockColumnIndexForColumnIndex(columnIndex); + return getTimestampByTsBlockColumnIndex(index); +} + +Optional IoTDBRpcDataSet::getTimestamp(const std::string &columnName) { + int32_t index = getTsBlockColumnIndexForColumnName(columnName); + return getTimestampByTsBlockColumnIndex(index); +} + +Optional +IoTDBRpcDataSet::getTimestampByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { + return getLongByTsBlockColumnIndex(tsBlockColumnIndex); +} + +Optional IoTDBRpcDataSet::getDateByIndex(int32_t columnIndex) { + int32_t index = getTsBlockColumnIndexForColumnIndex(columnIndex); + return getDateByTsBlockColumnIndex(index); +} + +Optional IoTDBRpcDataSet::getDate(const std::string &columnName) { + int32_t index = getTsBlockColumnIndexForColumnName(columnName); + return getDateByTsBlockColumnIndex(index); +} + +Optional +IoTDBRpcDataSet::getDateByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { + checkRecord(); + if (tsBlockColumnIndex < 0) { + throw IoTDBException("Cannot read date from time column"); + } + if (isNull(tsBlockColumnIndex, tsBlockIndex_)) { + lastReadWasNull_ = true; + return Optional::none(); + } + lastReadWasNull_ = false; + const auto column = curTsBlock_->getColumn(tsBlockColumnIndex); + switch (column->getDataType()) { + case TSDataType::INT32: + return parseIntToDate(column->getInt(tsBlockIndex_)); + case TSDataType::INT64: + return parseIntToDate(static_cast(column->getLong(tsBlockIndex_))); + case TSDataType::DOUBLE: + return parseIntToDate( + static_cast(column->getDouble(tsBlockIndex_))); + case TSDataType::FLOAT: + return parseIntToDate( + static_cast(column->getFloat(tsBlockIndex_))); + case TSDataType::TEXT: + case TSDataType::STRING: + case TSDataType::BLOB: + case TSDataType::OBJECT: { + const auto binary = column->getBinary(tsBlockIndex_); + const std::string &text = binary->getStringValue(); + if (text.empty()) { + return IoTDBDate::notADate(); + } + // Server may return DATE as packed int in text or ISO-8601 string. + try { + return parseIntToDate(static_cast(std::stol(text))); + } catch (const std::exception &) { + int year = 0, month = 0, day = 0; + if (std::sscanf(text.c_str(), "%d-%d-%d", &year, &month, &day) == 3) { + return IoTDBDate(year, month, day); + } + throw IoTDBException("Cannot parse DATE value: " + text); + } + } + default: + return parseIntToDate( + getIntByTsBlockColumnIndex(tsBlockColumnIndex).value()); + } +} + +TSDataType::TSDataType +IoTDBRpcDataSet::getDataTypeByIndex(int32_t columnIndex) { + int32_t index = getTsBlockColumnIndexForColumnIndex(columnIndex); + return getDataTypeByTsBlockColumnIndex(index); +} + +TSDataType::TSDataType +IoTDBRpcDataSet::getDataType(const std::string &columnName) { + int32_t index = getTsBlockColumnIndexForColumnName(columnName); + return getDataTypeByTsBlockColumnIndex(index); +} + +TSDataType::TSDataType +IoTDBRpcDataSet::getColumnPhysicalDataType(const std::string &columnName) { + int32_t index = getTsBlockColumnIndexForColumnName(columnName); + if (index < 0) { + return TSDataType::TIMESTAMP; + } + checkRecord(); + return curTsBlock_->getColumn(index)->getDataType(); +} + +int32_t +IoTDBRpcDataSet::getTsBlockColumnIndexForColumnIndex(int32_t columnIndex) { + const int32_t adjusted_index = columnIndex - 1; + if (adjusted_index >= + static_cast(columnIndex2TsBlockColumnIndexList_.size()) || + adjusted_index < 0) { + throw std::out_of_range( + "Index " + std::to_string(adjusted_index) + " out of range [0, " + + std::to_string(columnIndex2TsBlockColumnIndexList_.size()) + ")"); + } + return columnIndex2TsBlockColumnIndexList_[adjusted_index]; +} + +TSDataType::TSDataType +IoTDBRpcDataSet::getDataTypeByTsBlockColumnIndex(int32_t tsBlockColumnIndex) { + if (tsBlockColumnIndex < 0) { + return TSDataType::TIMESTAMP; + } else { + return dataTypeForTsBlockColumn_[tsBlockColumnIndex]; + } +} + +int32_t IoTDBRpcDataSet::findColumn(const std::string &columnName) { + auto it = columnOrdinalMap_.find(columnName); + if (it != columnOrdinalMap_.end()) { + return it->second; + } + return -1; +} + +std::string IoTDBRpcDataSet::findColumnNameByIndex(int32_t columnIndex) { + if (columnIndex <= 0) { + throw IoTDBException("column index should start from 1"); + } + if (columnIndex > static_cast(columnNameList_.size())) { + throw IoTDBException("Column index " + std::to_string(columnIndex) + + " is out of range. Valid range is 0 to " + + std::to_string(columnNameList_.size() - 1)); + } + return columnNameList_[columnIndex - 1]; +} + +int32_t IoTDBRpcDataSet::getTsBlockColumnIndexForColumnName( + const std::string &columnName) { + auto it = columnName2TsBlockColumnIndexMap_.find(columnName); + if (it == columnName2TsBlockColumnIndexMap_.end()) { + throw IoTDBException("unknown column name: " + columnName); + } + return it->second; +} + +void IoTDBRpcDataSet::checkRecord() { + if (queryResultIndex_ > queryResultSize_ || tsBlockIndex_ >= tsBlockSize_ || + queryResult_.empty() || !curTsBlock_) { + throw IoTDBException("no record remains"); + } +} + +int32_t IoTDBRpcDataSet::getValueColumnStartIndex() const { + return getValueColumnNameListIndex(0); +} + +int32_t IoTDBRpcDataSet::getServerColumnCount() const { return columnSize_; } + +int32_t +IoTDBRpcDataSet::getValueColumnNameListIndex(int32_t valueColumnOrdinal) const { + if (valueColumnOrdinal < 0 || valueColumnOrdinal >= columnSize_) { + throw std::out_of_range( + "value column ordinal " + std::to_string(valueColumnOrdinal) + + " out of range [0, " + std::to_string(columnSize_) + ")"); + } + if (!columnNameList_.empty() && columnNameList_[0] == TimestampColumnName) { + return valueColumnOrdinal + 1; + } + return valueColumnOrdinal; +} + +int32_t IoTDBRpcDataSet::getColumnSize() const { + return static_cast(columnNameList_.size()); +} + +const std::vector &IoTDBRpcDataSet::getColumnTypeList() const { + return columnTypeList_; +} + +const std::vector &IoTDBRpcDataSet::getColumnNameList() const { + return columnNameList_; +} + +bool IoTDBRpcDataSet::isClosed() const { return isClosed_; } + +int32_t IoTDBRpcDataSet::getFetchSize() const { return fetchSize_; } + +void IoTDBRpcDataSet::setFetchSize(int32_t fetchSize) { + fetchSize_ = fetchSize; +} + +bool IoTDBRpcDataSet::hasCachedRecord() const { return hasCachedRecord_; } + +void IoTDBRpcDataSet::setHasCachedRecord(bool hasCachedRecord) { + hasCachedRecord_ = hasCachedRecord; +} + +bool IoTDBRpcDataSet::isLastReadWasNull() const { return lastReadWasNull_; } + +int64_t IoTDBRpcDataSet::getCurrentRowTime() const { return time_; } + +bool IoTDBRpcDataSet::isIgnoreTimestamp() const { return ignoreTimestamp_; } + +bool IoTDBRpcDataSet::hasCachedBlock() const { + return curTsBlock_ && tsBlockIndex_ < tsBlockSize_ - 1; +} + +bool IoTDBRpcDataSet::hasCachedByteBuffer() const { + return !queryResult_.empty() && queryResultIndex_ < queryResultSize_; +} diff --git a/iotdb-client/client-cpp/src/rpc/IoTDBRpcDataSet.h b/iotdb-client/client-cpp/src/rpc/IoTDBRpcDataSet.h new file mode 100644 index 0000000000000..2a880ae619212 --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/IoTDBRpcDataSet.h @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef IOTDB_RPC_DATA_SET_H +#define IOTDB_RPC_DATA_SET_H + +#include "Date.h" +#include "IClientRPCService.h" +#include "Optional.h" +#include "TsBlock.h" +#include +#include +#include +#include +#include + +class IoTDBRpcDataSet { +public: + static const int32_t startIndex; + static const std::string TimestampColumnName; + + static const std::string DEFAULT_TIME_FORMAT; + static const std::string TIME_PRECISION; + static const std::string MILLISECOND; + static const std::string MICROSECOND; + static const std::string NANOSECOND; + + IoTDBRpcDataSet(const std::string &sql, + const std::vector &columnNameList, + const std::vector &columnTypeList, + const std::map &columnNameIndex, + bool ignoreTimestamp, bool moreData, int64_t queryId, + int64_t statementId, + std::shared_ptr client, + int64_t sessionId, + const std::vector &queryResult, + int32_t fetchSize, int64_t timeout, const std::string &zoneId, + const std::string &timeFormat); + + ~IoTDBRpcDataSet(); + + bool next(); + void close(bool forceClose = false); + + bool hasCachedBlock() const; + bool hasCachedByteBuffer() const; + + bool isNull(int32_t index, int32_t rowNum); + bool isNullByIndex(int32_t columnIndex); + bool isNullByColumnName(const std::string &columnName); + Optional getBooleanByIndex(int32_t columnIndex); + Optional getBoolean(const std::string &columnName); + Optional getDoubleByIndex(int32_t columnIndex); + Optional getDouble(const std::string &columnName); + Optional getFloatByIndex(int32_t columnIndex); + Optional getFloat(const std::string &columnName); + Optional getIntByIndex(int32_t columnIndex); + Optional getInt(const std::string &columnName); + Optional getLongByIndex(int32_t columnIndex); + Optional getLong(const std::string &columnName); + std::shared_ptr getBinaryByIndex(int32_t columnIndex); + std::shared_ptr getBinary(const std::string &columnName); + Optional getStringByIndex(int32_t columnIndex); + Optional getString(const std::string &columnName); + Optional getTimestampByIndex(int32_t columnIndex); + Optional getTimestamp(const std::string &columnName); + Optional getDateByIndex(int32_t columnIndex); + Optional getDate(const std::string &columnName); + + TSDataType::TSDataType getDataTypeByIndex(int32_t columnIndex); + TSDataType::TSDataType getDataType(const std::string &columnName); + TSDataType::TSDataType + getColumnPhysicalDataType(const std::string &columnName); + int32_t findColumn(const std::string &columnName); + std::string findColumnNameByIndex(int32_t columnIndex); + int32_t getValueColumnStartIndex() const; + int32_t getServerColumnCount() const; + int32_t getValueColumnNameListIndex(int32_t valueColumnOrdinal) const; + int32_t getColumnSize() const; + const std::vector &getColumnTypeList() const; + const std::vector &getColumnNameList() const; + bool isClosed() const; + int32_t getFetchSize() const; + void setFetchSize(int32_t fetchSize); + bool hasCachedRecord() const; + void setHasCachedRecord(bool hasCachedRecord); + bool isLastReadWasNull() const; + int64_t getCurrentRowTime() const; + bool isIgnoreTimestamp() const; + +private: + bool fetchResults(); + void constructOneRow(); + void constructOneTsBlock(); + int32_t getTsBlockColumnIndexForColumnName(const std::string &columnName); + int32_t getTsBlockColumnIndexForColumnIndex(int32_t columnIndex); + void checkRecord(); + TSDataType::TSDataType + getDataTypeByTsBlockColumnIndex(int32_t tsBlockColumnIndex); + Optional getBooleanByTsBlockColumnIndex(int32_t tsBlockColumnIndex); + std::string + getStringByTsBlockColumnIndexAndDataType(int32_t index, + TSDataType::TSDataType tsDataType); + Optional getDoubleByTsBlockColumnIndex(int32_t tsBlockColumnIndex); + Optional getFloatByTsBlockColumnIndex(int32_t tsBlockColumnIndex); + Optional getIntByTsBlockColumnIndex(int32_t tsBlockColumnIndex); + Optional getLongByTsBlockColumnIndex(int32_t tsBlockColumnIndex); + std::shared_ptr + getBinaryByTsBlockColumnIndex(int32_t tsBlockColumnIndex); + Optional + getStringByTsBlockColumnIndex(int32_t tsBlockColumnIndex); + Optional getDateByTsBlockColumnIndex(int32_t tsBlockColumnIndex); + Optional + getTimestampByTsBlockColumnIndex(int32_t tsBlockColumnIndex); + + std::string sql_; + bool isClosed_; + std::shared_ptr client_; + std::vector columnNameList_; + std::vector columnTypeList_; + std::map columnOrdinalMap_; + std::map columnName2TsBlockColumnIndexMap_; + std::vector columnIndex2TsBlockColumnIndexList_; + std::vector dataTypeForTsBlockColumn_; + int32_t fetchSize_; + int64_t timeout_; + bool hasCachedRecord_; + bool lastReadWasNull_; + int32_t columnSize_; + int64_t sessionId_; + int64_t queryId_; + int64_t statementId_; + int64_t time_; + bool ignoreTimestamp_; + bool moreData_; + std::vector queryResult_; + std::shared_ptr curTsBlock_; + int32_t queryResultSize_; + int32_t queryResultIndex_; + int32_t tsBlockSize_; + int32_t tsBlockIndex_; + std::string timeZoneId_; + std::string timeFormat_; + int32_t timeFactor_{1000}; + std::string timePrecision_; +}; + +#endif // IOTDB_RPC_DATA_SET_H diff --git a/iotdb-client/client-cpp/src/rpc/NodesSupplier.cpp b/iotdb-client/client-cpp/src/rpc/NodesSupplier.cpp new file mode 100644 index 0000000000000..5f79c89e14c88 --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/NodesSupplier.cpp @@ -0,0 +1,247 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "NodesSupplier.h" +#include "Session.h" +#include "SessionDataSet.h" +#include +#include +#include + +const std::string NodesSupplier::SHOW_AVAILABLE_URLS_COMMAND = + "SHOW AVAILABLE URLS"; +const std::string NodesSupplier::RUNNING_STATUS = "Running"; +const std::string NodesSupplier::STATUS_COLUMN_NAME = "Status"; +const std::string NodesSupplier::IP_COLUMN_NAME = "RpcAddress"; +const std::string NodesSupplier::PORT_COLUMN_NAME = "RpcPort"; +const std::string NodesSupplier::REMOVING_STATUS = "Removing"; + +const int64_t NodesSupplier::TIMEOUT_IN_MS = 60000; +const int NodesSupplier::THRIFT_DEFAULT_BUFFER_SIZE = 4096; +const int NodesSupplier::THRIFT_MAX_FRAME_SIZE = 1048576; +const int NodesSupplier::CONNECTION_TIMEOUT_IN_MS = 1000; + +TEndPoint RoundRobinPolicy::select(const std::vector &nodes) { + static std::atomic_uint index{0}; + + if (nodes.empty()) { + throw IoTDBException("No available nodes"); + } + + return nodes[index++ % nodes.size()]; +} + +StaticNodesSupplier::StaticNodesSupplier(const std::vector &nodes, + NodeSelectionPolicy policy) + : availableNodes_(nodes), policy_(std::move(policy)) {} + +Optional StaticNodesSupplier::getQueryEndPoint() { + try { + if (availableNodes_.empty()) { + return Optional::none(); + } + return policy_(availableNodes_); + } catch (const IoTDBException &e) { + return Optional::none(); + } +} + +std::vector StaticNodesSupplier::getEndPointList() { + return availableNodes_; +} + +StaticNodesSupplier::~StaticNodesSupplier() = default; + +std::shared_ptr NodesSupplier::create( + const std::vector &endpoints, const std::string &userName, + const std::string &password, bool useSSL, + const std::string &trustCertFilePath, const std::string &zoneId, + int32_t thriftDefaultBufferSize, int32_t thriftMaxFrameSize, + int32_t connectionTimeoutInMs, bool enableRPCCompression, + const std::string &version, std::chrono::milliseconds refreshInterval, + NodeSelectionPolicy policy) { + if (endpoints.empty()) { + return nullptr; + } + auto supplier = std::make_shared( + userName, password, useSSL, trustCertFilePath, zoneId, + thriftDefaultBufferSize, thriftMaxFrameSize, connectionTimeoutInMs, + enableRPCCompression, version, endpoints, policy); + supplier->startBackgroundRefresh(refreshInterval); + return supplier; +} + +NodesSupplier::NodesSupplier( + const std::string &userName, const std::string &password, bool useSSL, + const std::string &trustCertFilePath, const std::string &zoneId, + int32_t thriftDefaultBufferSize, int32_t thriftMaxFrameSize, + int32_t connectionTimeoutInMs, bool enableRPCCompression, + const std::string &version, const std::vector &endpoints, + NodeSelectionPolicy policy) + : userName_(userName), password_(password), zoneId_(zoneId), + thriftDefaultBufferSize_(thriftDefaultBufferSize), + thriftMaxFrameSize_(thriftMaxFrameSize), + connectionTimeoutInMs_(connectionTimeoutInMs), useSSL_(useSSL), + trustCertFilePath_(trustCertFilePath), + enableRPCCompression_(enableRPCCompression), version_(version), + endpoints_(endpoints), selectionPolicy_(policy) { + deduplicateEndpoints(); +} + +std::vector NodesSupplier::getEndPointList() { + std::lock_guard lock(mutex_); + return endpoints_; +} + +TEndPoint NodesSupplier::selectQueryEndpoint() { + std::lock_guard lock(mutex_); + try { + return selectionPolicy_(endpoints_); + } catch (const std::exception &e) { + log_error("NodesSupplier::selectQueryEndpoint exception: %s", e.what()); + throw IoTDBException("NodesSupplier::selectQueryEndpoint exception, " + + std::string(e.what())); + } +} + +Optional NodesSupplier::getQueryEndPoint() { + try { + return selectQueryEndpoint(); + } catch (const IoTDBException &e) { + return Optional::none(); + } +} + +NodesSupplier::~NodesSupplier() { + stopBackgroundRefresh(); + if (client_ != nullptr) { + client_->close(); + } +} + +void NodesSupplier::deduplicateEndpoints() { + std::vector uniqueEndpoints; + uniqueEndpoints.reserve(endpoints_.size()); + for (const auto &endpoint : endpoints_) { + if (std::find(uniqueEndpoints.begin(), uniqueEndpoints.end(), endpoint) == + uniqueEndpoints.end()) { + uniqueEndpoints.push_back(endpoint); + } + } + endpoints_ = std::move(uniqueEndpoints); +} + +void NodesSupplier::startBackgroundRefresh(std::chrono::milliseconds interval) { + isRunning_ = true; + refreshEndpointList(); + refreshThread_ = std::thread([this, interval] { + while (isRunning_) { + refreshEndpointList(); + std::unique_lock cvLock(this->mutex_); + refreshCondition_.wait_for(cvLock, interval, + [this]() { return !isRunning_.load(); }); + } + }); +} + +std::vector NodesSupplier::fetchLatestEndpoints() { + for (const auto &endpoint : endpoints_) { + try { + if (client_ == nullptr) { + client_ = std::make_shared(endpoint); + client_->init(userName_, password_, enableRPCCompression_, useSSL_, + trustCertFilePath_, zoneId_, version_); + } + + auto sessionDataSet = + client_->executeQueryStatement(SHOW_AVAILABLE_URLS_COMMAND); + + uint32_t columnAddrIdx = -1, columnPortIdx = -1; + auto columnNames = sessionDataSet->getColumnNames(); + for (uint32_t i = 0; i < columnNames.size(); i++) { + if (columnNames[i] == IP_COLUMN_NAME) { + columnAddrIdx = i; + } else if (columnNames[i] == PORT_COLUMN_NAME) { + columnPortIdx = i; + } + } + + if (columnAddrIdx == -1 || columnPortIdx == -1) { + throw IoTDBException("Required columns not found in query result."); + } + + std::vector ret; + while (sessionDataSet->hasNext()) { + auto record = sessionDataSet->next(); + std::string ip; + int32_t port = 0; + + if (record->fields.at(columnAddrIdx).stringV.is_initialized()) { + ip = record->fields.at(columnAddrIdx).stringV.value(); + } + if (record->fields.at(columnPortIdx).intV.is_initialized()) { + port = record->fields.at(columnPortIdx).intV.value(); + } + + if (ip == "0.0.0.0") { + log_warn("Skipping invalid node: " + ip + ":" + std::to_string(port)); + continue; + } + TEndPoint newEndpoint; + newEndpoint.ip = ip; + newEndpoint.port = port; + ret.emplace_back(newEndpoint); + } + return ret; // success + } catch (const std::exception &e) { + log_warn("Failed to fetch endpoints from " + endpoint.ip + ":" + + std::to_string(endpoint.port) + " , error=" + e.what()); + client_.reset(); // reset client before retrying next endpoint + continue; // try next endpoint + } + } + log_warn( + "NodesSupplier::fetchLatestEndpoints: SHOW AVAILABLE URLS unavailable, " + "keeping configured endpoints"); + return endpoints_; +} + +void NodesSupplier::refreshEndpointList() { + try { + auto newEndpoints = fetchLatestEndpoints(); + if (newEndpoints.empty()) { + return; + } + + std::lock_guard lock(mutex_); + endpoints_.swap(newEndpoints); + deduplicateEndpoints(); + } catch (const IoTDBException &e) { + log_error(std::string("NodesSupplier::refreshEndpointList failed: ") + + e.what()); + } +} + +void NodesSupplier::stopBackgroundRefresh() noexcept { + if (isRunning_.exchange(false)) { + refreshCondition_.notify_all(); + if (refreshThread_.joinable()) { + refreshThread_.join(); + } + } +} \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/rpc/NodesSupplier.h b/iotdb-client/client-cpp/src/rpc/NodesSupplier.h new file mode 100644 index 0000000000000..57f16524d18dd --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/NodesSupplier.h @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_NODES_SUPPLIER_H +#define IOTDB_NODES_SUPPLIER_H + +#include "Optional.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ThriftConnection.h" + +class TEndPoint; + +class RoundRobinPolicy { +public: + static TEndPoint select(const std::vector &nodes); +}; + +class INodesSupplier { +public: + virtual ~INodesSupplier() = default; + virtual Optional getQueryEndPoint() = 0; + virtual std::vector getEndPointList() = 0; + using NodeSelectionPolicy = + std::function &)>; +}; + +class StaticNodesSupplier : public INodesSupplier { +public: + explicit StaticNodesSupplier( + const std::vector &nodes, + NodeSelectionPolicy policy = RoundRobinPolicy::select); + + Optional getQueryEndPoint() override; + + std::vector getEndPointList() override; + + ~StaticNodesSupplier() override; + +private: + const std::vector availableNodes_; + NodeSelectionPolicy policy_; +}; + +class NodesSupplier : public INodesSupplier { +public: + static const std::string SHOW_AVAILABLE_URLS_COMMAND; + static const std::string RUNNING_STATUS; + static const std::string STATUS_COLUMN_NAME; + static const std::string IP_COLUMN_NAME; + static const std::string PORT_COLUMN_NAME; + static const std::string REMOVING_STATUS; + + static const int64_t TIMEOUT_IN_MS; + static const int THRIFT_DEFAULT_BUFFER_SIZE; + static const int THRIFT_MAX_FRAME_SIZE; + static const int CONNECTION_TIMEOUT_IN_MS; + + static std::shared_ptr create( + const std::vector &endpoints, const std::string &userName, + const std::string &password, bool useSSL = false, + const std::string &trustCertFilePath = "", const std::string &zoneId = "", + int32_t thriftDefaultBufferSize = + ThriftConnection::THRIFT_DEFAULT_BUFFER_SIZE, + int32_t thriftMaxFrameSize = ThriftConnection::THRIFT_MAX_FRAME_SIZE, + int32_t connectionTimeoutInMs = + ThriftConnection::CONNECTION_TIMEOUT_IN_MS, + bool enableRPCCompression = false, const std::string &version = "V_1_0", + std::chrono::milliseconds refreshInterval = + std::chrono::milliseconds(TIMEOUT_IN_MS), + NodeSelectionPolicy policy = RoundRobinPolicy::select); + + NodesSupplier(const std::string &userName, const std::string &password, + bool useSSL, const std::string &trustCertFilePath, + const std::string &zoneId, int32_t thriftDefaultBufferSize, + int32_t thriftMaxFrameSize, int32_t connectionTimeoutInMs, + bool enableRPCCompression, const std::string &version, + const std::vector &endpoints, + NodeSelectionPolicy policy); + + std::vector getEndPointList() override; + + Optional getQueryEndPoint() override; + + ~NodesSupplier() override; + +private: + std::string userName_; + std::string password_; + int32_t thriftDefaultBufferSize_; + int32_t thriftMaxFrameSize_; + int32_t connectionTimeoutInMs_; + bool useSSL_; + std::string trustCertFilePath_; + bool enableRPCCompression_; + std::string version_; + std::string zoneId_; + + std::mutex mutex_; + std::vector endpoints_; + NodeSelectionPolicy selectionPolicy_; + + std::atomic isRunning_{false}; + std::thread refreshThread_; + std::condition_variable refreshCondition_; + + std::shared_ptr client_; + + void deduplicateEndpoints(); + + void startBackgroundRefresh(std::chrono::milliseconds interval); + + std::vector fetchLatestEndpoints(); + + void refreshEndpointList(); + + TEndPoint selectQueryEndpoint(); + + void stopBackgroundRefresh() noexcept; +}; + +#endif \ No newline at end of file diff --git a/iotdb-client/client-cpp/src/rpc/RpcCommon.cpp b/iotdb-client/client-cpp/src/rpc/RpcCommon.cpp new file mode 100644 index 0000000000000..0ec345a1c5ca8 --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/RpcCommon.cpp @@ -0,0 +1,216 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "RpcCommon.h" + +#include +#include +#include + +#include "ThriftConvert.h" +#include "client_types.h" +#include "common_types.h" + +using namespace std; + +RpcUtils::RpcUtils() { + SUCCESS_STATUS = std::make_shared(); + SUCCESS_STATUS->__set_code(TSStatusCode::SUCCESS_STATUS); +} + +void RpcUtils::verifySuccess(const TSStatus &status) { + if (status.code == TSStatusCode::MULTIPLE_ERROR) { + verifySuccess(status.subStatus); + return; + } + if (status.code != TSStatusCode::SUCCESS_STATUS && + status.code != TSStatusCode::REDIRECTION_RECOMMEND) { + throw ExecutionException(to_string(status.code) + ": " + status.message, + statusFromThrift(status)); + } +} + +void RpcUtils::verifySuccessWithRedirection(const TSStatus &status) { + verifySuccess(status); + if (status.__isset.redirectNode) { + throw RedirectException(to_string(status.code) + ": " + status.message, + endpointFromThrift(status.redirectNode)); + } + if (status.__isset.subStatus) { + auto statusSubStatus = status.subStatus; + vector endPointList(statusSubStatus.size()); + int count = 0; + for (const TSStatus &subStatus : statusSubStatus) { + if (subStatus.__isset.redirectNode) { + endPointList[count++] = subStatus.redirectNode; + } else { + TEndPoint endPoint; + endPointList[count++] = endPoint; + } + } + if (!endPointList.empty()) { + throw RedirectException(to_string(status.code) + ": " + status.message, + endpointListFromThrift(endPointList)); + } + } +} + +void RpcUtils::verifySuccessWithRedirectionForMultiDevices( + const TSStatus &status, vector devices) { + verifySuccess(status); + + if (status.code == TSStatusCode::MULTIPLE_ERROR || + status.code == TSStatusCode::REDIRECTION_RECOMMEND) { + map deviceEndPointMap; + const vector &statusSubStatus = status.subStatus; + for (size_t i = 0; i < statusSubStatus.size() && i < devices.size(); i++) { + const TSStatus &subStatus = statusSubStatus[i]; + if (subStatus.__isset.redirectNode) { + deviceEndPointMap.insert(make_pair(devices[i], subStatus.redirectNode)); + } + } + throw RedirectException(to_string(status.code) + ": " + status.message, + endpointMapFromThrift(deviceEndPointMap)); + } + + if (status.__isset.redirectNode) { + throw RedirectException(to_string(status.code) + ": " + status.message, + endpointFromThrift(status.redirectNode)); + } + if (status.__isset.subStatus) { + auto statusSubStatus = status.subStatus; + vector endPointList(statusSubStatus.size()); + int count = 0; + for (const TSStatus &subStatus : statusSubStatus) { + if (subStatus.__isset.redirectNode) { + endPointList[count++] = subStatus.redirectNode; + } else { + TEndPoint endPoint; + endPointList[count++] = endPoint; + } + } + if (!endPointList.empty()) { + throw RedirectException(to_string(status.code) + ": " + status.message, + endpointListFromThrift(endPointList)); + } + } +} + +void RpcUtils::verifySuccess(const vector &statuses) { + for (const TSStatus &status : statuses) { + if (status.code != TSStatusCode::SUCCESS_STATUS) { + vector publicStatuses; + publicStatuses.reserve(statuses.size()); + for (const TSStatus &s : statuses) { + publicStatuses.push_back(statusFromThrift(s)); + } + throw BatchExecutionException(status.message, publicStatuses); + } + } +} + +TSStatus RpcUtils::getStatus(TSStatusCode::TSStatusCode tsStatusCode) { + TSStatus status; + status.__set_code(tsStatusCode); + return status; +} + +TSStatus RpcUtils::getStatus(int code, const string &message) { + TSStatus status; + status.__set_code(code); + status.__set_message(message); + return status; +} + +shared_ptr +RpcUtils::getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode) { + TSStatus status = getStatus(tsStatusCode); + return getTSExecuteStatementResp(status); +} + +shared_ptr +RpcUtils::getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode, + const string &message) { + TSStatus status = getStatus(tsStatusCode, message); + return getTSExecuteStatementResp(status); +} + +shared_ptr +RpcUtils::getTSExecuteStatementResp(const TSStatus &status) { + shared_ptr resp(new TSExecuteStatementResp()); + resp->__set_status(status); + return resp; +} + +shared_ptr +RpcUtils::getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode) { + TSStatus status = getStatus(tsStatusCode); + return getTSFetchResultsResp(status); +} + +shared_ptr +RpcUtils::getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode, + const string &appendMessage) { + TSStatus status = getStatus(tsStatusCode, appendMessage); + return getTSFetchResultsResp(status); +} + +shared_ptr +RpcUtils::getTSFetchResultsResp(const TSStatus &status) { + shared_ptr resp(new TSFetchResultsResp()); + resp->__set_status(status); + return resp; +} + +const string UrlUtils::PORT_SEPARATOR = ":"; +const string UrlUtils::ABB_COLON = "["; + +TEndPoint UrlUtils::parseTEndPointIpv4AndIpv6Url(const string &endPointUrl) { + TEndPoint endPoint; + + if (endPointUrl.empty()) { + return endPoint; + } + + size_t portSeparatorPos = endPointUrl.find_last_of(PORT_SEPARATOR); + + if (portSeparatorPos == string::npos) { + endPoint.__set_ip(endPointUrl); + return endPoint; + } + + string portStr = endPointUrl.substr(portSeparatorPos + 1); + string ip = endPointUrl.substr(0, portSeparatorPos); + + if (ip.find(ABB_COLON) != string::npos) { + if (ip.size() >= 2 && ip.front() == '[' && ip.back() == ']') { + ip = ip.substr(1, ip.size() - 2); + } + } + + try { + int port = stoi(portStr); + endPoint.__set_ip(ip); + endPoint.__set_port(port); + } catch (const exception &) { + endPoint.__set_ip(endPointUrl); + } + + return endPoint; +} diff --git a/iotdb-client/client-cpp/src/rpc/RpcCommon.h b/iotdb-client/client-cpp/src/rpc/RpcCommon.h new file mode 100644 index 0000000000000..0416321d5d1ba --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/RpcCommon.h @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_RPC_COMMON_H +#define IOTDB_RPC_COMMON_H + +#include +#include +#include + +#include "Common.h" + +class TSStatus; +class TSExecuteStatementResp; +class TSFetchResultsResp; +class TEndPoint; + +class RpcUtils { +public: + std::shared_ptr SUCCESS_STATUS; + + RpcUtils(); + + static void verifySuccess(const TSStatus &status); + + static void verifySuccessWithRedirection(const TSStatus &status); + + static void + verifySuccessWithRedirectionForMultiDevices(const TSStatus &status, + std::vector devices); + + static void verifySuccess(const std::vector &statuses); + + static TSStatus getStatus(TSStatusCode::TSStatusCode tsStatusCode); + + static TSStatus getStatus(int code, const std::string &message); + + static std::shared_ptr + getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode); + + static std::shared_ptr + getTSExecuteStatementResp(TSStatusCode::TSStatusCode tsStatusCode, + const std::string &message); + + static std::shared_ptr + getTSExecuteStatementResp(const TSStatus &status); + + static std::shared_ptr + getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode); + + static std::shared_ptr + getTSFetchResultsResp(TSStatusCode::TSStatusCode tsStatusCode, + const std::string &appendMessage); + + static std::shared_ptr + getTSFetchResultsResp(const TSStatus &status); +}; + +class UrlUtils { +private: + static const std::string PORT_SEPARATOR; + static const std::string ABB_COLON; + + UrlUtils() = delete; + ~UrlUtils() = delete; + +public: + static TEndPoint parseTEndPointIpv4AndIpv6Url(const std::string &endPointUrl); +}; + +#endif diff --git a/iotdb-client/client-cpp/src/rpc/SessionConnection.cpp b/iotdb-client/client-cpp/src/rpc/SessionConnection.cpp new file mode 100644 index 0000000000000..bc56da04aac00 --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/SessionConnection.cpp @@ -0,0 +1,680 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "SessionConnection.h" +#include "RpcCommon.h" +#include "SessionImpl.h" +#include "common_types.h" +#include +#include +#include +#include +#include + +#include +#include + +#include "SessionDataSet.h" +#include "SessionDataSetFactory.h" + +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; + +SessionConnection::SessionConnection( + Session::Impl *session_ptr, const TEndPoint &endpoint, + const std::string &zoneId, std::shared_ptr nodeSupplier, + int fetchSize, int maxRetries, int64_t retryInterval, + int64_t connectionTimeout, std::string dialect, std::string db) + : session(session_ptr), zoneId(zoneId), endPoint(endpoint), + availableNodes(std::move(nodeSupplier)), fetchSize(fetchSize), + maxRetryCount(maxRetries), retryIntervalMs(retryInterval), + connectionTimeoutInMs(connectionTimeout), sqlDialect(std::move(dialect)), + database(std::move(db)) { + this->zoneId = zoneId.empty() ? getSystemDefaultZoneId() : zoneId; + endPointList.push_back(endpoint); + init(endPoint, session->useSSL_, session->trustCertFilePath_); +} + +void SessionConnection::close() { + bool needThrowException = false; + std::string errMsg; + session = nullptr; + try { + TSCloseSessionReq req; + req.__set_sessionId(sessionId); + TSStatus tsStatus; + client->closeSession(tsStatus, req); + } catch (const TTransportException &e) { + log_debug(e.what()); + throw IoTDBConnectionException(e.what()); + } catch (const std::exception &e) { + log_debug(e.what()); + errMsg = errMsg + + "Session::close() client->closeSession() error, maybe remote " + "server is down. " + + e.what() + "\n"; + needThrowException = true; + } + + try { + if (transport->isOpen()) { + transport->close(); + } + } catch (const std::exception &e) { + log_debug(e.what()); + errMsg = errMsg + "Session::close() transport->close() error. " + e.what() + + "\n"; + needThrowException = true; + } + + if (needThrowException) { + throw IoTDBException(errMsg); + } +} + +SessionConnection::~SessionConnection() { + try { + close(); + } catch (const std::exception &e) { + log_debug(e.what()); + } +} + +void SessionConnection::init(const TEndPoint &endpoint, bool useSSL, + const std::string &trustCertFilePath) { + if (useSSL) { +#if WITH_SSL + socketFactory_->loadTrustedCertificates(trustCertFilePath.c_str()); + socketFactory_->authenticate(false); + auto sslSocket = socketFactory_->createSocket(endPoint.ip, endPoint.port); + sslSocket->setConnTimeout(connectionTimeoutInMs); + transport = std::make_shared(sslSocket); +#else + throw IoTDBException("SSL/TLS support is not enabled in this build. " + "Please rebuild with -DWITH_SSL=ON flag " + "or use non-SSL connection."); +#endif + } else { + auto socket = std::make_shared(endPoint.ip, endPoint.port); + socket->setConnTimeout(connectionTimeoutInMs); + transport = std::make_shared(socket); + } + if (!transport->isOpen()) { + try { + transport->open(); + } catch (TTransportException &e) { + log_debug(e.what()); + throw IoTDBConnectionException(e.what()); + } + } + if (enableRPCCompression) { + std::shared_ptr protocol(new TCompactProtocol(transport)); + client = std::make_shared(protocol); + } else { + std::shared_ptr protocol(new TBinaryProtocol(transport)); + client = std::make_shared(protocol); + } + + std::map configuration; + configuration["version"] = session->getVersionString(session->version); + configuration["sql_dialect"] = sqlDialect; + if (database != "") { + configuration["db"] = database; + } + TSOpenSessionReq openReq; + openReq.__set_username(session->username_); + openReq.__set_password(session->password_); + openReq.__set_zoneId(zoneId); + openReq.__set_configuration(configuration); + try { + TSOpenSessionResp openResp; + client->openSession(openResp, openReq); + RpcUtils::verifySuccess(openResp.status); + if (session->protocolVersion_ != openResp.serverProtocolVersion) { + if (openResp.serverProtocolVersion == 0) { + // less than 0.10 + throw std::logic_error( + std::string("Protocol not supported, Client version is ") + + std::to_string(session->protocolVersion_) + + ", but Server version is " + + std::to_string(openResp.serverProtocolVersion)); + } + } + + sessionId = openResp.sessionId; + statementId = client->requestStatementId(sessionId); + + if (!zoneId.empty()) { + setTimeZone(zoneId); + } + } catch (const TTransportException &e) { + log_debug(e.what()); + transport->close(); + throw IoTDBConnectionException(e.what()); + } catch (const IoTDBException &e) { + log_debug(e.what()); + transport->close(); + throw; + } catch (const std::exception &e) { + log_debug(e.what()); + transport->close(); + throw; + } +} + +std::unique_ptr +SessionConnection::executeQueryStatement(const std::string &sql, + int64_t timeoutInMs) { + TSExecuteStatementReq req; + req.__set_sessionId(sessionId); + req.__set_statementId(statementId); + req.__set_statement(sql); + req.__set_timeout(timeoutInMs); + req.__set_enableRedirectQuery(true); + + auto result = callWithRetryAndReconnect( + [this, &req]() { + TSExecuteStatementResp resp; + client->executeQueryStatementV2(resp, req); + return resp; + }, + [](const TSExecuteStatementResp &resp) { return resp.status; }); + TSExecuteStatementResp resp = result.getResult(); + if (result.getRetryAttempts() == 0) { + RpcUtils::verifySuccessWithRedirection(resp.status); + } else { + RpcUtils::verifySuccess(resp.status); + } + + return createSessionDataSet( + sql, resp.columns, resp.dataTypeList, resp.columnNameIndexMap, + resp.queryId, statementId, client, sessionId, resp.queryResult, + resp.ignoreTimeStamp, timeoutInMs, resp.moreData, fetchSize, zoneId); +} + +std::unique_ptr +SessionConnection::executeRawDataQuery(const std::vector &paths, + int64_t startTime, int64_t endTime) { + TSRawDataQueryReq req; + req.__set_sessionId(sessionId); + req.__set_statementId(statementId); + req.__set_fetchSize(fetchSize); + req.__set_paths(paths); + req.__set_startTime(startTime); + req.__set_endTime(endTime); + auto result = callWithRetryAndReconnect( + [this, &req]() { + TSExecuteStatementResp resp; + client->executeRawDataQueryV2(resp, req); + return resp; + }, + [](const TSExecuteStatementResp &resp) { return resp.status; }); + TSExecuteStatementResp resp = result.getResult(); + if (result.getRetryAttempts() == 0) { + RpcUtils::verifySuccessWithRedirection(resp.status); + } else { + RpcUtils::verifySuccess(resp.status); + } + return createSessionDataSet("", resp.columns, resp.dataTypeList, + resp.columnNameIndexMap, resp.queryId, + statementId, client, sessionId, resp.queryResult, + resp.ignoreTimeStamp, connectionTimeoutInMs, + resp.moreData, fetchSize, zoneId); +} + +std::unique_ptr +SessionConnection::executeLastDataQuery(const std::vector &paths, + int64_t lastTime) { + TSLastDataQueryReq req; + req.__set_sessionId(sessionId); + req.__set_statementId(statementId); + req.__set_fetchSize(fetchSize); + req.__set_paths(paths); + req.__set_time(lastTime); + + auto result = callWithRetryAndReconnect( + [this, &req]() { + TSExecuteStatementResp resp; + client->executeLastDataQuery(resp, req); + return resp; + }, + [](const TSExecuteStatementResp &resp) { return resp.status; }); + TSExecuteStatementResp resp = result.getResult(); + if (result.getRetryAttempts() == 0) { + RpcUtils::verifySuccessWithRedirection(resp.status); + } else { + RpcUtils::verifySuccess(resp.status); + } + return createSessionDataSet("", resp.columns, resp.dataTypeList, + resp.columnNameIndexMap, resp.queryId, + statementId, client, sessionId, resp.queryResult, + resp.ignoreTimeStamp, connectionTimeoutInMs, + resp.moreData, fetchSize, zoneId); +} + +void SessionConnection::executeNonQueryStatement(const std::string &sql) { + TSExecuteStatementReq req; + req.__set_sessionId(sessionId); + req.__set_statementId(statementId); + req.__set_statement(sql); + req.__set_timeout( + 0); // 0 means no timeout. This value keep consistent to JAVA SDK. + TSExecuteStatementResp resp; + try { + client->executeUpdateStatementV2(resp, req); + RpcUtils::verifySuccess(resp.status); + } catch (const TTransportException &e) { + log_debug(e.what()); + throw IoTDBConnectionException(e.what()); + } catch (const IoTDBException &e) { + log_debug(e.what()); + throw; + } catch (const std::exception &e) { + throw IoTDBException(e.what()); + } +} + +const TEndPoint &SessionConnection::getEndPoint() { return endPoint; } + +void SessionConnection::setTimeZone(const std::string &newZoneId) { + TSSetTimeZoneReq req; + req.__set_sessionId(sessionId); + req.__set_timeZone(newZoneId); + + try { + TSStatus tsStatus; + client->setTimeZone(tsStatus, req); + zoneId = newZoneId; + } catch (const TException &e) { + throw IoTDBConnectionException(e.what()); + } +} + +std::string SessionConnection::getSystemDefaultZoneId() { + time_t ts = 0; + struct tm tmv {}; +#if defined(_WIN64) || defined(WIN32) || defined(_WIN32) + localtime_s(&tmv, &ts); +#else + localtime_r(&ts, &tmv); +#endif + char zoneStr[32]; + strftime(zoneStr, sizeof(zoneStr), "%z", &tmv); + return zoneStr; +} + +bool SessionConnection::reconnect() { + bool reconnect = false; + for (int i = 1; i <= 3; i++) { + if (transport != nullptr) { + transport->close(); + endPointList = std::move(availableNodes->getEndPointList()); + int currHostIndex = rand() % endPointList.size(); + int tryHostNum = 0; + for (int j = currHostIndex; j < endPointList.size(); j++) { + if (tryHostNum == endPointList.size()) { + break; + } + this->endPoint = endPointList[j]; + if (j == endPointList.size() - 1) { + j = -1; + } + tryHostNum++; + try { + init(this->endPoint, this->session->useSSL_, + this->session->trustCertFilePath_); + reconnect = true; + } catch (const IoTDBConnectionException &e) { + log_warn( + "The current node may have been down, connection exception: %s", + e.what()); + continue; + } catch (std::exception &e) { + log_warn("login in failed, because %s", e.what()); + } + break; + } + } + if (reconnect) { + session->removeBrokenSessionConnection(shared_from_this()); + session->defaultEndPoint_ = this->endPoint; + session->defaultSessionConnection_ = shared_from_this(); + session->endPointToSessionConnection.insert( + std::make_pair(this->endPoint, shared_from_this())); + } + } + return reconnect; +} + +void SessionConnection::insertStringRecord( + const TSInsertStringRecordReq &request) { + auto rpc = [this, request]() { + return this->insertStringRecordInternal(request); + }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertRecord(const TSInsertRecordReq &request) { + auto rpc = [this, request]() { return this->insertRecordInternal(request); }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertStringRecords( + const TSInsertStringRecordsReq &request) { + auto rpc = [this, request]() { + return this->insertStringRecordsInternal(request); + }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertRecords(const TSInsertRecordsReq &request) { + auto rpc = [this, request]() { return this->insertRecordsInternal(request); }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertRecordsOfOneDevice( + TSInsertRecordsOfOneDeviceReq request) { + auto rpc = [this, request]() { + return this->insertRecordsOfOneDeviceInternal(request); + }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertStringRecordsOfOneDevice( + TSInsertStringRecordsOfOneDeviceReq request) { + auto rpc = [this, request]() { + return this->insertStringRecordsOfOneDeviceInternal(request); + }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertTablet(TSInsertTabletReq request) { + auto rpc = [this, request]() { return this->insertTabletInternal(request); }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::insertTablets(TSInsertTabletsReq request) { + auto rpc = [this, request]() { return this->insertTabletsInternal(request); }; + callWithRetryAndVerifyWithRedirection(rpc); +} + +void SessionConnection::testInsertStringRecord( + TSInsertStringRecordReq &request) { + auto rpc = [this, &request]() { + request.sessionId = sessionId; + TSStatus ret; + client->testInsertStringRecord(ret, request); + return ret; + }; + auto status = callWithRetryAndReconnect(rpc).getResult(); + RpcUtils::verifySuccess(status); +} + +void SessionConnection::testInsertTablet(TSInsertTabletReq &request) { + auto rpc = [this, &request]() { + request.sessionId = sessionId; + TSStatus ret; + client->testInsertTablet(ret, request); + return ret; + }; + auto status = callWithRetryAndReconnect(rpc).getResult(); + RpcUtils::verifySuccess(status); +} + +void SessionConnection::testInsertRecords(TSInsertRecordsReq &request) { + auto rpc = [this, &request]() { + request.sessionId = sessionId; + TSStatus ret; + client->testInsertRecords(ret, request); + return ret; + }; + auto status = callWithRetryAndReconnect(rpc).getResult(); + RpcUtils::verifySuccess(status); +} + +void SessionConnection::deleteTimeseries( + const std::vector &paths) { + auto rpc = [this, &paths]() { + TSStatus ret; + client->deleteTimeseries(ret, sessionId, paths); + return ret; + }; + callWithRetryAndVerify(rpc); +} + +void SessionConnection::deleteData(const TSDeleteDataReq &request) { + auto rpc = [this, request]() { return this->deleteDataInternal(request); }; + callWithRetryAndVerify(rpc); +} + +void SessionConnection::setStorageGroup(const std::string &storageGroupId) { + auto rpc = [this, &storageGroupId]() { + TSStatus ret; + client->setStorageGroup(ret, sessionId, storageGroupId); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::deleteStorageGroups( + const std::vector &storageGroups) { + auto rpc = [this, &storageGroups]() { + TSStatus ret; + client->deleteStorageGroups(ret, sessionId, storageGroups); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::createTimeseries(TSCreateTimeseriesReq &req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->createTimeseries(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::createMultiTimeseries(TSCreateMultiTimeseriesReq &req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->createMultiTimeseries(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::createAlignedTimeseries( + TSCreateAlignedTimeseriesReq &req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->createAlignedTimeseries(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +TSGetTimeZoneResp SessionConnection::getTimeZone() { + auto rpc = [this]() { + TSGetTimeZoneResp resp; + client->getTimeZone(resp, sessionId); + zoneId = resp.timeZone; + return resp; + }; + auto ret = callWithRetryAndReconnect( + rpc, [](const TSGetTimeZoneResp &resp) { return resp.status; }); + RpcUtils::verifySuccess(ret.getResult().status); + return ret.result; +} + +void SessionConnection::setTimeZone(TSSetTimeZoneReq &req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->setTimeZone(ret, req); + zoneId = req.timeZone; + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::createSchemaTemplate(TSCreateSchemaTemplateReq req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->createSchemaTemplate(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::setSchemaTemplate(TSSetSchemaTemplateReq req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->setSchemaTemplate(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::unsetSchemaTemplate(TSUnsetSchemaTemplateReq req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->unsetSchemaTemplate(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::appendSchemaTemplate(TSAppendSchemaTemplateReq req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->appendSchemaTemplate(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +void SessionConnection::pruneSchemaTemplate(TSPruneSchemaTemplateReq req) { + auto rpc = [this, &req]() { + TSStatus ret; + req.sessionId = sessionId; + client->pruneSchemaTemplate(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect(rpc); + RpcUtils::verifySuccess(ret.getResult()); +} + +TSQueryTemplateResp +SessionConnection::querySchemaTemplate(TSQueryTemplateReq req) { + auto rpc = [this, &req]() { + TSQueryTemplateResp ret; + req.sessionId = sessionId; + client->querySchemaTemplate(ret, req); + return ret; + }; + auto ret = callWithRetryAndReconnect( + rpc, [](const TSQueryTemplateResp &resp) { return resp.status; }); + RpcUtils::verifySuccess(ret.getResult().status); + return ret.getResult(); +} + +TSStatus +SessionConnection::insertStringRecordInternal(TSInsertStringRecordReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertStringRecord(ret, request); + return ret; +} + +TSStatus SessionConnection::insertRecordInternal(TSInsertRecordReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertRecord(ret, request); + return ret; +} + +TSStatus SessionConnection::insertStringRecordsInternal( + TSInsertStringRecordsReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertStringRecords(ret, request); + return ret; +} + +TSStatus SessionConnection::insertRecordsInternal(TSInsertRecordsReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertRecords(ret, request); + return ret; +} + +TSStatus SessionConnection::insertRecordsOfOneDeviceInternal( + TSInsertRecordsOfOneDeviceReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertRecordsOfOneDevice(ret, request); + return ret; +} + +TSStatus SessionConnection::insertStringRecordsOfOneDeviceInternal( + TSInsertStringRecordsOfOneDeviceReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertStringRecordsOfOneDevice(ret, request); + return ret; +} + +TSStatus SessionConnection::insertTabletInternal(TSInsertTabletReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertTablet(ret, request); + return ret; +} + +TSStatus SessionConnection::insertTabletsInternal(TSInsertTabletsReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->insertTablets(ret, request); + return ret; +} + +TSStatus SessionConnection::deleteDataInternal(TSDeleteDataReq request) { + request.sessionId = sessionId; + TSStatus ret; + client->deleteData(ret, request); + return ret; +} diff --git a/iotdb-client/client-cpp/src/rpc/SessionConnection.h b/iotdb-client/client-cpp/src/rpc/SessionConnection.h new file mode 100644 index 0000000000000..08525ae799272 --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/SessionConnection.h @@ -0,0 +1,364 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_SESSIONCONNECTION_H +#define IOTDB_SESSIONCONNECTION_H + +#include +#include +#include +#include +#if WITH_SSL +#include +#endif + +#include "Common.h" +#include "IClientRPCService.h" +#include "NodesSupplier.h" +#include "RpcCommon.h" +#include "Session.h" +#include "SessionConfig.h" +#include "common_types.h" + +class SessionDataSet; + +class SessionConnection + : public std::enable_shared_from_this { +public: + SessionConnection( + Session::Impl *session_ptr, const TEndPoint &endpoint, + const std::string &zoneId, std::shared_ptr nodeSupplier, + int fetchSize = iotdb::session::DEFAULT_FETCH_SIZE, + int maxRetries = iotdb::session::DEFAULT_MAX_RETRIES, + int64_t retryInterval = iotdb::session::DEFAULT_RETRY_DELAY_MS, + int64_t connectionTimeoutMs = iotdb::session::DEFAULT_CONNECT_TIMEOUT_MS, + std::string dialect = "tree", std::string db = ""); + + ~SessionConnection(); + + void setTimeZone(const std::string &newZoneId); + + const TEndPoint &getEndPoint(); + + void init(const TEndPoint &endpoint, bool useSSL, + const std::string &trustCertFilePath); + + void insertStringRecord(const TSInsertStringRecordReq &request); + + void insertRecord(const TSInsertRecordReq &request); + + void insertStringRecords(const TSInsertStringRecordsReq &request); + + void insertRecords(const TSInsertRecordsReq &request); + + void insertRecordsOfOneDevice(TSInsertRecordsOfOneDeviceReq request); + + void + insertStringRecordsOfOneDevice(TSInsertStringRecordsOfOneDeviceReq request); + + void insertTablet(TSInsertTabletReq request); + + void insertTablets(TSInsertTabletsReq request); + + void testInsertStringRecord(TSInsertStringRecordReq &request); + + void testInsertTablet(TSInsertTabletReq &request); + + void testInsertRecords(TSInsertRecordsReq &request); + + void deleteTimeseries(const std::vector &paths); + + void deleteData(const TSDeleteDataReq &request); + + void setStorageGroup(const std::string &storageGroupId); + + void deleteStorageGroups(const std::vector &storageGroups); + + void createTimeseries(TSCreateTimeseriesReq &req); + + void createMultiTimeseries(TSCreateMultiTimeseriesReq &req); + + void createAlignedTimeseries(TSCreateAlignedTimeseriesReq &req); + + TSGetTimeZoneResp getTimeZone(); + + void setTimeZone(TSSetTimeZoneReq &req); + + void createSchemaTemplate(TSCreateSchemaTemplateReq req); + + void setSchemaTemplate(TSSetSchemaTemplateReq req); + + void unsetSchemaTemplate(TSUnsetSchemaTemplateReq req); + + void appendSchemaTemplate(TSAppendSchemaTemplateReq req); + + void pruneSchemaTemplate(TSPruneSchemaTemplateReq req); + + TSQueryTemplateResp querySchemaTemplate(TSQueryTemplateReq req); + + std::unique_ptr + executeRawDataQuery(const std::vector &paths, int64_t startTime, + int64_t endTime); + + std::unique_ptr + executeLastDataQuery(const std::vector &paths, int64_t lastTime); + + void executeNonQueryStatement(const std::string &sql); + + std::unique_ptr + executeQueryStatement(const std::string &sql, int64_t timeoutInMs = -1); + + std::shared_ptr getSessionClient() { return client; } + + friend class Session; + +private: + void close(); + std::string getSystemDefaultZoneId(); + bool reconnect(); + + template struct RetryResult { + T result; + std::exception_ptr exception; + int retryAttempts; + + RetryResult(T r, std::exception_ptr e, int a) + : result(r), exception(e), retryAttempts(a) {} + + int getRetryAttempts() const { return retryAttempts; } + T getResult() const { return result; } + std::exception_ptr getException() const { return exception; } + }; + + template + void callWithRetryAndVerifyWithRedirection(std::function rpc); + + template + void callWithRetryAndVerifyWithRedirectionForMultipleDevices( + std::function rpc, const std::vector &deviceIds); + + template + RetryResult callWithRetryAndVerify(std::function rpc); + + template RetryResult callWithRetry(std::function rpc); + + template + RetryResult callWithRetryAndReconnect(RpcFunc rpc); + + template + RetryResult callWithRetryAndReconnect(RpcFunc rpc, + StatusGetter statusGetter); + + template + RetryResult callWithRetryAndReconnect(RpcFunc rpc, ShouldRetry shouldRetry, + ForceReconnect forceReconnect); + + TSStatus insertStringRecordInternal(TSInsertStringRecordReq request); + + TSStatus insertRecordInternal(TSInsertRecordReq request); + + TSStatus insertStringRecordsInternal(TSInsertStringRecordsReq request); + + TSStatus insertRecordsInternal(TSInsertRecordsReq request); + + TSStatus + insertRecordsOfOneDeviceInternal(TSInsertRecordsOfOneDeviceReq request); + + TSStatus insertStringRecordsOfOneDeviceInternal( + TSInsertStringRecordsOfOneDeviceReq request); + + TSStatus insertTabletInternal(TSInsertTabletReq request); + + TSStatus insertTabletsInternal(TSInsertTabletsReq request); + + TSStatus deleteDataInternal(TSDeleteDataReq request); +#if WITH_SSL + std::shared_ptr socketFactory_ = + std::make_shared(); +#endif + std::shared_ptr transport; + std::shared_ptr client; + Session::Impl *session; + int64_t sessionId; + int64_t statementId; + int64_t connectionTimeoutInMs; + bool enableRPCCompression = false; + std::string zoneId; + TEndPoint endPoint; + std::vector endPointList; + std::shared_ptr availableNodes; + int fetchSize; + int maxRetryCount; + int64_t retryIntervalMs; + std::string sqlDialect; + std::string database; + int timeFactor = 1000; +}; + +template +SessionConnection::RetryResult +SessionConnection::callWithRetry(std::function rpc) { + std::exception_ptr lastException = nullptr; + TSStatus status; + int i; + for (i = 0; i <= maxRetryCount; i++) { + if (i > 0) { + lastException = nullptr; + status = TSStatus(); + try { + std::this_thread::sleep_for(std::chrono::milliseconds(retryIntervalMs)); + } catch (const std::exception &e) { + break; + } + if (!reconnect()) { + continue; + } + } + + try { + status = rpc(); + if (status.__isset.needRetry && status.needRetry) { + continue; + } + break; + } catch (...) { + lastException = std::current_exception(); + } + } + return {status, lastException, i}; +} + +template +void SessionConnection::callWithRetryAndVerifyWithRedirection( + std::function rpc) { + auto result = callWithRetry(rpc); + + auto status = result.getResult(); + if (result.getRetryAttempts() == 0) { + RpcUtils::verifySuccessWithRedirection(status); + } else { + RpcUtils::verifySuccess(status); + } + + if (result.getException()) { + throw IoTDBConnectionException( + extractExceptionMessage(result.getException())); + } +} + +template +void SessionConnection::callWithRetryAndVerifyWithRedirectionForMultipleDevices( + std::function rpc, const std::vector &deviceIds) { + auto result = callWithRetry(rpc); + auto status = result.getResult(); + if (result.getRetryAttempts() == 0) { + RpcUtils::verifySuccessWithRedirectionForMultiDevices(status, deviceIds); + } else { + RpcUtils::verifySuccess(status); + } + if (result.getException()) { + throw IoTDBConnectionException( + extractExceptionMessage(result.getException())); + } + result.exception = nullptr; +} + +template +SessionConnection::RetryResult +SessionConnection::callWithRetryAndVerify(std::function rpc) { + auto result = callWithRetry(rpc); + RpcUtils::verifySuccess(result.getResult()); + if (result.getException()) { + throw IoTDBConnectionException( + extractExceptionMessage(result.getException())); + } + return result; +} + +template +SessionConnection::RetryResult +SessionConnection::callWithRetryAndReconnect(RpcFunc rpc) { + return callWithRetryAndReconnect( + rpc, + [](const TSStatus &status) { + return status.__isset.needRetry && status.needRetry; + }, + [](const TSStatus &status) { + return status.code == TSStatusCode::PLAN_FAILED_NETWORK_PARTITION; + }); +} + +template +SessionConnection::RetryResult +SessionConnection::callWithRetryAndReconnect(RpcFunc rpc, + StatusGetter statusGetter) { + auto shouldRetry = [&statusGetter](const T &t) { + auto status = statusGetter(t); + return status.__isset.needRetry && status.needRetry; + }; + auto forceReconnect = [&statusGetter](const T &t) { + auto status = statusGetter(t); + return status.code == TSStatusCode::PLAN_FAILED_NETWORK_PARTITION; + ; + }; + return callWithRetryAndReconnect(rpc, shouldRetry, forceReconnect); +} + +template +SessionConnection::RetryResult SessionConnection::callWithRetryAndReconnect( + RpcFunc rpc, ShouldRetry shouldRetry, ForceReconnect forceReconnect) { + std::exception_ptr lastException = nullptr; + T result; + int retryAttempt; + for (retryAttempt = 0; retryAttempt <= maxRetryCount; retryAttempt++) { + try { + result = rpc(); + lastException = nullptr; + } catch (...) { + result = T(); + lastException = std::current_exception(); + } + + if (!shouldRetry(result)) { + return {result, lastException, retryAttempt}; + } + + if (lastException != nullptr || + std::find(availableNodes->getEndPointList().begin(), + availableNodes->getEndPointList().end(), + this->endPoint) == availableNodes->getEndPointList().end() || + forceReconnect(result)) { + reconnect(); + } + + try { + std::this_thread::sleep_for(std::chrono::milliseconds(retryIntervalMs)); + } catch (const std::exception &e) { + log_debug("Thread was interrupted during retry " + + std::to_string(retryAttempt) + " with wait time " + + std::to_string(retryIntervalMs) + " ms. Exiting retry loop."); + break; + } + } + + return {result, lastException, retryAttempt}; +} + +#endif diff --git a/iotdb-client/client-cpp/src/rpc/SessionDataSetFactory.h b/iotdb-client/client-cpp/src/rpc/SessionDataSetFactory.h new file mode 100644 index 0000000000000..3b7429d3e3d0e --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/SessionDataSetFactory.h @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_SESSION_DATA_SET_FACTORY_H +#define IOTDB_SESSION_DATA_SET_FACTORY_H + +#include +#include +#include +#include +#include + +#include "IClientRPCService.h" + +class SessionDataSet; + +std::unique_ptr createSessionDataSet( + const std::string &sql, const std::vector &columnNameList, + const std::vector &columnTypeList, + const std::map &columnNameIndex, int64_t queryId, + int64_t statementId, std::shared_ptr client, + int64_t sessionId, const std::vector &queryResult, + bool ignoreTimestamp, int64_t timeout, bool moreData, int32_t fetchSize, + const std::string &zoneId); + +#endif diff --git a/iotdb-client/client-cpp/src/rpc/SessionImpl.h b/iotdb-client/client-cpp/src/rpc/SessionImpl.h new file mode 100644 index 0000000000000..6ccba7dd84056 --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/SessionImpl.h @@ -0,0 +1,230 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_SESSION_IMPL_H +#define IOTDB_SESSION_IMPL_H + +#include +#include +#include +#include +#include +#include + +#include "AbstractSessionBuilder.h" +#include "Common.h" +#include "Endpoint.h" +#include "NodesSupplier.h" +#include "Session.h" +#include "SessionConnection.h" +#include "ThriftConvert.h" +#include "client_types.h" +#include "common_types.h" + +class Session::Impl { +public: + std::string host_; + int rpcPort_ = 6667; + bool useSSL_ = false; + std::string trustCertFilePath_; + std::vector nodeUrls_; + std::string username_ = "root"; + std::string password_ = "root"; + const TSProtocolVersion::type protocolVersion_ = + TSProtocolVersion::IOTDB_SERVICE_PROTOCOL_V3; + bool isClosed_ = true; + std::string zoneId_; + int fetchSize_ = iotdb::session::DEFAULT_FETCH_SIZE; + static const int DEFAULT_TIMEOUT_MS = 0; + int connectTimeoutMs_ = iotdb::session::DEFAULT_CONNECT_TIMEOUT_MS; + Version::Version version = Version::V_1_0; + std::string sqlDialect_ = "tree"; + std::string database_; + bool enableAutoFetch_ = true; + bool enableRedirection_ = true; + std::shared_ptr nodesSupplier_; + std::shared_ptr defaultSessionConnection_; + + TEndPoint defaultEndPoint_; + + struct TEndPointHash { + size_t operator()(const TEndPoint &endpoint) const { + return std::hash()(endpoint.ip) ^ + std::hash()(endpoint.port); + } + }; + + struct TEndPointEqual { + bool operator()(const TEndPoint &lhs, const TEndPoint &rhs) const { + return lhs.ip == rhs.ip && lhs.port == rhs.port; + } + }; + + using EndPointSessionMap = + std::unordered_map, + TEndPointHash, TEndPointEqual>; + EndPointSessionMap endPointToSessionConnection; + std::unordered_map deviceIdToEndpoint; + + void removeBrokenSessionConnection( + std::shared_ptr sessionConnection); + + static bool checkSorted(const Tablet &tablet); + static bool checkSorted(const std::vector ×); + static void sortTablet(Tablet &tablet); + static void sortIndexByTimestamp(int *index, std::vector ×tamps, + int length); + + void appendValues(std::string &buffer, const char *value, int size); + void putValuesIntoBuffer(const std::vector &types, + const std::vector &values, std::string &buf); + int8_t getDataTypeNumber(TSDataType::TSDataType type); + + struct TsCompare { + std::vector ×tamps; + explicit TsCompare(std::vector &inTimestamps) + : timestamps(inTimestamps) {} + bool operator()(int i, int j) { return timestamps[i] < timestamps[j]; } + }; + + std::string getVersionString(Version::Version version); + + void initZoneId(); + void initNodesSupplier( + const std::vector &nodeUrls = std::vector()); + void initDefaultSessionConnection(); + + template + void insertByGroup( + std::unordered_map, T> &insertGroup, + InsertConsumer insertConsumer); + + template + void insertOnce( + std::unordered_map, T> &insertGroup, + InsertConsumer insertConsumer); + + void insertStringRecordsWithLeaderCache( + std::vector deviceIds, std::vector times, + std::vector> measurementsList, + std::vector> valuesList, bool isAligned); + + void insertRecordsWithLeaderCache( + std::vector deviceIds, std::vector times, + std::vector> measurementsList, + const std::vector> &typesList, + std::vector> valuesList, bool isAligned); + + void insertTabletsWithLeaderCache( + std::unordered_map &tablets, bool sorted, + bool isAligned); + + std::shared_ptr getQuerySessionConnection(); + std::shared_ptr getSessionConnection(std::string deviceId); + + void handleQueryRedirection(TEndPoint endPoint); + void handleRedirection(const std::string &deviceId, TEndPoint endPoint); + + static void buildInsertTabletReq(TSInsertTabletReq &request, Tablet &tablet, + bool sorted); + void insertTablet(TSInsertTabletReq request); +}; + +template +void Session::Impl::insertByGroup( + std::unordered_map, T> &insertGroup, + InsertConsumer insertConsumer) { + std::vector> futures; + + for (auto &entry : insertGroup) { + auto connection = entry.first; + auto &req = entry.second; + futures.emplace_back(std::async(std::launch::async, [=, &req]() mutable { + try { + insertConsumer(connection, req); + } catch (const RedirectException &e) { + for (const auto &deviceEndPoint : e.deviceEndPointMap) { + handleRedirection(deviceEndPoint.first, + endpointToThrift(deviceEndPoint.second)); + } + } catch (const IoTDBConnectionException &) { + if (endPointToSessionConnection.size() > 1) { + removeBrokenSessionConnection(connection); + try { + insertConsumer(defaultSessionConnection_, req); + } catch (const RedirectException &) { + } + } else { + throw; + } + } catch (const std::exception &e) { + log_debug(e.what()); + throw IoTDBException(e.what()); + } + })); + } + + std::string errorMessages; + for (auto &f : futures) { + try { + f.get(); + } catch (const IoTDBConnectionException &) { + throw; + } catch (const std::exception &e) { + if (!errorMessages.empty()) { + errorMessages += ";"; + } + errorMessages += e.what(); + } + } + + if (!errorMessages.empty()) { + throw StatementExecutionException(errorMessages); + } +} + +template +void Session::Impl::insertOnce( + std::unordered_map, T> &insertGroup, + InsertConsumer insertConsumer) { + auto connection = insertGroup.begin()->first; + auto req = insertGroup.begin()->second; + try { + insertConsumer(connection, req); + } catch (const RedirectException &e) { + for (const auto &deviceEndPoint : e.deviceEndPointMap) { + handleRedirection(deviceEndPoint.first, + endpointToThrift(deviceEndPoint.second)); + } + } catch (const IoTDBConnectionException &) { + if (endPointToSessionConnection.size() > 1) { + removeBrokenSessionConnection(connection); + try { + insertConsumer(defaultSessionConnection_, req); + } catch (const RedirectException &) { + } + } else { + throw; + } + } catch (const std::exception &e) { + log_debug(e.what()); + throw IoTDBException(e.what()); + } +} + +#endif diff --git a/iotdb-client/client-cpp/src/rpc/ThriftConnection.cpp b/iotdb-client/client-cpp/src/rpc/ThriftConnection.cpp new file mode 100644 index 0000000000000..832d664cd94e2 --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/ThriftConnection.cpp @@ -0,0 +1,187 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "ThriftConnection.h" +#include +#include +#include +#include +#include +#include +#include + +#include "RpcCommon.h" +#include "SessionDataSet.h" +#include "SessionDataSetFactory.h" + +using namespace apache::thrift; +using namespace apache::thrift::protocol; +using namespace apache::thrift::transport; + +const int ThriftConnection::THRIFT_DEFAULT_BUFFER_SIZE = 4096; +const int ThriftConnection::THRIFT_MAX_FRAME_SIZE = 1048576; +const int ThriftConnection::CONNECTION_TIMEOUT_IN_MS = 1000; + +ThriftConnection::ThriftConnection(const TEndPoint &endPoint, + int thriftDefaultBufferSize, + int thriftMaxFrameSize, + int connectionTimeoutInMs, int fetchSize) + : endPoint_(endPoint), thriftDefaultBufferSize_(thriftDefaultBufferSize), + thriftMaxFrameSize_(thriftMaxFrameSize), + connectionTimeoutInMs_(connectionTimeoutInMs), fetchSize_(fetchSize) {} + +ThriftConnection::~ThriftConnection() = default; + +void ThriftConnection::initZoneId() { + if (!zoneId_.empty()) { + return; + } + + time_t ts = 0; + struct tm tmv {}; +#if defined(_WIN64) || defined(WIN32) || defined(_WIN32) + localtime_s(&tmv, &ts); +#else + localtime_r(&ts, &tmv); +#endif + + char zoneStr[32]; + strftime(zoneStr, sizeof(zoneStr), "%z", &tmv); + zoneId_ = zoneStr; +} + +void ThriftConnection::init(const std::string &username, + const std::string &password, + bool enableRPCCompression, bool useSSL, + const std::string &trustCertFilePath, + const std::string &zoneId, + const std::string &version) { + if (useSSL) { +#if WITH_SSL + socketFactory_->loadTrustedCertificates(trustCertFilePath.c_str()); + socketFactory_->authenticate(false); + auto sslSocket = socketFactory_->createSocket(endPoint_.ip, endPoint_.port); + sslSocket->setConnTimeout(connectionTimeoutInMs_); + transport_ = std::make_shared(sslSocket); +#else + throw IoTDBException("SSL/TLS support is not enabled in this build. " + "Please rebuild with -DWITH_SSL=ON flag " + "or use non-SSL connection."); +#endif + } else { + auto socket = std::make_shared(endPoint_.ip, endPoint_.port); + socket->setConnTimeout(connectionTimeoutInMs_); + transport_ = std::make_shared(socket); + } + if (!transport_->isOpen()) { + try { + transport_->open(); + } catch (TTransportException &e) { + throw IoTDBConnectionException(e.what()); + } + } + if (zoneId.empty()) { + initZoneId(); + } else { + this->zoneId_ = zoneId; + } + + if (enableRPCCompression) { + std::shared_ptr protocol( + new TCompactProtocol(transport_)); + client_ = std::make_shared(protocol); + } else { + std::shared_ptr protocol(new TBinaryProtocol(transport_)); + client_ = std::make_shared(protocol); + } + + std::map configuration; + configuration["version"] = version; + TSOpenSessionReq openReq; + openReq.__set_username(username); + openReq.__set_password(password); + openReq.__set_zoneId(this->zoneId_); + openReq.__set_configuration(configuration); + try { + TSOpenSessionResp openResp; + client_->openSession(openResp, openReq); + RpcUtils::verifySuccess(openResp.status); + sessionId_ = openResp.sessionId; + statementId_ = client_->requestStatementId(sessionId_); + } catch (const TTransportException &e) { + transport_->close(); + throw IoTDBConnectionException(e.what()); + } catch (const IoTDBException &e) { + transport_->close(); + throw IoTDBException(e.what()); + } catch (const std::exception &e) { + transport_->close(); + throw IoTDBException(e.what()); + } +} + +std::unique_ptr +ThriftConnection::executeQueryStatement(const std::string &sql, + int64_t timeoutInMs) { + TSExecuteStatementReq req; + req.__set_sessionId(sessionId_); + req.__set_statementId(statementId_); + req.__set_statement(sql); + req.__set_timeout(timeoutInMs); + TSExecuteStatementResp resp; + try { + client_->executeQueryStatementV2(resp, req); + RpcUtils::verifySuccess(resp.status); + } catch (const TTransportException &e) { + throw IoTDBConnectionException(e.what()); + } catch (const IoTDBException &e) { + throw IoTDBConnectionException(e.what()); + } catch (const std::exception &e) { + throw IoTDBException(e.what()); + } + std::shared_ptr queryDataSet( + new TSQueryDataSet(resp.queryDataSet)); + return createSessionDataSet( + "", resp.columns, resp.dataTypeList, resp.columnNameIndexMap, + resp.queryId, statementId_, client_, sessionId_, resp.queryResult, + resp.ignoreTimeStamp, connectionTimeoutInMs_, resp.moreData, fetchSize_, + zoneId_); +} + +void ThriftConnection::close() { + try { + if (client_) { + TSCloseSessionReq req; + req.__set_sessionId(sessionId_); + TSStatus tsStatus; + client_->closeSession(tsStatus, req); + } + } catch (const TTransportException &e) { + throw IoTDBConnectionException(e.what()); + } catch (const std::exception &e) { + throw IoTDBConnectionException(e.what()); + } + + try { + if (transport_->isOpen()) { + transport_->close(); + } + } catch (const std::exception &e) { + throw IoTDBConnectionException(e.what()); + } +} diff --git a/iotdb-client/client-cpp/src/rpc/ThriftConnection.h b/iotdb-client/client-cpp/src/rpc/ThriftConnection.h new file mode 100644 index 0000000000000..d79a63fb1be01 --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/ThriftConnection.h @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_THRIFTCONNECTION_H +#define IOTDB_THRIFTCONNECTION_H + +#include +#if WITH_SSL +#include +#endif +#include "IClientRPCService.h" +#include "SessionConfig.h" + +class SessionDataSet; + +class ThriftConnection { +public: + static const int THRIFT_DEFAULT_BUFFER_SIZE; + static const int THRIFT_MAX_FRAME_SIZE; + static const int CONNECTION_TIMEOUT_IN_MS; + + explicit ThriftConnection( + const TEndPoint &endPoint, + int thriftDefaultBufferSize = THRIFT_DEFAULT_BUFFER_SIZE, + int thriftMaxFrameSize = THRIFT_MAX_FRAME_SIZE, + int connectionTimeoutInMs = CONNECTION_TIMEOUT_IN_MS, + int fetchSize = iotdb::session::DEFAULT_FETCH_SIZE); + + ~ThriftConnection(); + + void init(const std::string &username, const std::string &password, + bool enableRPCCompression = false, bool useSSL = false, + const std::string &trustCertFilePath = "", + const std::string &zoneId = std::string(), + const std::string &version = "V_1_0"); + + std::unique_ptr + executeQueryStatement(const std::string &sql, int64_t timeoutInMs = -1); + + void close(); + +private: + TEndPoint endPoint_; + + int thriftDefaultBufferSize_; + int thriftMaxFrameSize_; + int connectionTimeoutInMs_; + int fetchSize_; + +#if WITH_SSL + std::shared_ptr socketFactory_ = + std::make_shared(); +#endif + std::shared_ptr transport_; + std::shared_ptr client_; + int64_t sessionId_{}; + int64_t statementId_{}; + std::string zoneId_; + int timeFactor_{}; + + void initZoneId(); +}; + +#endif diff --git a/iotdb-client/client-cpp/src/rpc/ThriftConvert.cpp b/iotdb-client/client-cpp/src/rpc/ThriftConvert.cpp new file mode 100644 index 0000000000000..714be579f8489 --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/ThriftConvert.cpp @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "ThriftConvert.h" + +#include "common_types.h" + +Status statusFromThrift(const TSStatus &tsStatus) { + Status status; + status.code = tsStatus.code; + status.message = tsStatus.message; + return status; +} + +Endpoint endpointFromThrift(const TEndPoint &endPoint) { + Endpoint endpoint; + endpoint.host = endPoint.ip; + endpoint.port = endPoint.port; + return endpoint; +} + +TEndPoint endpointToThrift(const Endpoint &endpoint) { + TEndPoint endPoint; + endPoint.__set_ip(endpoint.host); + endPoint.__set_port(endpoint.port); + return endPoint; +} + +std::map endpointMapFromThrift( + const std::map &deviceEndPointMap) { + std::map result; + for (const auto &entry : deviceEndPointMap) { + result.emplace(entry.first, endpointFromThrift(entry.second)); + } + return result; +} + +std::vector +endpointListFromThrift(const std::vector &endPointList) { + std::vector result; + result.reserve(endPointList.size()); + for (const auto &endPoint : endPointList) { + result.push_back(endpointFromThrift(endPoint)); + } + return result; +} diff --git a/iotdb-client/client-cpp/src/rpc/ThriftConvert.h b/iotdb-client/client-cpp/src/rpc/ThriftConvert.h new file mode 100644 index 0000000000000..effb3da097344 --- /dev/null +++ b/iotdb-client/client-cpp/src/rpc/ThriftConvert.h @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef IOTDB_THRIFT_CONVERT_H +#define IOTDB_THRIFT_CONVERT_H + +#include +#include +#include + +#include "Endpoint.h" +#include "Status.h" + +class TEndPoint; +class TSStatus; + +Status statusFromThrift(const TSStatus &tsStatus); +Endpoint endpointFromThrift(const TEndPoint &endPoint); +TEndPoint endpointToThrift(const Endpoint &endpoint); + +std::map endpointMapFromThrift( + const std::map &deviceEndPointMap); +std::vector +endpointListFromThrift(const std::vector &endPointList); + +#endif diff --git a/iotdb-client/client-cpp/src/session/Column.cpp b/iotdb-client/client-cpp/src/session/Column.cpp new file mode 100644 index 0000000000000..d01921598c606 --- /dev/null +++ b/iotdb-client/client-cpp/src/session/Column.cpp @@ -0,0 +1,459 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "Column.h" +#include "ColumnDecoder.h" + +TimeColumn::TimeColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &values) + : arrayOffset_(arrayOffset), positionCount_(positionCount), + values_(values) { + if (arrayOffset < 0) + throw IoTDBException("arrayOffset is negative"); + if (positionCount < 0) + throw IoTDBException("positionCount is negative"); + if (static_cast(values.size()) - arrayOffset < positionCount) { + throw IoTDBException("values length is less than positionCount"); + } +} + +TSDataType::TSDataType TimeColumn::getDataType() const { + return TSDataType::INT64; +} +ColumnEncoding TimeColumn::getEncoding() const { + return ColumnEncoding::Int64Array; +} + +int64_t TimeColumn::getLong(int32_t position) const { + return values_[position + arrayOffset_]; +} + +bool TimeColumn::mayHaveNull() const { return false; } +bool TimeColumn::isNull(int32_t position) const { return false; } +std::vector TimeColumn::isNulls() const { return {}; } + +int32_t TimeColumn::getPositionCount() const { return positionCount_; } + +int64_t TimeColumn::getStartTime() const { return values_[arrayOffset_]; } +int64_t TimeColumn::getEndTime() const { + return values_[positionCount_ + arrayOffset_ - 1]; +} + +const std::vector &TimeColumn::getTimes() const { return values_; } +std::vector TimeColumn::getLongs() const { return getTimes(); } + +BinaryColumn::BinaryColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &valueIsNull, + const std::vector> &values) + : arrayOffset_(arrayOffset), positionCount_(positionCount), + valueIsNull_(valueIsNull), values_(values) { + if (arrayOffset < 0) + throw IoTDBException("arrayOffset is negative"); + if (positionCount < 0) + throw IoTDBException("positionCount is negative"); + if (static_cast(values.size()) - arrayOffset < positionCount) { + throw IoTDBException("values length is less than positionCount"); + } + if (!valueIsNull.empty() && + static_cast(valueIsNull.size()) - arrayOffset < positionCount) { + throw IoTDBException("isNull length is less than positionCount"); + } +} + +TSDataType::TSDataType BinaryColumn::getDataType() const { + return TSDataType::TSDataType::TEXT; +} +ColumnEncoding BinaryColumn::getEncoding() const { + return ColumnEncoding::BinaryArray; +} + +std::shared_ptr BinaryColumn::getBinary(int32_t position) const { + return values_[position + arrayOffset_]; +} + +std::vector> BinaryColumn::getBinaries() const { + return values_; +} + +bool BinaryColumn::mayHaveNull() const { return !valueIsNull_.empty(); } + +bool BinaryColumn::isNull(int32_t position) const { + return !valueIsNull_.empty() && valueIsNull_[position + arrayOffset_]; +} + +std::vector BinaryColumn::isNulls() const { + if (!valueIsNull_.empty()) + return valueIsNull_; + + std::vector result(positionCount_, false); + return result; +} + +int32_t BinaryColumn::getPositionCount() const { return positionCount_; } + +IntColumn::IntColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &valueIsNull, + const std::vector &values) + : arrayOffset_(arrayOffset), positionCount_(positionCount), + valueNull_(valueIsNull), values_(values) { + if (arrayOffset < 0) + throw IoTDBException("arrayOffset is negative"); + if (positionCount < 0) + throw IoTDBException("positionCount is negative"); + if (static_cast(values.size()) - arrayOffset < positionCount) { + throw IoTDBException("values length is less than positionCount"); + } + if (!valueIsNull.empty() && + static_cast(valueIsNull.size()) - arrayOffset < positionCount) { + throw IoTDBException("isNull length is less than positionCount"); + } +} + +TSDataType::TSDataType IntColumn::getDataType() const { + return TSDataType::INT32; +} +ColumnEncoding IntColumn::getEncoding() const { + return ColumnEncoding::Int32Array; +} + +int32_t IntColumn::getInt(int32_t position) const { + return values_[position + arrayOffset_]; +} + +int64_t IntColumn::getLong(int32_t position) const { + return values_[position + arrayOffset_]; +} + +float IntColumn::getFloat(int32_t position) const { + return values_[position + arrayOffset_]; +} + +double IntColumn::getDouble(int32_t position) const { + return values_[position + arrayOffset_]; +} + +std::vector IntColumn::getInts() const { return values_; } + +bool IntColumn::mayHaveNull() const { return !valueNull_.empty(); } + +bool IntColumn::isNull(int32_t position) const { + return !valueNull_.empty() && valueNull_[position + arrayOffset_]; +} + +std::vector IntColumn::isNulls() const { + if (!valueNull_.empty()) + return valueNull_; + + std::vector result(positionCount_, false); + return result; +} + +int32_t IntColumn::getPositionCount() const { return positionCount_; } + +FloatColumn::FloatColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &valueIsNull, + const std::vector &values) + : arrayOffset_(arrayOffset), positionCount_(positionCount), + valueIsNull_(valueIsNull), values_(values) { + if (arrayOffset < 0) + throw IoTDBException("arrayOffset is negative"); + if (positionCount < 0) + throw IoTDBException("positionCount is negative"); + if (static_cast(values.size()) - arrayOffset < positionCount) { + throw IoTDBException("values length is less than positionCount"); + } + if (!valueIsNull.empty() && + static_cast(valueIsNull.size()) - arrayOffset < positionCount) { + throw IoTDBException("isNull length is less than positionCount"); + } +} + +TSDataType::TSDataType FloatColumn::getDataType() const { + return TSDataType::TSDataType::FLOAT; +} +ColumnEncoding FloatColumn::getEncoding() const { + return ColumnEncoding::Int32Array; +} + +float FloatColumn::getFloat(int32_t position) const { + return values_[position + arrayOffset_]; +} + +double FloatColumn::getDouble(int32_t position) const { + return values_[position + arrayOffset_]; +} + +std::vector FloatColumn::getFloats() const { return values_; } + +bool FloatColumn::mayHaveNull() const { return !valueIsNull_.empty(); } + +bool FloatColumn::isNull(int32_t position) const { + return !valueIsNull_.empty() && valueIsNull_[position + arrayOffset_]; +} + +std::vector FloatColumn::isNulls() const { + if (!valueIsNull_.empty()) + return valueIsNull_; + + std::vector result(positionCount_, false); + return result; +} + +int32_t FloatColumn::getPositionCount() const { return positionCount_; } + +LongColumn::LongColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &valueIsNull, + const std::vector &values) + : arrayOffset_(arrayOffset), positionCount_(positionCount), + valueIsNull_(valueIsNull), values_(values) { + if (arrayOffset < 0) + throw IoTDBException("arrayOffset is negative"); + if (positionCount < 0) + throw IoTDBException("positionCount is negative"); + if (static_cast(values.size()) - arrayOffset < positionCount) { + throw IoTDBException("values length is less than positionCount"); + } + if (!valueIsNull.empty() && + static_cast(valueIsNull.size()) - arrayOffset < positionCount) { + throw IoTDBException("isNull length is less than positionCount"); + } +} + +TSDataType::TSDataType LongColumn::getDataType() const { + return TSDataType::TSDataType::INT64; +} +ColumnEncoding LongColumn::getEncoding() const { + return ColumnEncoding::Int64Array; +} + +int64_t LongColumn::getLong(int32_t position) const { + return values_[position + arrayOffset_]; +} + +double LongColumn::getDouble(int32_t position) const { + return values_[position + arrayOffset_]; +} + +std::vector LongColumn::getLongs() const { return values_; } + +bool LongColumn::mayHaveNull() const { return !valueIsNull_.empty(); } + +bool LongColumn::isNull(int32_t position) const { + return !valueIsNull_.empty() && valueIsNull_[position + arrayOffset_]; +} + +std::vector LongColumn::isNulls() const { + if (!valueIsNull_.empty()) + return valueIsNull_; + + std::vector result(positionCount_, false); + return result; +} + +int32_t LongColumn::getPositionCount() const { return positionCount_; } + +DoubleColumn::DoubleColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &valueIsNull, + const std::vector &values) + : arrayOffset_(arrayOffset), positionCount_(positionCount), + valueIsNull_(valueIsNull), values_(values) { + if (arrayOffset < 0) + throw IoTDBException("arrayOffset is negative"); + if (positionCount < 0) + throw IoTDBException("positionCount is negative"); + if (static_cast(values.size()) - arrayOffset < positionCount) { + throw IoTDBException("values length is less than positionCount"); + } + if (!valueIsNull.empty() && + static_cast(valueIsNull.size()) - arrayOffset < positionCount) { + throw IoTDBException("isNull length is less than positionCount"); + } +} + +TSDataType::TSDataType DoubleColumn::getDataType() const { + return TSDataType::TSDataType::DOUBLE; +} +ColumnEncoding DoubleColumn::getEncoding() const { + return ColumnEncoding::Int64Array; +} + +double DoubleColumn::getDouble(int32_t position) const { + return values_[position + arrayOffset_]; +} + +std::vector DoubleColumn::getDoubles() const { return values_; } + +bool DoubleColumn::mayHaveNull() const { return !valueIsNull_.empty(); } + +bool DoubleColumn::isNull(int32_t position) const { + return !valueIsNull_.empty() && valueIsNull_[position + arrayOffset_]; +} + +std::vector DoubleColumn::isNulls() const { + if (!valueIsNull_.empty()) + return valueIsNull_; + + std::vector result(positionCount_, false); + return result; +} + +int32_t DoubleColumn::getPositionCount() const { return positionCount_; } + +BooleanColumn::BooleanColumn(int32_t arrayOffset, int32_t positionCount, + const std::vector &valueIsNull, + const std::vector &values) + : arrayOffset_(arrayOffset), positionCount_(positionCount), + valueIsNull_(valueIsNull), values_(values) { + if (arrayOffset < 0) + throw IoTDBException("arrayOffset is negative"); + if (positionCount < 0) + throw IoTDBException("positionCount is negative"); + if (static_cast(values.size()) - arrayOffset < positionCount) { + throw IoTDBException("values length is less than positionCount"); + } + if (!valueIsNull.empty() && + static_cast(valueIsNull.size()) - arrayOffset < positionCount) { + throw IoTDBException("isNull length is less than positionCount"); + } +} + +TSDataType::TSDataType BooleanColumn::getDataType() const { + return TSDataType::TSDataType::BOOLEAN; +} +ColumnEncoding BooleanColumn::getEncoding() const { + return ColumnEncoding::ByteArray; +} + +bool BooleanColumn::getBoolean(int32_t position) const { + return values_[position + arrayOffset_]; +} + +std::vector BooleanColumn::getBooleans() const { return values_; } + +bool BooleanColumn::mayHaveNull() const { return !valueIsNull_.empty(); } + +bool BooleanColumn::isNull(int32_t position) const { + return !valueIsNull_.empty() && valueIsNull_[position + arrayOffset_]; +} + +std::vector BooleanColumn::isNulls() const { + if (!valueIsNull_.empty()) + return valueIsNull_; + + std::vector result(positionCount_, false); + return result; +} + +int32_t BooleanColumn::getPositionCount() const { return positionCount_; } + +RunLengthEncodedColumn::RunLengthEncodedColumn(std::shared_ptr value, + int32_t positionCount) + : value_(value), positionCount_(positionCount) { + if (!value) + throw IoTDBException("value is null"); + if (value->getPositionCount() != 1) { + throw IoTDBException("Expected value to contain a single position"); + } + if (positionCount < 0) + throw IoTDBException("positionCount is negative"); +} + +std::shared_ptr RunLengthEncodedColumn::getValue() const { + return value_; +} + +TSDataType::TSDataType RunLengthEncodedColumn::getDataType() const { + return value_->getDataType(); +} +ColumnEncoding RunLengthEncodedColumn::getEncoding() const { + return ColumnEncoding::Rle; +} + +bool RunLengthEncodedColumn::getBoolean(int32_t position) const { + return value_->getBoolean(0); +} + +int32_t RunLengthEncodedColumn::getInt(int32_t position) const { + return value_->getInt(0); +} + +int64_t RunLengthEncodedColumn::getLong(int32_t position) const { + return value_->getLong(0); +} + +float RunLengthEncodedColumn::getFloat(int32_t position) const { + return value_->getFloat(0); +} + +double RunLengthEncodedColumn::getDouble(int32_t position) const { + return value_->getDouble(0); +} + +std::shared_ptr +RunLengthEncodedColumn::getBinary(int32_t position) const { + return value_->getBinary(0); +} + +std::vector RunLengthEncodedColumn::getBooleans() const { + bool v = value_->getBoolean(0); + return std::vector(positionCount_, v); +} + +std::vector RunLengthEncodedColumn::getInts() const { + int32_t v = value_->getInt(0); + return std::vector(positionCount_, v); +} + +std::vector RunLengthEncodedColumn::getLongs() const { + int64_t v = value_->getLong(0); + return std::vector(positionCount_, v); +} + +std::vector RunLengthEncodedColumn::getFloats() const { + float v = value_->getFloat(0); + return std::vector(positionCount_, v); +} + +std::vector RunLengthEncodedColumn::getDoubles() const { + double v = value_->getDouble(0); + return std::vector(positionCount_, v); +} + +std::vector> +RunLengthEncodedColumn::getBinaries() const { + auto v = value_->getBinary(0); + return std::vector>(positionCount_, v); +} + +bool RunLengthEncodedColumn::mayHaveNull() const { + return value_->mayHaveNull(); +} + +bool RunLengthEncodedColumn::isNull(int32_t position) const { + return value_->isNull(0); +} + +std::vector RunLengthEncodedColumn::isNulls() const { + bool v = value_->isNull(0); + return std::vector(positionCount_, v); +} + +int32_t RunLengthEncodedColumn::getPositionCount() const { + return positionCount_; +} diff --git a/iotdb-client/client-cpp/src/session/ColumnDecoder.cpp b/iotdb-client/client-cpp/src/session/ColumnDecoder.cpp new file mode 100644 index 0000000000000..f455ac5a8ce5a --- /dev/null +++ b/iotdb-client/client-cpp/src/session/ColumnDecoder.cpp @@ -0,0 +1,207 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "ColumnDecoder.h" + +#include "Column.h" + +std::vector deserializeNullIndicators(MyStringBuffer &buffer, + int32_t positionCount) { + uint8_t mayHaveNullByte = buffer.getChar(); + + bool mayHaveNull = mayHaveNullByte != 0; + if (!mayHaveNull) { + return {}; + } + + return deserializeBooleanArray(buffer, positionCount); +} + +std::vector deserializeBooleanArray(MyStringBuffer &buffer, + int32_t size) { + const int32_t packedSize = (size + 7) / 8; + std::vector packedBytes(packedSize); + for (int i = 0; i < packedSize; i++) { + packedBytes[i] = buffer.getChar(); + } + + std::vector output(size); + int currentByte = 0; + const int fullGroups = size & ~0b111; + + for (int pos = 0; pos < fullGroups; pos += 8) { + const uint8_t b = packedBytes[currentByte++]; + output[pos + 0] = (b & 0b10000000) != 0; + output[pos + 1] = (b & 0b01000000) != 0; + output[pos + 2] = (b & 0b00100000) != 0; + output[pos + 3] = (b & 0b00010000) != 0; + output[pos + 4] = (b & 0b00001000) != 0; + output[pos + 5] = (b & 0b00000100) != 0; + output[pos + 6] = (b & 0b00000010) != 0; + output[pos + 7] = (b & 0b00000001) != 0; + } + + if ((size & 0b111) > 0) { + const uint8_t b = packedBytes.back(); + uint8_t mask = 0b10000000; + + for (int pos = fullGroups; pos < size; pos++) { + output[pos] = (b & mask) != 0; + mask >>= 1; + } + } + + return output; +} + +std::unique_ptr +BaseColumnDecoder::readColumn(MyStringBuffer &buffer, + TSDataType::TSDataType dataType, + int32_t positionCount) { + return nullptr; +} + +std::unique_ptr +Int32ArrayColumnDecoder::readColumn(MyStringBuffer &buffer, + TSDataType::TSDataType dataType, + int32_t positionCount) { + auto nullIndicators = deserializeNullIndicators(buffer, positionCount); + + switch (dataType) { + case TSDataType::INT32: + case TSDataType::DATE: { + std::vector intValues(positionCount); + for (int32_t i = 0; i < positionCount; i++) { + if (!nullIndicators.empty() && nullIndicators[i]) + continue; + intValues[i] = buffer.getInt(); + } + return std::unique_ptr( + new IntColumn(0, positionCount, nullIndicators, intValues)); + } + case TSDataType::FLOAT: { + std::vector floatValues(positionCount); + for (int32_t i = 0; i < positionCount; i++) { + if (!nullIndicators.empty() && nullIndicators[i]) + continue; + floatValues[i] = buffer.getFloat(); + } + return std::unique_ptr( + new FloatColumn(0, positionCount, nullIndicators, floatValues)); + } + default: + throw IoTDBException("Invalid data type for Int32ArrayColumnDecoder"); + } +} + +std::unique_ptr +Int64ArrayColumnDecoder::readColumn(MyStringBuffer &buffer, + TSDataType::TSDataType dataType, + int32_t positionCount) { + auto nullIndicators = deserializeNullIndicators(buffer, positionCount); + + switch (dataType) { + case TSDataType::INT64: + case TSDataType::TIMESTAMP: { + std::vector values(positionCount); + for (int32_t i = 0; i < positionCount; i++) { + if (!nullIndicators.empty() && nullIndicators[i]) + continue; + values[i] = buffer.getInt64(); + } + return std::unique_ptr( + new LongColumn(0, positionCount, nullIndicators, values)); + } + case TSDataType::DOUBLE: { + std::vector values(positionCount); + for (int32_t i = 0; i < positionCount; i++) { + if (!nullIndicators.empty() && nullIndicators[i]) + continue; + values[i] = buffer.getDouble(); + } + return std::unique_ptr( + new DoubleColumn(0, positionCount, nullIndicators, values)); + } + default: + throw IoTDBException("Invalid data type for Int64ArrayColumnDecoder"); + } +} + +std::unique_ptr +ByteArrayColumnDecoder::readColumn(MyStringBuffer &buffer, + TSDataType::TSDataType dataType, + int32_t positionCount) { + if (dataType != TSDataType::BOOLEAN) { + throw IoTDBException("Invalid data type for ByteArrayColumnDecoder"); + } + + auto nullIndicators = deserializeNullIndicators(buffer, positionCount); + auto values = deserializeBooleanArray(buffer, positionCount); + return std::unique_ptr( + new BooleanColumn(0, positionCount, nullIndicators, values)); +} + +std::unique_ptr +BinaryArrayColumnDecoder::readColumn(MyStringBuffer &buffer, + TSDataType::TSDataType dataType, + int32_t positionCount) { + if (dataType != TSDataType::TEXT) { + throw IoTDBException("Invalid data type for BinaryArrayColumnDecoder"); + } + + auto nullIndicators = deserializeNullIndicators(buffer, positionCount); + std::vector> values(positionCount); + + for (int32_t i = 0; i < positionCount; i++) { + if (!nullIndicators.empty() && nullIndicators[i]) + continue; + + int32_t length = buffer.getInt(); + if (length < 0) { + throw IoTDBException("BinaryArrayColumnDecoder: negative TEXT length"); + } + + std::vector value(length); + for (int32_t j = 0; j < length; j++) { + value[j] = buffer.getChar(); + } + + values[i] = std::make_shared(value); + } + + return std::unique_ptr( + new BinaryColumn(0, positionCount, nullIndicators, values)); +} + +std::unique_ptr +RunLengthColumnDecoder::readColumn(MyStringBuffer &buffer, + TSDataType::TSDataType dataType, + int32_t positionCount) { + uint8_t encodingByte = buffer.getChar(); + + auto columnEncoding = static_cast(encodingByte); + auto decoder = getColumnDecoder(columnEncoding); + + auto column = decoder->readColumn(buffer, dataType, 1); + if (!column) { + throw IoTDBException("Failed to read inner column"); + } + return std::unique_ptr( + new RunLengthEncodedColumn(move(column), positionCount)); +} diff --git a/iotdb-client/client-cpp/src/session/Common.cpp b/iotdb-client/client-cpp/src/session/Common.cpp new file mode 100644 index 0000000000000..6f449ea840af8 --- /dev/null +++ b/iotdb-client/client-cpp/src/session/Common.cpp @@ -0,0 +1,315 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "Common.h" + +#include +#include +#include + +LogLevelType LOG_LEVEL = LEVEL_WARN; + +std::string extractExceptionMessage(const std::exception &exception) { + const char *what = exception.what(); + if (what != nullptr) { + std::string message(what); + if (!message.empty() && message != "std::exception") { + return message; + } + } + return std::string("Unhandled exception type: ") + typeid(exception).name(); +} + +std::string extractExceptionMessage(const std::exception_ptr &exceptionPtr) { + if (exceptionPtr == nullptr) { + return "Unknown exception"; + } + try { + std::rethrow_exception(exceptionPtr); + } catch (const std::exception &exception) { + return extractExceptionMessage(exception); + } catch (...) { + return "Unknown non-std exception"; + } +} + +std::string getTimePrecision(int32_t timeFactor) { + if (timeFactor >= 1000000) + return "us"; + if (timeFactor >= 1000) + return "ms"; + return "s"; +} + +std::string formatDatetime(const std::string &format, + const std::string &precision, int64_t timestamp, + const std::string &zoneId) { + (void)precision; + (void)zoneId; + std::time_t time = static_cast(timestamp); + std::tm *tm = std::localtime(&time); + char buffer[80]; + strftime(buffer, sizeof(buffer), format.c_str(), tm); + return std::string(buffer); +} + +std::tm convertToTimestamp(int64_t value, int32_t timeFactor) { + std::time_t time = static_cast(value / timeFactor); + return *std::localtime(&time); +} + +TSDataType::TSDataType getDataTypeByStr(const std::string &typeStr) { + if (typeStr == "BOOLEAN") + return TSDataType::BOOLEAN; + if (typeStr == "INT32") + return TSDataType::INT32; + if (typeStr == "INT64") + return TSDataType::INT64; + if (typeStr == "FLOAT") + return TSDataType::FLOAT; + if (typeStr == "DOUBLE") + return TSDataType::DOUBLE; + if (typeStr == "TEXT") + return TSDataType::TEXT; + if (typeStr == "TIMESTAMP") + return TSDataType::TIMESTAMP; + if (typeStr == "DATE") + return TSDataType::DATE; + if (typeStr == "BLOB") + return TSDataType::BLOB; + if (typeStr == "STRING") + return TSDataType::STRING; + if (typeStr == "OBJECT") + return TSDataType::OBJECT; + return TSDataType::UNKNOWN; +} + +std::tm int32ToDate(int32_t value) { + std::time_t time = static_cast(value) * 86400; + return *std::localtime(&time); +} + +MyStringBuffer::MyStringBuffer() : pos(0) { checkBigEndian(); } + +MyStringBuffer::MyStringBuffer(const std::string &str) : str(str), pos(0) { + checkBigEndian(); +} + +void MyStringBuffer::reserve(size_t n) { str.reserve(n); } + +void MyStringBuffer::clear() { + str.clear(); + pos = 0; +} + +bool MyStringBuffer::hasRemaining() { return pos < str.size(); } + +int MyStringBuffer::getInt() { return *(int *)getOrderedByte(4); } + +IoTDBDate MyStringBuffer::getDate() { return parseIntToDate(getInt()); } + +int64_t MyStringBuffer::getInt64() { +#ifdef ARCH32 + const char *buf_addr = getOrderedByte(8); + if (reinterpret_cast(buf_addr) % 4 == 0) { + return *(int64_t *)buf_addr; + } else { + char tmp_buf[8]; + memcpy(tmp_buf, buf_addr, 8); + return *(int64_t *)tmp_buf; + } +#else + return *(int64_t *)getOrderedByte(8); +#endif +} + +float MyStringBuffer::getFloat() { return *(float *)getOrderedByte(4); } + +double MyStringBuffer::getDouble() { +#ifdef ARCH32 + const char *buf_addr = getOrderedByte(8); + if (reinterpret_cast(buf_addr) % 4 == 0) { + return *(double *)buf_addr; + } else { + char tmp_buf[8]; + memcpy(tmp_buf, buf_addr, 8); + return *(double *)tmp_buf; + } +#else + return *(double *)getOrderedByte(8); +#endif +} + +char MyStringBuffer::getChar() { + if (pos >= str.size()) { + throw IoTDBException( + "MyStringBuffer::getChar: read past end (pos=" + std::to_string(pos) + + ", size=" + std::to_string(str.size()) + ")"); + } + return str[pos++]; +} + +bool MyStringBuffer::getBool() { return getChar() == 1; } + +std::string MyStringBuffer::getString() { + const int lenInt = getInt(); + if (lenInt < 0) { + throw IoTDBException("MyStringBuffer::getString: negative length"); + } + const size_t len = static_cast(lenInt); + if (pos > str.size() || len > str.size() - pos) { + throw IoTDBException( + "MyStringBuffer::getString: length exceeds buffer (pos=" + + std::to_string(pos) + ", len=" + std::to_string(len) + + ", size=" + std::to_string(str.size()) + ")"); + } + const size_t tmpPos = pos; + pos += len; + return str.substr(tmpPos, len); +} + +void MyStringBuffer::putInt(int ins) { putOrderedByte((char *)&ins, 4); } + +void MyStringBuffer::putDate(IoTDBDate date) { + putInt(parseDateExpressionToInt(date)); +} + +void MyStringBuffer::putInt64(int64_t ins) { putOrderedByte((char *)&ins, 8); } + +void MyStringBuffer::putFloat(float ins) { putOrderedByte((char *)&ins, 4); } + +void MyStringBuffer::putDouble(double ins) { putOrderedByte((char *)&ins, 8); } + +void MyStringBuffer::putChar(char ins) { str += ins; } + +void MyStringBuffer::putBool(bool ins) { + char tmp = ins ? 1 : 0; + str += tmp; +} + +void MyStringBuffer::putString(const std::string &ins) { + putInt((int)(ins.size())); + str += ins; +} + +void MyStringBuffer::concat(const std::string &ins) { str.append(ins); } + +void MyStringBuffer::checkBigEndian() { + static int chk = 0x0201; + isBigEndian = (0x01 != *(char *)(&chk)); +} + +const char *MyStringBuffer::getOrderedByte(size_t len) { + if (pos > str.size() || len > str.size() - pos) { + throw IoTDBException("MyStringBuffer::getOrderedByte: read past end (pos=" + + std::to_string(pos) + ", len=" + std::to_string(len) + + ", size=" + std::to_string(str.size()) + ")"); + } + const char *p = nullptr; + if (isBigEndian) { + p = str.c_str() + pos; + } else { + const char *tmp = str.c_str(); + for (size_t i = pos; i < pos + len; i++) { + numericBuf[pos + len - 1 - i] = tmp[i]; + } + p = numericBuf; + } + pos += len; + return p; +} + +void MyStringBuffer::putOrderedByte(char *buf, int len) { + if (isBigEndian) { + str.append(buf, len); + } else { + for (int i = len - 1; i > -1; i--) { + str += buf[i]; + } + } +} + +BitMap::BitMap(size_t size) { resize(size); } + +void BitMap::resize(size_t size) { + this->size = size; + this->bits.resize((size >> 3) + 1); + reset(); +} + +bool BitMap::mark(size_t position) { + if (position >= size) + return false; + + bits[position >> 3] |= (char)1 << (position % 8); + return true; +} + +bool BitMap::unmark(size_t position) { + if (position >= size) + return false; + + bits[position >> 3] &= ~((char)1 << (position % 8)); + return true; +} + +void BitMap::markAll() { std::fill(bits.begin(), bits.end(), (char)0XFF); } + +void BitMap::reset() { std::fill(bits.begin(), bits.end(), (char)0); } + +bool BitMap::isMarked(size_t position) const { + if (position >= size) + return false; + + return (bits[position >> 3] & ((char)1 << (position % 8))) != 0; +} + +bool BitMap::isAllUnmarked() const { + size_t j; + for (j = 0; j < size >> 3; j++) { + if (bits[j] != (char)0) { + return false; + } + } + for (j = 0; j < size % 8; j++) { + if ((bits[size >> 3] & ((char)1 << j)) != 0) { + return false; + } + } + return true; +} + +bool BitMap::isAllMarked() const { + size_t j; + for (j = 0; j < size >> 3; j++) { + if (bits[j] != (char)0XFF) { + return false; + } + } + for (j = 0; j < size % 8; j++) { + if ((bits[size >> 3] & ((char)1 << j)) == 0) { + return false; + } + } + return true; +} + +const std::vector &BitMap::getByteArray() const { return this->bits; } + +size_t BitMap::getSize() const { return this->size; } diff --git a/iotdb-client/client-cpp/src/session/Date.cpp b/iotdb-client/client-cpp/src/session/Date.cpp new file mode 100644 index 0000000000000..094dad6b6aba6 --- /dev/null +++ b/iotdb-client/client-cpp/src/session/Date.cpp @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "Date.h" + +#include +#include +#include + +#include "Common.h" + +std::string IoTDBDate::toIsoExtendedString() const { + if (!valid_) { + return ""; + } + char buf[16]; + std::snprintf(buf, sizeof(buf), "%04d-%02d-%02d", year_, month_, day_); + return std::string(buf); +} + +int32_t parseDateExpressionToInt(const IoTDBDate &date) { + if (date.is_not_a_date()) { + throw IoTDBException("Date expression is null or empty."); + } + + const int year = date.year(); + if (year < 1000 || year > 9999) { + throw DateTimeParseException("Year must be between 1000 and 9999.", + date.toIsoExtendedString(), 0); + } + + const int64_t result = + static_cast(year) * 10000 + date.month() * 100 + date.day(); + if (result > INT32_MAX || result < INT32_MIN) { + throw DateTimeParseException("Date value overflow. ", + date.toIsoExtendedString(), 0); + } + return static_cast(result); +} + +IoTDBDate parseIntToDate(int32_t dateInt) { + if (dateInt == EMPTY_DATE_INT) { + return IoTDBDate::notADate(); + } + const int year = dateInt / 10000; + const int month = (dateInt % 10000) / 100; + const int day = dateInt % 100; + return IoTDBDate(year, month, day); +} diff --git a/iotdb-client/client-cpp/src/session/Session.cpp b/iotdb-client/client-cpp/src/session/Session.cpp new file mode 100644 index 0000000000000..d9b9edb1ebf18 --- /dev/null +++ b/iotdb-client/client-cpp/src/session/Session.cpp @@ -0,0 +1,2174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "Session.h" +#include "SessionDataSet.h" +#include "SessionImpl.h" +#include "ThriftConvert.h" +#include +#include +#include +#include +#include +#include +#include +#include + +using apache::thrift::transport::TTransportException; + +using namespace std; + +/** + * Timeout of query can be set by users. + * A negative number means using the default configuration of server. + * And value 0 will disable the function of query timeout. + */ +static const int64_t QUERY_TIMEOUT_MS = -1; + +TSDataType::TSDataType getTSDataTypeFromString(const string &str) { + // BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT, STRING, BLOB, TIMESTAMP, DATE, + // NULLTYPE + if (str == "BOOLEAN") { + return TSDataType::BOOLEAN; + } else if (str == "INT32") { + return TSDataType::INT32; + } else if (str == "INT64") { + return TSDataType::INT64; + } else if (str == "FLOAT") { + return TSDataType::FLOAT; + } else if (str == "DOUBLE") { + return TSDataType::DOUBLE; + } else if (str == "TEXT") { + return TSDataType::TEXT; + } else if (str == "TIMESTAMP") { + return TSDataType::TIMESTAMP; + } else if (str == "DATE") { + return TSDataType::DATE; + } else if (str == "BLOB") { + return TSDataType::BLOB; + } else if (str == "STRING") { + return TSDataType::STRING; + } else if (str == "OBJECT") { + return TSDataType::OBJECT; + } + return TSDataType::UNKNOWN; +} + +void Tablet::createColumns() { + for (size_t i = 0; i < schemas.size(); i++) { + TSDataType::TSDataType dataType = schemas[i].second; + switch (dataType) { + case TSDataType::BOOLEAN: + values[i] = new bool[maxRowNumber]; + break; + case TSDataType::DATE: + values[i] = new IoTDBDate[maxRowNumber]; + break; + case TSDataType::INT32: + values[i] = new int[maxRowNumber]; + break; + case TSDataType::TIMESTAMP: + case TSDataType::INT64: + values[i] = new int64_t[maxRowNumber]; + break; + case TSDataType::FLOAT: + values[i] = new float[maxRowNumber]; + break; + case TSDataType::DOUBLE: + values[i] = new double[maxRowNumber]; + break; + case TSDataType::STRING: + case TSDataType::BLOB: + case TSDataType::OBJECT: + case TSDataType::TEXT: + values[i] = new string[maxRowNumber]; + break; + default: + throw UnSupportedDataTypeException( + string("Data type ") + to_string(dataType) + " is not supported."); + } + } +} + +void Tablet::deleteColumns() { + for (size_t i = 0; i < schemas.size(); i++) { + if (!values[i]) + continue; + TSDataType::TSDataType dataType = schemas[i].second; + switch (dataType) { + case TSDataType::BOOLEAN: { + bool *valueBuf = (bool *)(values[i]); + delete[] valueBuf; + break; + } + case TSDataType::INT32: { + int *valueBuf = (int *)(values[i]); + delete[] valueBuf; + break; + } + case TSDataType::DATE: { + IoTDBDate *valueBuf = (IoTDBDate *)(values[i]); + delete[] valueBuf; + break; + } + case TSDataType::TIMESTAMP: + case TSDataType::INT64: { + int64_t *valueBuf = (int64_t *)(values[i]); + delete[] valueBuf; + break; + } + case TSDataType::FLOAT: { + float *valueBuf = (float *)(values[i]); + delete[] valueBuf; + break; + } + case TSDataType::DOUBLE: { + double *valueBuf = (double *)(values[i]); + delete[] valueBuf; + break; + } + case TSDataType::STRING: + case TSDataType::BLOB: + case TSDataType::OBJECT: + case TSDataType::TEXT: { + string *valueBuf = (string *)(values[i]); + delete[] valueBuf; + break; + } + default: + throw UnSupportedDataTypeException( + string("Data type ") + to_string(dataType) + " is not supported."); + } + values[i] = nullptr; + } +} + +void Tablet::deepCopyTabletColValue(void *const *srcPtr, void **destPtr, + TSDataType::TSDataType type, + int maxRowNumber) { + void *src = *srcPtr; + switch (type) { + case TSDataType::BOOLEAN: + *destPtr = new bool[maxRowNumber]; + memcpy(*destPtr, src, maxRowNumber * sizeof(bool)); + break; + case TSDataType::INT32: + *destPtr = new int32_t[maxRowNumber]; + memcpy(*destPtr, src, maxRowNumber * sizeof(int32_t)); + break; + case TSDataType::INT64: + case TSDataType::TIMESTAMP: + *destPtr = new int64_t[maxRowNumber]; + memcpy(*destPtr, src, maxRowNumber * sizeof(int64_t)); + break; + case TSDataType::FLOAT: + *destPtr = new float[maxRowNumber]; + memcpy(*destPtr, src, maxRowNumber * sizeof(float)); + break; + case TSDataType::DOUBLE: + *destPtr = new double[maxRowNumber]; + memcpy(*destPtr, src, maxRowNumber * sizeof(double)); + break; + case TSDataType::DATE: { + *destPtr = new IoTDBDate[maxRowNumber]; + IoTDBDate *srcDate = static_cast(src); + IoTDBDate *destDate = static_cast(*destPtr); + for (size_t j = 0; j < maxRowNumber; ++j) { + destDate[j] = srcDate[j]; + } + break; + } + case TSDataType::STRING: + case TSDataType::TEXT: + case TSDataType::OBJECT: + case TSDataType::BLOB: { + *destPtr = new std::string[maxRowNumber]; + std::string *srcStr = static_cast(src); + std::string *destStr = static_cast(*destPtr); + for (size_t j = 0; j < maxRowNumber; ++j) { + destStr[j] = srcStr[j]; + } + break; + } + default: + break; + } +} + +void Tablet::reset() { + rowSize = 0; + for (size_t i = 0; i < schemas.size(); i++) { + bitMaps[i].reset(); + } +} + +size_t Tablet::getTimeBytesSize() { return rowSize * 8; } + +size_t Tablet::getValueByteSize() { + size_t valueOccupation = 0; + for (size_t i = 0; i < schemas.size(); i++) { + switch (schemas[i].second) { + case TSDataType::BOOLEAN: + valueOccupation += rowSize; + break; + case TSDataType::INT32: + valueOccupation += rowSize * 4; + break; + case TSDataType::DATE: + valueOccupation += rowSize * 4; + break; + case TSDataType::TIMESTAMP: + case TSDataType::INT64: + valueOccupation += rowSize * 8; + break; + case TSDataType::FLOAT: + valueOccupation += rowSize * 4; + break; + case TSDataType::DOUBLE: + valueOccupation += rowSize * 8; + break; + case TSDataType::STRING: + case TSDataType::BLOB: + case TSDataType::OBJECT: + case TSDataType::TEXT: { + valueOccupation += rowSize * 4; + string *valueBuf = (string *)(values[i]); + for (size_t j = 0; j < rowSize; j++) { + valueOccupation += valueBuf[j].size(); + } + break; + } + default: + throw UnSupportedDataTypeException(string("Data type ") + + to_string(schemas[i].second) + + " is not supported."); + } + } + return valueOccupation; +} + +void Tablet::setAligned(bool isAligned) { this->isAligned = isAligned; } + +string SessionUtils::getTime(const Tablet &tablet) { + MyStringBuffer timeBuffer; + unsigned int n = 8u * tablet.rowSize; + if (n > timeBuffer.str.capacity()) { + timeBuffer.reserve(n); + } + + for (size_t i = 0; i < tablet.rowSize; i++) { + timeBuffer.putInt64(tablet.timestamps[i]); + } + return timeBuffer.str; +} + +string SessionUtils::getValue(const Tablet &tablet) { + MyStringBuffer valueBuffer; + unsigned int n = 8u * tablet.schemas.size() * tablet.rowSize; + if (n > valueBuffer.str.capacity()) { + valueBuffer.reserve(n); + } + for (size_t i = 0; i < tablet.schemas.size(); i++) { + TSDataType::TSDataType dataType = tablet.schemas[i].second; + const BitMap &bitMap = tablet.bitMaps[i]; + switch (dataType) { + case TSDataType::BOOLEAN: { + bool *valueBuf = (bool *)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { + valueBuffer.putBool(valueBuf[index]); + } else { + valueBuffer.putBool(false); + } + } + break; + } + case TSDataType::INT32: { + int *valueBuf = (int *)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { + valueBuffer.putInt(valueBuf[index]); + } else { + valueBuffer.putInt((numeric_limits::min)()); + } + } + break; + } + case TSDataType::DATE: { + IoTDBDate *valueBuf = (IoTDBDate *)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { + valueBuffer.putDate(valueBuf[index]); + } else { + valueBuffer.putInt(EMPTY_DATE_INT); + } + } + break; + } + case TSDataType::TIMESTAMP: + case TSDataType::INT64: { + int64_t *valueBuf = (int64_t *)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { + valueBuffer.putInt64(valueBuf[index]); + } else { + valueBuffer.putInt64((numeric_limits::min)()); + } + } + break; + } + case TSDataType::FLOAT: { + float *valueBuf = (float *)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { + valueBuffer.putFloat(valueBuf[index]); + } else { + valueBuffer.putFloat((numeric_limits::min)()); + } + } + break; + } + case TSDataType::DOUBLE: { + double *valueBuf = (double *)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { + valueBuffer.putDouble(valueBuf[index]); + } else { + valueBuffer.putDouble((numeric_limits::min)()); + } + } + break; + } + case TSDataType::STRING: + case TSDataType::BLOB: + case TSDataType::OBJECT: + case TSDataType::TEXT: { + string *valueBuf = (string *)(tablet.values[i]); + for (size_t index = 0; index < tablet.rowSize; index++) { + if (!bitMap.isMarked(index)) { + valueBuffer.putString(valueBuf[index]); + } else { + valueBuffer.putString(""); + } + } + break; + } + default: + throw UnSupportedDataTypeException( + string("Data type ") + to_string(dataType) + " is not supported."); + } + } + for (size_t i = 0; i < tablet.schemas.size(); i++) { + const BitMap &bitMap = tablet.bitMaps[i]; + bool columnHasNull = !bitMap.isAllUnmarked(); + valueBuffer.putChar(columnHasNull ? (char)1 : (char)0); + if (columnHasNull) { + const vector &bytes = bitMap.getByteArray(); + for (size_t index = 0; index < tablet.rowSize / 8 + 1; index++) { + valueBuffer.putChar(bytes[index]); + } + } + } + return valueBuffer.str; +} + +string MeasurementNode::serialize() const { + MyStringBuffer buffer; + buffer.putString(getName()); + buffer.putChar(getDataType()); + buffer.putChar(getEncoding()); + buffer.putChar(getCompressionType()); + return buffer.str; +} + +string Template::serialize() const { + MyStringBuffer buffer; + std::stack>> nodeStack; + unordered_set alignedPrefix; + buffer.putString(getName()); + buffer.putBool(isAligned()); + if (isAligned()) { + alignedPrefix.emplace(""); + } + + for (const auto &child : children_) { + nodeStack.push(make_pair("", child.second)); + } + + while (!nodeStack.empty()) { + auto cur = nodeStack.top(); + nodeStack.pop(); + + string prefix = cur.first; + shared_ptr cur_node_ptr = cur.second; + string fullPath(prefix); + + if (!cur_node_ptr->isMeasurement()) { + if (!prefix.empty()) { + fullPath.append("."); + } + fullPath.append(cur_node_ptr->getName()); + if (cur_node_ptr->isAligned()) { + alignedPrefix.emplace(fullPath); + } + for (const auto &child : cur_node_ptr->getChildren()) { + nodeStack.push(make_pair(fullPath, child.second)); + } + } else { + buffer.putString(prefix); + buffer.putBool(alignedPrefix.find(prefix) != alignedPrefix.end()); + buffer.concat(cur_node_ptr->serialize()); + } + } + + return buffer.str; +} + +Session::Session(const std::string &host, int rpcPort) : impl_(new Impl()) { + impl_->username_ = "root"; + impl_->password_ = "root"; + impl_->version = Version::V_1_0; + impl_->host_ = host; + impl_->rpcPort_ = rpcPort; + impl_->initZoneId(); + impl_->initNodesSupplier(); +} + +Session::Session(const std::vector &nodeUrls, + const std::string &username, const std::string &password) + : impl_(new Impl()) { + impl_->nodeUrls_ = nodeUrls; + impl_->username_ = username; + impl_->password_ = password; + impl_->version = Version::V_1_0; + impl_->initZoneId(); + impl_->initNodesSupplier(impl_->nodeUrls_); +} + +Session::Session(const std::string &host, int rpcPort, + const std::string &username, const std::string &password) + : impl_(new Impl()) { + impl_->host_ = host; + impl_->rpcPort_ = rpcPort; + impl_->username_ = username; + impl_->password_ = password; + impl_->fetchSize_ = iotdb::session::DEFAULT_FETCH_SIZE; + impl_->version = Version::V_1_0; + impl_->initZoneId(); + impl_->initNodesSupplier(); +} + +Session::Session(const std::string &host, int rpcPort, + const std::string &username, const std::string &password, + const std::string &zoneId, int fetchSize) + : impl_(new Impl()) { + impl_->host_ = host; + impl_->rpcPort_ = rpcPort; + impl_->username_ = username; + impl_->password_ = password; + impl_->zoneId_ = zoneId; + impl_->fetchSize_ = fetchSize; + impl_->version = Version::V_1_0; + impl_->initZoneId(); + impl_->initNodesSupplier(); +} + +Session::Session(const std::string &host, const std::string &rpcPort, + const std::string &username, const std::string &password, + const std::string &zoneId, int fetchSize) + : impl_(new Impl()) { + impl_->host_ = host; + impl_->rpcPort_ = stoi(rpcPort); + impl_->username_ = username; + impl_->password_ = password; + impl_->zoneId_ = zoneId; + impl_->fetchSize_ = fetchSize; + impl_->version = Version::V_1_0; + impl_->initZoneId(); + impl_->initNodesSupplier(); +} + +Session::Session(AbstractSessionBuilder *builder) : impl_(new Impl()) { + impl_->host_ = builder->host; + impl_->rpcPort_ = builder->rpcPort; + impl_->username_ = builder->username; + impl_->password_ = builder->password; + impl_->zoneId_ = builder->zoneId; + impl_->fetchSize_ = builder->fetchSize; + impl_->version = Version::V_1_0; + impl_->sqlDialect_ = builder->sqlDialect; + impl_->database_ = builder->database; + impl_->enableAutoFetch_ = builder->enableAutoFetch; + impl_->enableRedirection_ = builder->enableRedirections; + impl_->connectTimeoutMs_ = builder->connectTimeoutMs; + impl_->nodeUrls_ = builder->nodeUrls; + impl_->useSSL_ = builder->useSSL; + impl_->trustCertFilePath_ = builder->trustCertFilePath; + impl_->initZoneId(); + impl_->initNodesSupplier(impl_->nodeUrls_); +} + +/** + * When delete variable, make sure release all resource. + */ +Session::~Session() { + try { + close(); + } catch (const exception &e) { + log_debug(e.what()); + } +} + +void Session::Impl::removeBrokenSessionConnection( + shared_ptr sessionConnection) { + if (enableRedirection_) { + this->endPointToSessionConnection.erase(sessionConnection->getEndPoint()); + } + + auto it1 = deviceIdToEndpoint.begin(); + while (it1 != deviceIdToEndpoint.end()) { + if (it1->second == sessionConnection->getEndPoint()) { + it1 = deviceIdToEndpoint.erase(it1); + } else { + ++it1; + } + } +} + +/** + * check whether the batch has been sorted + * + * @return whether the batch has been sorted + */ +bool Session::Impl::checkSorted(const Tablet &tablet) { + for (size_t i = 1; i < tablet.rowSize; i++) { + if (tablet.timestamps[i] < tablet.timestamps[i - 1]) { + return false; + } + } + return true; +} + +bool Session::Impl::checkSorted(const vector ×) { + for (size_t i = 1; i < times.size(); i++) { + if (times[i] < times[i - 1]) { + return false; + } + } + return true; +} + +template +std::vector sortList(const std::vector &valueList, const int *index, + int indexLength) { + std::vector sortedValues(valueList.size()); + for (int i = 0; i < indexLength; i++) { + sortedValues[i] = valueList[index[i]]; + } + return sortedValues; +} + +template +void sortValuesList(T *valueList, const int *index, size_t indexLength) { + T *sortedValues = new T[indexLength]; + for (int i = 0; i < indexLength; i++) { + sortedValues[i] = valueList[index[i]]; + } + for (int i = 0; i < indexLength; i++) { + valueList[i] = sortedValues[i]; + } + delete[] sortedValues; +} + +void Session::Impl::sortTablet(Tablet &tablet) { + /* + * following part of code sort the batch data by time, + * so we can insert continuous data in value list to get a better performance + */ + // sort to get index, and use index to sort value list + int *index = new int[tablet.rowSize]; + for (size_t i = 0; i < tablet.rowSize; i++) { + index[i] = i; + } + + sortIndexByTimestamp(index, tablet.timestamps, tablet.rowSize); + tablet.timestamps = sortList(tablet.timestamps, index, tablet.rowSize); + for (size_t i = 0; i < tablet.schemas.size(); i++) { + TSDataType::TSDataType dataType = tablet.schemas[i].second; + switch (dataType) { + case TSDataType::BOOLEAN: { + sortValuesList((bool *)(tablet.values[i]), index, tablet.rowSize); + break; + } + case TSDataType::INT32: { + sortValuesList((int *)(tablet.values[i]), index, tablet.rowSize); + break; + } + case TSDataType::DATE: { + sortValuesList((IoTDBDate *)(tablet.values[i]), index, tablet.rowSize); + break; + } + case TSDataType::TIMESTAMP: + case TSDataType::INT64: { + sortValuesList((int64_t *)(tablet.values[i]), index, tablet.rowSize); + break; + } + case TSDataType::FLOAT: { + sortValuesList((float *)(tablet.values[i]), index, tablet.rowSize); + break; + } + case TSDataType::DOUBLE: { + sortValuesList((double *)(tablet.values[i]), index, tablet.rowSize); + break; + } + case TSDataType::STRING: + case TSDataType::BLOB: + case TSDataType::OBJECT: + case TSDataType::TEXT: { + sortValuesList((string *)(tablet.values[i]), index, tablet.rowSize); + break; + } + default: + throw UnSupportedDataTypeException( + string("Data type ") + to_string(dataType) + " is not supported."); + } + } + + delete[] index; +} + +void Session::Impl::sortIndexByTimestamp(int *index, + std::vector ×tamps, + int length) { + if (length <= 1) { + return; + } + + TsCompare tsCompareObj(timestamps); + std::sort(&index[0], &index[length], tsCompareObj); +} + +/** + * Append value into buffer in Big Endian order to comply with IoTDB server + */ +void Session::Impl::appendValues(string &buffer, const char *value, int size) { + static bool hasCheckedEndianFlag = false; + static bool localCpuIsBigEndian = false; + if (!hasCheckedEndianFlag) { + hasCheckedEndianFlag = true; + int chk = + 0x0201; // used to distinguish CPU's type (BigEndian or LittleEndian) + localCpuIsBigEndian = (0x01 != *(char *)(&chk)); + } + + if (localCpuIsBigEndian) { + buffer.append(value, size); + } else { + for (int i = size - 1; i >= 0; i--) { + buffer.append(value + i, 1); + } + } +} + +void Session::Impl::putValuesIntoBuffer( + const vector &types, const vector &values, + string &buf) { + int32_t date; + for (size_t i = 0; i < values.size(); i++) { + int8_t typeNum = getDataTypeNumber(types[i]); + buf.append((char *)(&typeNum), sizeof(int8_t)); + switch (types[i]) { + case TSDataType::BOOLEAN: + buf.append(values[i], 1); + break; + case TSDataType::INT32: + appendValues(buf, values[i], sizeof(int32_t)); + break; + case TSDataType::DATE: + date = parseDateExpressionToInt(*(IoTDBDate *)values[i]); + appendValues(buf, (char *)&date, sizeof(int32_t)); + break; + case TSDataType::TIMESTAMP: + case TSDataType::INT64: + appendValues(buf, values[i], sizeof(int64_t)); + break; + case TSDataType::FLOAT: + appendValues(buf, values[i], sizeof(float)); + break; + case TSDataType::DOUBLE: + appendValues(buf, values[i], sizeof(double)); + break; + case TSDataType::STRING: + case TSDataType::BLOB: + case TSDataType::OBJECT: + case TSDataType::TEXT: { + int32_t len = (uint32_t)strlen(values[i]); + appendValues(buf, (char *)(&len), sizeof(uint32_t)); + // no need to change the byte order of string value + buf.append(values[i], len); + break; + } + default: + break; + } + } +} + +int8_t Session::Impl::getDataTypeNumber(TSDataType::TSDataType type) { + switch (type) { + case TSDataType::BOOLEAN: + return 0; + case TSDataType::INT32: + return 1; + case TSDataType::INT64: + return 2; + case TSDataType::FLOAT: + return 3; + case TSDataType::DOUBLE: + return 4; + case TSDataType::TEXT: + return 5; + case TSDataType::TIMESTAMP: + return 8; + case TSDataType::DATE: + return 9; + case TSDataType::BLOB: + return 10; + case TSDataType::STRING: + return 11; + case TSDataType::OBJECT: + return 12; + default: + return -1; + } +} + +string Session::Impl::getVersionString(Version::Version version) { + switch (version) { + case Version::V_0_12: + return "V_0_12"; + case Version::V_0_13: + return "V_0_13"; + case Version::V_1_0: + return "V_1_0"; + default: + return "V_0_12"; + } +} + +void Session::Impl::initZoneId() { + if (!zoneId_.empty()) { + return; + } + + time_t ts = 0; + struct tm tmv; +#if defined(_WIN64) || defined(WIN32) || defined(_WIN32) + localtime_s(&tmv, &ts); +#else + localtime_r(&ts, &tmv); +#endif + + char zoneStr[32]; + strftime(zoneStr, sizeof(zoneStr), "%z", &tmv); + zoneId_ = zoneStr; +} + +void Session::Impl::initNodesSupplier( + const std::vector &nodeUrls) { + std::vector endPoints; + std::unordered_set uniqueEndpoints; + + if (nodeUrls.empty() && host_.empty()) { + throw IoTDBException("No available nodes"); + } + + // Process provided node URLs + if (!nodeUrls.empty()) { + for (auto &url : nodeUrls) { + try { + TEndPoint endPoint = UrlUtils::parseTEndPointIpv4AndIpv6Url(url); + if (endPoint.port == 0) + continue; // Skip invalid endpoints + + std::string endpointKey = + endPoint.ip + ":" + std::to_string(endPoint.port); + if (uniqueEndpoints.find(endpointKey) == uniqueEndpoints.end()) { + endPoints.emplace_back(std::move(endPoint)); + uniqueEndpoints.insert(std::move(endpointKey)); + } + } catch (...) { + continue; // Skip malformed URLs + } + } + } + + // Fallback to local endpoint if no valid endpoints found + if (endPoints.empty()) { + if (host_.empty() || rpcPort_ == 0) { + throw IoTDBException("No valid endpoints available"); + } + TEndPoint endPoint; + endPoint.__set_ip(host_); + endPoint.__set_port(rpcPort_); + endPoints.emplace_back(std::move(endPoint)); + } + + if (enableAutoFetch_) { + nodesSupplier_ = NodesSupplier::create(endPoints, username_, password_, + useSSL_, trustCertFilePath_); + } else { + nodesSupplier_ = make_shared(endPoints); + } +} + +void Session::Impl::initDefaultSessionConnection() { + // Try all endpoints from supplier until a connection is established. + auto endpoints = nodesSupplier_->getEndPointList(); + bool connected = false; + + for (const auto &endpoint : endpoints) { + try { + host_ = endpoint.ip; + rpcPort_ = endpoint.port; + + defaultEndPoint_.__set_ip(host_); + defaultEndPoint_.__set_port(rpcPort_); + + defaultSessionConnection_ = std::make_shared( + this, defaultEndPoint_, zoneId_, nodesSupplier_, fetchSize_, 3, 500, + connectTimeoutMs_, sqlDialect_, database_); + + connected = true; + break; + } catch (const IoTDBException &e) { + log_debug(e.what()); + throw; + } catch (const std::exception &e) { + log_warn(e.what()); + } + } + + if (!connected) { + throw std::runtime_error( + "No available node to establish SessionConnection."); + } +} + +void Session::Impl::insertStringRecordsWithLeaderCache( + vector deviceIds, vector times, + vector> measurementsList, vector> valuesList, + bool isAligned) { + std::unordered_map, + TSInsertStringRecordsReq> + recordsGroup; + for (int i = 0; i < deviceIds.size(); i++) { + auto connection = getSessionConnection(deviceIds[i]); + if (recordsGroup.find(connection) == recordsGroup.end()) { + TSInsertStringRecordsReq request; + std::vector emptyPrefixPaths; + std::vector> emptyMeasurementsList; + vector> emptyValuesList; + std::vector emptyTimestamps; + request.__set_isAligned(isAligned); + request.__set_prefixPaths(emptyPrefixPaths); + request.__set_timestamps(emptyTimestamps); + request.__set_measurementsList(emptyMeasurementsList); + request.__set_valuesList(emptyValuesList); + recordsGroup.insert(make_pair(connection, request)); + } + TSInsertStringRecordsReq &existingReq = recordsGroup[connection]; + existingReq.prefixPaths.emplace_back(deviceIds[i]); + existingReq.timestamps.emplace_back(times[i]); + existingReq.measurementsList.emplace_back(measurementsList[i]); + existingReq.valuesList.emplace_back(valuesList[i]); + } + std::function, + const TSInsertStringRecordsReq &)> + consumer = + [](const std::shared_ptr &c, + const TSInsertStringRecordsReq &r) { c->insertStringRecords(r); }; + if (recordsGroup.size() == 1) { + insertOnce(recordsGroup, consumer); + } else { + insertByGroup(recordsGroup, consumer); + } +} + +void Session::Impl::insertRecordsWithLeaderCache( + vector deviceIds, vector times, + vector> measurementsList, + const vector> &typesList, + vector> valuesList, bool isAligned) { + std::unordered_map, TSInsertRecordsReq> + recordsGroup; + for (int i = 0; i < deviceIds.size(); i++) { + auto connection = getSessionConnection(deviceIds[i]); + if (recordsGroup.find(connection) == recordsGroup.end()) { + TSInsertRecordsReq request; + std::vector emptyPrefixPaths; + std::vector> emptyMeasurementsList; + std::vector emptyValuesList; + std::vector emptyTimestamps; + request.__set_isAligned(isAligned); + request.__set_prefixPaths(emptyPrefixPaths); + request.__set_timestamps(emptyTimestamps); + request.__set_measurementsList(emptyMeasurementsList); + request.__set_valuesList(emptyValuesList); + recordsGroup.insert(make_pair(connection, request)); + } + TSInsertRecordsReq &existingReq = recordsGroup[connection]; + existingReq.prefixPaths.emplace_back(deviceIds[i]); + existingReq.timestamps.emplace_back(times[i]); + existingReq.measurementsList.emplace_back(measurementsList[i]); + vector bufferList; + string buffer; + putValuesIntoBuffer(typesList[i], valuesList[i], buffer); + existingReq.valuesList.emplace_back(buffer); + recordsGroup[connection] = existingReq; + } + std::function, + const TSInsertRecordsReq &)> + consumer = [](const std::shared_ptr &c, + const TSInsertRecordsReq &r) { c->insertRecords(r); }; + if (recordsGroup.size() == 1) { + insertOnce(recordsGroup, consumer); + } else { + insertByGroup(recordsGroup, consumer); + } +} + +void Session::Impl::insertTabletsWithLeaderCache( + unordered_map &tablets, bool sorted, bool isAligned) { + std::unordered_map, TSInsertTabletsReq> + tabletsGroup; + if (tablets.empty()) { + throw BatchExecutionException("No tablet is inserting!"); + } + for (const auto &item : tablets) { + if (isAligned != item.second->isAligned) { + throw BatchExecutionException( + "The tablets should be all aligned or non-aligned!"); + } + if (!checkSorted(*(item.second))) { + sortTablet(*(item.second)); + } + auto deviceId = item.first; + auto tablet = item.second; + auto connection = getSessionConnection(deviceId); + auto it = tabletsGroup.find(connection); + if (it == tabletsGroup.end()) { + TSInsertTabletsReq request; + tabletsGroup[connection] = request; + } + TSInsertTabletsReq &existingReq = tabletsGroup[connection]; + existingReq.prefixPaths.emplace_back(tablet->deviceId); + existingReq.timestampsList.emplace_back( + move(SessionUtils::getTime(*tablet))); + existingReq.valuesList.emplace_back(move(SessionUtils::getValue(*tablet))); + existingReq.sizeList.emplace_back(tablet->rowSize); + vector dataTypes; + vector measurements; + for (pair schema : tablet->schemas) { + measurements.push_back(schema.first); + dataTypes.push_back(schema.second); + } + existingReq.measurementsList.emplace_back(measurements); + existingReq.typesList.emplace_back(dataTypes); + } + + std::function, + const TSInsertTabletsReq &)> + consumer = [](const std::shared_ptr &c, + const TSInsertTabletsReq &r) { c->insertTablets(r); }; + if (tabletsGroup.size() == 1) { + insertOnce(tabletsGroup, consumer); + } else { + insertByGroup(tabletsGroup, consumer); + } +} + +void Session::open() { open(false, Impl::DEFAULT_TIMEOUT_MS); } + +void Session::open(bool enableRPCCompression) { + open(enableRPCCompression, Impl::DEFAULT_TIMEOUT_MS); +} + +void Session::open(bool enableRPCCompression, int connectionTimeoutInMs) { + if (!impl_->isClosed_) { + return; + } + + try { + impl_->initDefaultSessionConnection(); + } catch (const exception &e) { + log_debug(e.what()); + throw IoTDBException(e.what()); + } + impl_->zoneId_ = impl_->defaultSessionConnection_->zoneId; + + if (impl_->enableRedirection_) { + impl_->endPointToSessionConnection.insert( + make_pair(impl_->defaultEndPoint_, impl_->defaultSessionConnection_)); + } + + impl_->isClosed_ = false; +} + +void Session::close() { + if (impl_->isClosed_) { + return; + } + impl_->isClosed_ = true; +} + +void Session::insertRecord(const string &deviceId, int64_t time, + const vector &measurements, + const vector &values) { + TSInsertStringRecordReq req; + req.__set_prefixPath(deviceId); + req.__set_timestamp(time); + req.__set_measurements(measurements); + req.__set_values(values); + req.__set_isAligned(false); + try { + impl_->getSessionConnection(deviceId)->insertStringRecord(req); + } catch (RedirectException &e) { + impl_->handleRedirection(deviceId, endpointToThrift(e.endPoint)); + } catch (const IoTDBConnectionException &e) { + if (impl_->enableRedirection_ && impl_->deviceIdToEndpoint.find(deviceId) != + impl_->deviceIdToEndpoint.end()) { + impl_->deviceIdToEndpoint.erase(deviceId); + try { + impl_->defaultSessionConnection_->insertStringRecord(req); + } catch (RedirectException &e) { + } + } else { + throw; + } + } +} + +void Session::insertRecord(const string &deviceId, int64_t time, + const vector &measurements, + const vector &types, + const vector &values) { + TSInsertRecordReq req; + req.__set_prefixPath(deviceId); + req.__set_timestamp(time); + req.__set_measurements(measurements); + string buffer; + impl_->putValuesIntoBuffer(types, values, buffer); + req.__set_values(buffer); + req.__set_isAligned(false); + try { + impl_->getSessionConnection(deviceId)->insertRecord(req); + } catch (RedirectException &e) { + impl_->handleRedirection(deviceId, endpointToThrift(e.endPoint)); + } catch (const IoTDBConnectionException &e) { + if (impl_->enableRedirection_ && impl_->deviceIdToEndpoint.find(deviceId) != + impl_->deviceIdToEndpoint.end()) { + impl_->deviceIdToEndpoint.erase(deviceId); + try { + impl_->defaultSessionConnection_->insertRecord(req); + } catch (RedirectException &e) { + } + } else { + throw; + } + } +} + +void Session::insertAlignedRecord(const string &deviceId, int64_t time, + const vector &measurements, + const vector &values) { + TSInsertStringRecordReq req; + req.__set_prefixPath(deviceId); + req.__set_timestamp(time); + req.__set_measurements(measurements); + req.__set_values(values); + req.__set_isAligned(true); + try { + impl_->getSessionConnection(deviceId)->insertStringRecord(req); + } catch (RedirectException &e) { + impl_->handleRedirection(deviceId, endpointToThrift(e.endPoint)); + } catch (const IoTDBConnectionException &e) { + if (impl_->enableRedirection_ && impl_->deviceIdToEndpoint.find(deviceId) != + impl_->deviceIdToEndpoint.end()) { + impl_->deviceIdToEndpoint.erase(deviceId); + try { + impl_->defaultSessionConnection_->insertStringRecord(req); + } catch (RedirectException &e) { + } + } else { + throw; + } + } +} + +void Session::insertAlignedRecord(const string &deviceId, int64_t time, + const vector &measurements, + const vector &types, + const vector &values) { + TSInsertRecordReq req; + req.__set_prefixPath(deviceId); + req.__set_timestamp(time); + req.__set_measurements(measurements); + string buffer; + impl_->putValuesIntoBuffer(types, values, buffer); + req.__set_values(buffer); + req.__set_isAligned(false); + try { + impl_->getSessionConnection(deviceId)->insertRecord(req); + } catch (RedirectException &e) { + impl_->handleRedirection(deviceId, endpointToThrift(e.endPoint)); + } catch (const IoTDBConnectionException &e) { + if (impl_->enableRedirection_ && impl_->deviceIdToEndpoint.find(deviceId) != + impl_->deviceIdToEndpoint.end()) { + impl_->deviceIdToEndpoint.erase(deviceId); + try { + impl_->defaultSessionConnection_->insertRecord(req); + } catch (RedirectException &e) { + } + } else { + throw; + } + } +} + +void Session::insertRecords(const vector &deviceIds, + const vector ×, + const vector> &measurementsList, + const vector> &valuesList) { + size_t len = deviceIds.size(); + if (len != times.size() || len != measurementsList.size() || + len != valuesList.size()) { + logic_error e("deviceIds, times, measurementsList and valuesList's size " + "should be equal"); + throw exception(e); + } + + if (impl_->enableRedirection_) { + impl_->insertStringRecordsWithLeaderCache( + deviceIds, times, measurementsList, valuesList, false); + } else { + TSInsertStringRecordsReq request; + request.__set_prefixPaths(deviceIds); + request.__set_timestamps(times); + request.__set_measurementsList(measurementsList); + request.__set_valuesList(valuesList); + request.__set_isAligned(false); + try { + impl_->defaultSessionConnection_->insertStringRecords(request); + } catch (RedirectException &e) { + } + } +} + +void Session::insertRecords( + const vector &deviceIds, const vector ×, + const vector> &measurementsList, + const vector> &typesList, + const vector> &valuesList) { + size_t len = deviceIds.size(); + if (len != times.size() || len != measurementsList.size() || + len != valuesList.size()) { + logic_error e("deviceIds, times, measurementsList and valuesList's size " + "should be equal"); + throw exception(e); + } + + if (impl_->enableRedirection_) { + impl_->insertRecordsWithLeaderCache(deviceIds, times, measurementsList, + typesList, valuesList, false); + } else { + TSInsertRecordsReq request; + request.__set_prefixPaths(deviceIds); + request.__set_timestamps(times); + request.__set_measurementsList(measurementsList); + vector bufferList; + for (size_t i = 0; i < valuesList.size(); i++) { + string buffer; + impl_->putValuesIntoBuffer(typesList[i], valuesList[i], buffer); + bufferList.push_back(buffer); + } + request.__set_valuesList(bufferList); + request.__set_isAligned(false); + try { + impl_->defaultSessionConnection_->insertRecords(request); + } catch (RedirectException &e) { + } + } +} + +void Session::insertAlignedRecords( + const vector &deviceIds, const vector ×, + const vector> &measurementsList, + const vector> &valuesList) { + size_t len = deviceIds.size(); + if (len != times.size() || len != measurementsList.size() || + len != valuesList.size()) { + logic_error e("deviceIds, times, measurementsList and valuesList's size " + "should be equal"); + throw exception(e); + } + + if (impl_->enableRedirection_) { + impl_->insertStringRecordsWithLeaderCache( + deviceIds, times, measurementsList, valuesList, true); + } else { + TSInsertStringRecordsReq request; + request.__set_prefixPaths(deviceIds); + request.__set_timestamps(times); + request.__set_measurementsList(measurementsList); + request.__set_valuesList(valuesList); + request.__set_isAligned(true); + try { + impl_->defaultSessionConnection_->insertStringRecords(request); + } catch (RedirectException &e) { + } + } +} + +void Session::insertAlignedRecords( + const vector &deviceIds, const vector ×, + const vector> &measurementsList, + const vector> &typesList, + const vector> &valuesList) { + size_t len = deviceIds.size(); + if (len != times.size() || len != measurementsList.size() || + len != valuesList.size()) { + logic_error e("deviceIds, times, measurementsList and valuesList's size " + "should be equal"); + throw exception(e); + } + + if (impl_->enableRedirection_) { + impl_->insertRecordsWithLeaderCache(deviceIds, times, measurementsList, + typesList, valuesList, true); + } else { + TSInsertRecordsReq request; + request.__set_prefixPaths(deviceIds); + request.__set_timestamps(times); + request.__set_measurementsList(measurementsList); + vector bufferList; + for (size_t i = 0; i < valuesList.size(); i++) { + string buffer; + impl_->putValuesIntoBuffer(typesList[i], valuesList[i], buffer); + bufferList.push_back(buffer); + } + request.__set_valuesList(bufferList); + request.__set_isAligned(false); + try { + impl_->defaultSessionConnection_->insertRecords(request); + } catch (RedirectException &e) { + } + } +} + +void Session::insertRecordsOfOneDevice( + const string &deviceId, vector ×, + vector> &measurementsList, + vector> &typesList, + vector> &valuesList) { + insertRecordsOfOneDevice(deviceId, times, measurementsList, typesList, + valuesList, false); +} + +void Session::insertRecordsOfOneDevice( + const string &deviceId, vector ×, + vector> &measurementsList, + vector> &typesList, + vector> &valuesList, bool sorted) { + if (!impl_->checkSorted(times)) { + int *index = new int[times.size()]; + for (size_t i = 0; i < times.size(); i++) { + index[i] = (int)i; + } + + impl_->sortIndexByTimestamp(index, times, (int)(times.size())); + times = sortList(times, index, (int)(times.size())); + measurementsList = sortList(measurementsList, index, (int)(times.size())); + typesList = sortList(typesList, index, (int)(times.size())); + valuesList = sortList(valuesList, index, (int)(times.size())); + delete[] index; + } + TSInsertRecordsOfOneDeviceReq request; + request.__set_prefixPath(deviceId); + request.__set_timestamps(times); + request.__set_measurementsList(measurementsList); + vector bufferList; + for (size_t i = 0; i < valuesList.size(); i++) { + string buffer; + impl_->putValuesIntoBuffer(typesList[i], valuesList[i], buffer); + bufferList.push_back(buffer); + } + request.__set_valuesList(bufferList); + request.__set_isAligned(false); + TSStatus respStatus; + try { + impl_->getSessionConnection(deviceId)->insertRecordsOfOneDevice(request); + } catch (RedirectException &e) { + impl_->handleRedirection(deviceId, endpointToThrift(e.endPoint)); + } catch (const IoTDBConnectionException &e) { + if (impl_->enableRedirection_ && impl_->deviceIdToEndpoint.find(deviceId) != + impl_->deviceIdToEndpoint.end()) { + impl_->deviceIdToEndpoint.erase(deviceId); + try { + impl_->defaultSessionConnection_->insertRecordsOfOneDevice(request); + } catch (RedirectException &e) { + } + } else { + throw; + } + } +} + +void Session::insertAlignedRecordsOfOneDevice( + const string &deviceId, vector ×, + vector> &measurementsList, + vector> &typesList, + vector> &valuesList) { + insertAlignedRecordsOfOneDevice(deviceId, times, measurementsList, typesList, + valuesList, false); +} + +void Session::insertAlignedRecordsOfOneDevice( + const string &deviceId, vector ×, + vector> &measurementsList, + vector> &typesList, + vector> &valuesList, bool sorted) { + if (!impl_->checkSorted(times)) { + int *index = new int[times.size()]; + for (size_t i = 0; i < times.size(); i++) { + index[i] = (int)i; + } + + impl_->sortIndexByTimestamp(index, times, (int)(times.size())); + times = sortList(times, index, (int)(times.size())); + measurementsList = sortList(measurementsList, index, (int)(times.size())); + typesList = sortList(typesList, index, (int)(times.size())); + valuesList = sortList(valuesList, index, (int)(times.size())); + delete[] index; + } + TSInsertRecordsOfOneDeviceReq request; + request.__set_prefixPath(deviceId); + request.__set_timestamps(times); + request.__set_measurementsList(measurementsList); + vector bufferList; + for (size_t i = 0; i < valuesList.size(); i++) { + string buffer; + impl_->putValuesIntoBuffer(typesList[i], valuesList[i], buffer); + bufferList.push_back(buffer); + } + request.__set_valuesList(bufferList); + request.__set_isAligned(true); + TSStatus respStatus; + try { + impl_->getSessionConnection(deviceId)->insertRecordsOfOneDevice(request); + } catch (RedirectException &e) { + impl_->handleRedirection(deviceId, endpointToThrift(e.endPoint)); + } catch (const IoTDBConnectionException &e) { + if (impl_->enableRedirection_ && impl_->deviceIdToEndpoint.find(deviceId) != + impl_->deviceIdToEndpoint.end()) { + impl_->deviceIdToEndpoint.erase(deviceId); + try { + impl_->defaultSessionConnection_->insertRecordsOfOneDevice(request); + } catch (RedirectException &e) { + } + } else { + throw; + } + } +} + +void Session::insertTablet(Tablet &tablet) { + try { + insertTablet(tablet, false); + } catch (const exception &e) { + log_debug(e.what()); + logic_error error(e.what()); + throw exception(error); + } +} + +void Session::Impl::buildInsertTabletReq(TSInsertTabletReq &request, + Tablet &tablet, bool sorted) { + if ((!sorted) && !checkSorted(tablet)) { + sortTablet(tablet); + } + + request.__set_prefixPath(tablet.deviceId); + + std::vector reqMeasurements; + reqMeasurements.reserve(tablet.schemas.size()); + std::vector types; + types.reserve(tablet.schemas.size()); + for (pair schema : tablet.schemas) { + reqMeasurements.push_back(schema.first); + types.push_back(schema.second); + } + request.__set_measurements(reqMeasurements); + request.__set_types(types); + request.__set_values(SessionUtils::getValue(tablet)); + request.__set_timestamps(SessionUtils::getTime(tablet)); + request.__set_size(tablet.rowSize); + request.__set_isAligned(tablet.isAligned); +} + +void Session::Impl::insertTablet(TSInsertTabletReq request) { + auto deviceId = request.prefixPath; + try { + getSessionConnection(deviceId)->insertTablet(request); + } catch (RedirectException &e) { + handleRedirection(deviceId, endpointToThrift(e.endPoint)); + } catch (const IoTDBConnectionException &e) { + if (enableRedirection_ && + deviceIdToEndpoint.find(deviceId) != deviceIdToEndpoint.end()) { + deviceIdToEndpoint.erase(deviceId); + try { + defaultSessionConnection_->insertTablet(request); + } catch (RedirectException &e) { + } + } else { + throw; + } + } +} + +void Session::insertTablet(Tablet &tablet, bool sorted) { + TSInsertTabletReq request; + impl_->buildInsertTabletReq(request, tablet, sorted); + impl_->insertTablet(request); +} + +void Session::insertAlignedTablet(Tablet &tablet) { + insertAlignedTablet(tablet, false); +} + +void Session::insertAlignedTablet(Tablet &tablet, bool sorted) { + tablet.setAligned(true); + try { + insertTablet(tablet, sorted); + } catch (const exception &e) { + log_debug(e.what()); + logic_error error(e.what()); + throw exception(error); + } +} + +void Session::insertTablets(unordered_map &tablets) { + try { + insertTablets(tablets, false); + } catch (const exception &e) { + log_debug(e.what()); + logic_error error(e.what()); + throw exception(error); + } +} + +void Session::insertTablets(unordered_map &tablets, + bool sorted) { + if (tablets.empty()) { + throw BatchExecutionException("No tablet is inserting!"); + } + auto beginIter = tablets.begin(); + bool isAligned = ((*beginIter).second)->isAligned; + if (impl_->enableRedirection_) { + impl_->insertTabletsWithLeaderCache(tablets, sorted, isAligned); + } else { + TSInsertTabletsReq request; + for (const auto &item : tablets) { + if (isAligned != item.second->isAligned) { + throw BatchExecutionException( + "The tablets should be all aligned or non-aligned!"); + } + if (!impl_->checkSorted(*(item.second))) { + impl_->sortTablet(*(item.second)); + } + request.prefixPaths.push_back(item.second->deviceId); + vector measurements; + vector dataTypes; + for (pair schema : item.second->schemas) { + measurements.push_back(schema.first); + dataTypes.push_back(schema.second); + } + request.measurementsList.push_back(measurements); + request.typesList.push_back(dataTypes); + request.timestampsList.push_back( + move(SessionUtils::getTime(*(item.second)))); + request.valuesList.push_back( + move(SessionUtils::getValue(*(item.second)))); + request.sizeList.push_back(item.second->rowSize); + } + request.__set_isAligned(isAligned); + try { + TSStatus respStatus; + impl_->defaultSessionConnection_->insertTablets(request); + RpcUtils::verifySuccess(respStatus); + } catch (RedirectException &e) { + } + } +} + +void Session::insertAlignedTablets(unordered_map &tablets, + bool sorted) { + for (auto iter = tablets.begin(); iter != tablets.end(); iter++) { + iter->second->setAligned(true); + } + try { + insertTablets(tablets, sorted); + } catch (const exception &e) { + log_debug(e.what()); + logic_error error(e.what()); + throw exception(error); + } +} + +void Session::testInsertRecord(const string &deviceId, int64_t time, + const vector &measurements, + const vector &values) { + TSInsertStringRecordReq req; + req.__set_prefixPath(deviceId); + req.__set_timestamp(time); + req.__set_measurements(measurements); + req.__set_values(values); + TSStatus tsStatus; + try { + impl_->defaultSessionConnection_->testInsertStringRecord(req); + RpcUtils::verifySuccess(tsStatus); + } catch (const TTransportException &e) { + log_debug(e.what()); + throw IoTDBConnectionException(e.what()); + } catch (const IoTDBException &e) { + log_debug(e.what()); + throw; + } catch (const exception &e) { + log_debug(e.what()); + throw IoTDBException(e.what()); + } +} + +void Session::testInsertTablet(const Tablet &tablet) { + TSInsertTabletReq request; + request.prefixPath = tablet.deviceId; + for (pair schema : tablet.schemas) { + request.measurements.push_back(schema.first); + request.types.push_back(schema.second); + } + request.__set_timestamps(move(SessionUtils::getTime(tablet))); + request.__set_values(move(SessionUtils::getValue(tablet))); + request.__set_size(tablet.rowSize); + try { + TSStatus tsStatus; + impl_->defaultSessionConnection_->testInsertTablet(request); + RpcUtils::verifySuccess(tsStatus); + } catch (const TTransportException &e) { + log_debug(e.what()); + throw IoTDBConnectionException(e.what()); + } catch (const IoTDBException &e) { + log_debug(e.what()); + throw; + } catch (const exception &e) { + log_debug(e.what()); + throw IoTDBException(e.what()); + } +} + +void Session::testInsertRecords(const vector &deviceIds, + const vector ×, + const vector> &measurementsList, + const vector> &valuesList) { + size_t len = deviceIds.size(); + if (len != times.size() || len != measurementsList.size() || + len != valuesList.size()) { + logic_error error("deviceIds, times, measurementsList and valuesList's " + "size should be equal"); + throw exception(error); + } + TSInsertStringRecordsReq request; + request.__set_prefixPaths(deviceIds); + request.__set_timestamps(times); + request.__set_measurementsList(measurementsList); + request.__set_valuesList(valuesList); + + try { + TSStatus tsStatus; + impl_->defaultSessionConnection_->getSessionClient()->insertStringRecords( + tsStatus, request); + RpcUtils::verifySuccess(tsStatus); + } catch (const TTransportException &e) { + log_debug(e.what()); + throw IoTDBConnectionException(e.what()); + } catch (const IoTDBException &e) { + log_debug(e.what()); + throw; + } catch (const exception &e) { + log_debug(e.what()); + throw IoTDBException(e.what()); + } +} + +void Session::deleteTimeseries(const string &path) { + vector paths; + paths.push_back(path); + deleteTimeseries(paths); +} + +void Session::deleteTimeseries(const vector &paths) { + impl_->defaultSessionConnection_->deleteTimeseries(paths); +} + +void Session::deleteData(const string &path, int64_t endTime) { + vector paths; + paths.push_back(path); + deleteData(paths, LONG_LONG_MIN, endTime); +} + +void Session::deleteData(const vector &paths, int64_t endTime) { + deleteData(paths, LONG_LONG_MIN, endTime); +} + +void Session::deleteData(const vector &paths, int64_t startTime, + int64_t endTime) { + TSDeleteDataReq req; + req.__set_paths(paths); + req.__set_startTime(startTime); + req.__set_endTime(endTime); + impl_->defaultSessionConnection_->deleteData(req); +} + +void Session::setStorageGroup(const string &storageGroupId) { + impl_->defaultSessionConnection_->setStorageGroup(storageGroupId); +} + +void Session::deleteStorageGroup(const string &storageGroup) { + vector storageGroups; + storageGroups.push_back(storageGroup); + deleteStorageGroups(storageGroups); +} + +void Session::deleteStorageGroups(const vector &storageGroups) { + impl_->defaultSessionConnection_->deleteStorageGroups(storageGroups); +} + +void Session::createDatabase(const string &database) { + this->setStorageGroup(database); +} + +void Session::deleteDatabase(const string &database) { + this->deleteStorageGroups(vector{database}); +} + +void Session::deleteDatabases(const vector &databases) { + this->deleteStorageGroups(databases); +} + +void Session::createTimeseries(const string &path, + TSDataType::TSDataType dataType, + TSEncoding::TSEncoding encoding, + CompressionType::CompressionType compressor) { + try { + createTimeseries(path, dataType, encoding, compressor, nullptr, nullptr, + nullptr, ""); + } catch (const exception &e) { + log_debug(e.what()); + throw IoTDBException(e.what()); + } +} + +void Session::createTimeseries(const string &path, + TSDataType::TSDataType dataType, + TSEncoding::TSEncoding encoding, + CompressionType::CompressionType compressor, + map *props, + map *tags, + map *attributes, + const string &measurementAlias) { + TSCreateTimeseriesReq req; + req.__set_path(path); + req.__set_dataType(dataType); + req.__set_encoding(encoding); + req.__set_compressor(compressor); + if (props != nullptr) { + req.__set_props(*props); + } + + if (tags != nullptr) { + req.__set_tags(*tags); + } + if (attributes != nullptr) { + req.__set_attributes(*attributes); + } + if (!measurementAlias.empty()) { + req.__set_measurementAlias(measurementAlias); + } + impl_->defaultSessionConnection_->createTimeseries(req); +} + +void Session::createMultiTimeseries( + const vector &paths, + const vector &dataTypes, + const vector &encodings, + const vector &compressors, + vector> *propsList, + vector> *tagsList, + vector> *attributesList, + vector *measurementAliasList) { + TSCreateMultiTimeseriesReq request; + request.__set_paths(paths); + + vector dataTypesOrdinal; + dataTypesOrdinal.reserve(dataTypes.size()); + for (TSDataType::TSDataType dataType : dataTypes) { + dataTypesOrdinal.push_back(dataType); + } + request.__set_dataTypes(dataTypesOrdinal); + + vector encodingsOrdinal; + encodingsOrdinal.reserve(encodings.size()); + for (TSEncoding::TSEncoding encoding : encodings) { + encodingsOrdinal.push_back(encoding); + } + request.__set_encodings(encodingsOrdinal); + + vector compressorsOrdinal; + compressorsOrdinal.reserve(compressors.size()); + for (CompressionType::CompressionType compressor : compressors) { + compressorsOrdinal.push_back(compressor); + } + request.__set_compressors(compressorsOrdinal); + + if (propsList != nullptr) { + request.__set_propsList(*propsList); + } + + if (tagsList != nullptr) { + request.__set_tagsList(*tagsList); + } + if (attributesList != nullptr) { + request.__set_attributesList(*attributesList); + } + if (measurementAliasList != nullptr) { + request.__set_measurementAliasList(*measurementAliasList); + } + + impl_->defaultSessionConnection_->createMultiTimeseries(request); +} + +void Session::createAlignedTimeseries( + const std::string &deviceId, const std::vector &measurements, + const std::vector &dataTypes, + const std::vector &encodings, + const std::vector &compressors) { + TSCreateAlignedTimeseriesReq request; + request.__set_prefixPath(deviceId); + request.__set_measurements(measurements); + + vector dataTypesOrdinal; + dataTypesOrdinal.reserve(dataTypes.size()); + for (TSDataType::TSDataType dataType : dataTypes) { + dataTypesOrdinal.push_back(dataType); + } + request.__set_dataTypes(dataTypesOrdinal); + + vector encodingsOrdinal; + encodingsOrdinal.reserve(encodings.size()); + for (TSEncoding::TSEncoding encoding : encodings) { + encodingsOrdinal.push_back(encoding); + } + request.__set_encodings(encodingsOrdinal); + + vector compressorsOrdinal; + compressorsOrdinal.reserve(compressors.size()); + for (CompressionType::CompressionType compressor : compressors) { + compressorsOrdinal.push_back(compressor); + } + request.__set_compressors(compressorsOrdinal); + + impl_->defaultSessionConnection_->createAlignedTimeseries(request); +} + +bool Session::checkTimeseriesExists(const string &path) { + try { + std::unique_ptr dataset = + executeQueryStatement("SHOW TIMESERIES " + path); + if (dataset == nullptr) { + throw IoTDBException("executeQueryStatement failed"); + } + bool isExisted = dataset->hasNext(); + dataset->closeOperationHandle(); + return isExisted; + } catch (const exception &e) { + log_debug(e.what()); + throw IoTDBException(e.what()); + } +} + +shared_ptr Session::Impl::getQuerySessionConnection() { + auto endPoint = nodesSupplier_->getQueryEndPoint(); + if (!endPoint.is_initialized() || endPointToSessionConnection.empty()) { + return defaultSessionConnection_; + } + + auto it = endPointToSessionConnection.find(endPoint.value()); + if (it != endPointToSessionConnection.end()) { + return it->second; + } + + shared_ptr newConnection; + try { + newConnection = make_shared( + this, endPoint.value(), zoneId_, nodesSupplier_, fetchSize_, 60, 500, + connectTimeoutMs_, sqlDialect_, database_); + endPointToSessionConnection.emplace(endPoint.value(), newConnection); + return newConnection; + } catch (exception &e) { + log_debug("Session::Impl::getQuerySessionConnection() exception: " + + e.what()); + return newConnection; + } +} + +shared_ptr +Session::Impl::getSessionConnection(std::string deviceId) { + if (!enableRedirection_ || + deviceIdToEndpoint.find(deviceId) == deviceIdToEndpoint.end() || + endPointToSessionConnection.find(deviceIdToEndpoint[deviceId]) == + endPointToSessionConnection.end()) { + return defaultSessionConnection_; + } + return endPointToSessionConnection.find(deviceIdToEndpoint[deviceId])->second; +} + +string Session::getTimeZone() { + auto ret = impl_->defaultSessionConnection_->getTimeZone(); + return ret.timeZone; +} + +void Session::setTimeZone(const string &zoneId) { + TSSetTimeZoneReq req; + req.__set_sessionId(impl_->defaultSessionConnection_->sessionId); + req.__set_timeZone(zoneId); + impl_->defaultSessionConnection_->setTimeZone(req); +} + +unique_ptr Session::executeQueryStatement(const string &sql) { + return executeQueryStatementMayRedirect(sql, QUERY_TIMEOUT_MS); +} + +unique_ptr Session::executeQueryStatement(const string &sql, + int64_t timeoutInMs) { + return executeQueryStatementMayRedirect(sql, timeoutInMs); +} + +void Session::Impl::handleQueryRedirection(TEndPoint endPoint) { + if (!enableRedirection_) + return; + shared_ptr newConnection; + auto it = endPointToSessionConnection.find(endPoint); + if (it != endPointToSessionConnection.end()) { + newConnection = it->second; + } else { + try { + newConnection = make_shared( + this, endPoint, zoneId_, nodesSupplier_, fetchSize_, 60, 500, + connectTimeoutMs_, sqlDialect_, database_); + + endPointToSessionConnection.emplace(endPoint, newConnection); + } catch (exception &e) { + throw IoTDBConnectionException(e.what()); + } + } + defaultSessionConnection_ = newConnection; +} + +void Session::Impl::handleRedirection(const std::string &deviceId, + TEndPoint endPoint) { + if (!enableRedirection_) + return; + if (endPoint.ip == "0.0.0.0") + return; + deviceIdToEndpoint[deviceId] = endPoint; + + shared_ptr newConnection; + auto it = endPointToSessionConnection.find(endPoint); + if (it != endPointToSessionConnection.end()) { + newConnection = it->second; + } else { + try { + newConnection = make_shared( + this, endPoint, zoneId_, nodesSupplier_, fetchSize_, 60, 500, 1000, + sqlDialect_, database_); + endPointToSessionConnection.emplace(endPoint, newConnection); + } catch (exception &e) { + deviceIdToEndpoint.erase(deviceId); + throw IoTDBConnectionException(e.what()); + } + } +} + +std::unique_ptr +Session::executeQueryStatementMayRedirect(const std::string &sql, + int64_t timeoutInMs) { + auto sessionConnection = impl_->getQuerySessionConnection(); + if (!sessionConnection) { + log_warn("Session connection not found"); + return nullptr; + } + try { + return sessionConnection->executeQueryStatement(sql, timeoutInMs); + } catch (RedirectException &e) { + log_warn("Session connection redirect exception: " + e.what()); + impl_->handleQueryRedirection(endpointToThrift(e.endPoint)); + try { + return impl_->defaultSessionConnection_->executeQueryStatement( + sql, timeoutInMs); + } catch (exception &e) { + log_error("Exception while executing redirected query statement: %s", + e.what()); + throw ExecutionException(e.what()); + } + } catch (exception &e) { + log_error("Exception while executing query statement: %s", e.what()); + throw; + } +} + +void Session::executeNonQueryStatement(const string &sql) { + try { + impl_->defaultSessionConnection_->executeNonQueryStatement(sql); + } catch (const exception &e) { + throw IoTDBException(e.what()); + } +} + +unique_ptr +Session::executeRawDataQuery(const vector &paths, int64_t startTime, + int64_t endTime) { + return impl_->defaultSessionConnection_->executeRawDataQuery(paths, startTime, + endTime); +} + +unique_ptr +Session::executeLastDataQuery(const vector &paths) { + return executeLastDataQuery(paths, LONG_LONG_MIN); +} + +unique_ptr +Session::executeLastDataQuery(const vector &paths, int64_t lastTime) { + return impl_->defaultSessionConnection_->executeLastDataQuery(paths, + lastTime); +} + +void Session::createSchemaTemplate(const Template &templ) { + TSCreateSchemaTemplateReq req; + req.__set_name(templ.getName()); + req.__set_serializedTemplate(templ.serialize()); + impl_->defaultSessionConnection_->createSchemaTemplate(req); +} + +void Session::setSchemaTemplate(const string &template_name, + const string &prefix_path) { + TSSetSchemaTemplateReq req; + req.__set_templateName(template_name); + req.__set_prefixPath(prefix_path); + impl_->defaultSessionConnection_->setSchemaTemplate(req); +} + +void Session::unsetSchemaTemplate(const string &prefix_path, + const string &template_name) { + TSUnsetSchemaTemplateReq req; + req.__set_templateName(template_name); + req.__set_prefixPath(prefix_path); + impl_->defaultSessionConnection_->unsetSchemaTemplate(req); +} + +void Session::addAlignedMeasurementsInTemplate( + const string &template_name, const vector &measurements, + const vector &dataTypes, + const vector &encodings, + const vector &compressors) { + TSAppendSchemaTemplateReq req; + req.__set_name(template_name); + req.__set_measurements(measurements); + req.__set_isAligned(true); + + vector dataTypesOrdinal; + dataTypesOrdinal.reserve(dataTypes.size()); + for (TSDataType::TSDataType dataType : dataTypes) { + dataTypesOrdinal.push_back(dataType); + } + req.__set_dataTypes(dataTypesOrdinal); + + vector encodingsOrdinal; + encodingsOrdinal.reserve(encodings.size()); + for (TSEncoding::TSEncoding encoding : encodings) { + encodingsOrdinal.push_back(encoding); + } + req.__set_encodings(encodingsOrdinal); + + vector compressorsOrdinal; + compressorsOrdinal.reserve(compressors.size()); + for (CompressionType::CompressionType compressor : compressors) { + compressorsOrdinal.push_back(compressor); + } + req.__set_compressors(compressorsOrdinal); + + impl_->defaultSessionConnection_->appendSchemaTemplate(req); +} + +void Session::addAlignedMeasurementsInTemplate( + const string &template_name, const string &measurement, + TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, + CompressionType::CompressionType compressor) { + vector measurements(1, measurement); + vector dataTypes(1, dataType); + vector encodings(1, encoding); + vector compressors(1, compressor); + addAlignedMeasurementsInTemplate(template_name, measurements, dataTypes, + encodings, compressors); +} + +void Session::addUnalignedMeasurementsInTemplate( + const string &template_name, const vector &measurements, + const vector &dataTypes, + const vector &encodings, + const vector &compressors) { + TSAppendSchemaTemplateReq req; + req.__set_name(template_name); + req.__set_measurements(measurements); + req.__set_isAligned(false); + + vector dataTypesOrdinal; + dataTypesOrdinal.reserve(dataTypes.size()); + for (TSDataType::TSDataType dataType : dataTypes) { + dataTypesOrdinal.push_back(dataType); + } + req.__set_dataTypes(dataTypesOrdinal); + + vector encodingsOrdinal; + encodingsOrdinal.reserve(encodings.size()); + for (TSEncoding::TSEncoding encoding : encodings) { + encodingsOrdinal.push_back(encoding); + } + req.__set_encodings(encodingsOrdinal); + + vector compressorsOrdinal; + compressorsOrdinal.reserve(compressors.size()); + for (CompressionType::CompressionType compressor : compressors) { + compressorsOrdinal.push_back(compressor); + } + req.__set_compressors(compressorsOrdinal); + + impl_->defaultSessionConnection_->appendSchemaTemplate(req); +} + +void Session::addUnalignedMeasurementsInTemplate( + const string &template_name, const string &measurement, + TSDataType::TSDataType dataType, TSEncoding::TSEncoding encoding, + CompressionType::CompressionType compressor) { + vector measurements(1, measurement); + vector dataTypes(1, dataType); + vector encodings(1, encoding); + vector compressors(1, compressor); + addUnalignedMeasurementsInTemplate(template_name, measurements, dataTypes, + encodings, compressors); +} + +void Session::deleteNodeInTemplate(const string &template_name, + const string &path) { + TSPruneSchemaTemplateReq req; + req.__set_name(template_name); + req.__set_path(path); + impl_->defaultSessionConnection_->pruneSchemaTemplate(req); +} + +int Session::countMeasurementsInTemplate(const string &template_name) { + TSQueryTemplateReq req; + req.__set_name(template_name); + req.__set_queryType(TemplateQueryType::COUNT_MEASUREMENTS); + TSQueryTemplateResp resp = + impl_->defaultSessionConnection_->querySchemaTemplate(req); + return resp.count; +} + +bool Session::isMeasurementInTemplate(const string &template_name, + const string &path) { + TSQueryTemplateReq req; + req.__set_name(template_name); + req.__set_measurement(path); + req.__set_queryType(TemplateQueryType::IS_MEASUREMENT); + TSQueryTemplateResp resp = + impl_->defaultSessionConnection_->querySchemaTemplate(req); + return resp.result; +} + +bool Session::isPathExistInTemplate(const string &template_name, + const string &path) { + TSQueryTemplateReq req; + req.__set_name(template_name); + req.__set_measurement(path); + req.__set_queryType(TemplateQueryType::PATH_EXIST); + TSQueryTemplateResp resp = + impl_->defaultSessionConnection_->querySchemaTemplate(req); + return resp.result; +} + +std::vector +Session::showMeasurementsInTemplate(const string &template_name) { + TSQueryTemplateReq req; + req.__set_name(template_name); + req.__set_measurement(""); + req.__set_queryType(TemplateQueryType::SHOW_MEASUREMENTS); + TSQueryTemplateResp resp = + impl_->defaultSessionConnection_->querySchemaTemplate(req); + return resp.measurements; +} + +std::vector +Session::showMeasurementsInTemplate(const string &template_name, + const string &pattern) { + TSQueryTemplateReq req; + req.__set_name(template_name); + req.__set_measurement(pattern); + req.__set_queryType(TemplateQueryType::SHOW_MEASUREMENTS); + TSQueryTemplateResp resp = + impl_->defaultSessionConnection_->querySchemaTemplate(req); + return resp.measurements; +} + +bool Session::checkTemplateExists(const string &template_name) { + try { + std::unique_ptr dataset = + executeQueryStatement("SHOW NODES IN DEVICE TEMPLATE " + template_name); + if (dataset == nullptr) { + throw IoTDBException("executeQueryStatement failed"); + } + bool isExisted = dataset->hasNext(); + dataset->closeOperationHandle(); + return isExisted; + } catch (const exception &e) { + if (strstr(e.what(), "does not exist") != NULL) { + return false; + } + log_debug(e.what()); + throw IoTDBException(e.what()); + } +} diff --git a/iotdb-client/client-cpp/src/session/SessionC.cpp b/iotdb-client/client-cpp/src/session/SessionC.cpp new file mode 100644 index 0000000000000..58eed24f2d2dd --- /dev/null +++ b/iotdb-client/client-cpp/src/session/SessionC.cpp @@ -0,0 +1,1410 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "SessionC.h" +#include "Session.h" +#include "SessionBuilder.h" +#include "SessionDataSet.h" + +#include +#include +#include +#include +#include +#include + +/* ============================================================ + * Internal wrapper structs — the opaque handles point to these + * ============================================================ */ + +struct CSession_ { + std::shared_ptr cpp; +}; + +struct CTablet_ { + Tablet cpp; +}; + +struct CSessionDataSet_ { + std::unique_ptr cpp; +}; + +struct CRowRecord_ { + std::shared_ptr cpp; +}; + +/* ============================================================ + * Thread-local error message buffer + * ============================================================ */ + +static thread_local std::string g_lastError; + +static void clearError() { g_lastError.clear(); } + +static TsStatus setError(TsStatus code, const std::string &msg) { + g_lastError = msg; + return code; +} + +static TsStatus setError(TsStatus code, const std::exception &e) { + g_lastError = e.what(); + return code; +} + +static TsStatus handleException(const std::exception &e) { +#if defined(_CPPRTTI) || defined(__GXX_RTTI) + // Try to classify exception type (requires RTTI enabled). + if (dynamic_cast(&e)) { + return setError(TS_ERR_CONNECTION, e); + } + if (dynamic_cast(&e) || + dynamic_cast(&e) || + dynamic_cast(&e)) { + return setError(TS_ERR_EXECUTION, e); + } +#endif + return setError(TS_ERR_UNKNOWN, e); +} + +extern "C" { + +const char *ts_get_last_error(void) { return g_lastError.c_str(); } + +} /* extern "C" */ + +/* ============================================================ + * Helpers — convert C arrays to C++ vectors + * ============================================================ */ + +static std::vector toStringVec(const char *const *arr, int count) { + std::vector v; + v.reserve(count); + for (int i = 0; i < count; i++) { + v.emplace_back(arr[i]); + } + return v; +} + +static std::vector toTypeVec(const TSDataType_C *arr, + int count) { + std::vector v; + v.reserve(count); + for (int i = 0; i < count; i++) { + v.push_back(static_cast(arr[i])); + } + return v; +} + +static std::vector +toEncodingVec(const TSEncoding_C *arr, int count) { + std::vector v; + v.reserve(count); + for (int i = 0; i < count; i++) { + v.push_back(static_cast(arr[i])); + } + return v; +} + +static std::vector +toCompressionVec(const TSCompressionType_C *arr, int count) { + std::vector v; + v.reserve(count); + for (int i = 0; i < count; i++) { + v.push_back(static_cast(arr[i])); + } + return v; +} + +static std::map +toStringMap(int count, const char *const *keys, const char *const *values) { + std::map m; + for (int i = 0; i < count; i++) { + m[keys[i]] = values[i]; + } + return m; +} + +/** + * Convert C typed values (void* const* values, TSDataType_C* types, int count) + * to C++ vector that Session expects. + * The caller must free the returned char* pointers using freeCharPtrVec(). + */ +static std::vector toCharPtrVec(const TSDataType_C *types, + const void *const *values, int count) { + std::vector result(count); + for (int i = 0; i < count; i++) { + switch (types[i]) { + case TS_TYPE_BOOLEAN: { + bool *p = new bool(*static_cast(values[i])); + result[i] = reinterpret_cast(p); + break; + } + case TS_TYPE_INT32: { + int32_t *p = new int32_t(*static_cast(values[i])); + result[i] = reinterpret_cast(p); + break; + } + case TS_TYPE_INT64: + case TS_TYPE_TIMESTAMP: { + int64_t *p = new int64_t(*static_cast(values[i])); + result[i] = reinterpret_cast(p); + break; + } + case TS_TYPE_FLOAT: { + float *p = new float(*static_cast(values[i])); + result[i] = reinterpret_cast(p); + break; + } + case TS_TYPE_DOUBLE: { + double *p = new double(*static_cast(values[i])); + result[i] = reinterpret_cast(p); + break; + } + case TS_TYPE_TEXT: + case TS_TYPE_STRING: + case TS_TYPE_BLOB: + default: { + const char *src = static_cast(values[i]); + size_t len = strlen(src) + 1; + char *p = new char[len]; + memcpy(p, src, len); + result[i] = p; + break; + } + } + } + return result; +} + +static void freeCharPtrVec(std::vector &vec, const TSDataType_C *types, + int count) { + for (int i = 0; i < count; i++) { + switch (types[i]) { + case TS_TYPE_BOOLEAN: + delete reinterpret_cast(vec[i]); + break; + case TS_TYPE_INT32: + delete reinterpret_cast(vec[i]); + break; + case TS_TYPE_INT64: + case TS_TYPE_TIMESTAMP: + delete reinterpret_cast(vec[i]); + break; + case TS_TYPE_FLOAT: + delete reinterpret_cast(vec[i]); + break; + case TS_TYPE_DOUBLE: + delete reinterpret_cast(vec[i]); + break; + default: + delete[] vec[i]; + break; + } + } +} + +/* ============================================================ + * Session Lifecycle — Tree Model + * ============================================================ */ + +extern "C" { + +CSession *ts_session_new(const char *host, int rpcPort, const char *username, + const char *password) { + clearError(); + try { + SessionBuilder builder; + builder.host(host) + ->rpcPort(rpcPort) + ->username(username) + ->password(password) + ->enableAutoFetch(false); + auto cpp = std::make_shared(&builder); + auto *cs = new CSession_(); + cs->cpp = std::move(cpp); + return cs; + } catch (const std::exception &e) { + handleException(e); + return nullptr; + } +} + +CSession *ts_session_new_with_zone(const char *host, int rpcPort, + const char *username, const char *password, + const char *zoneId, int fetchSize) { + clearError(); + try { + SessionBuilder builder; + builder.host(host) + ->rpcPort(rpcPort) + ->username(username) + ->password(password) + ->zoneId(zoneId) + ->fetchSize(fetchSize) + ->enableAutoFetch(false); + auto cpp = std::make_shared(&builder); + auto *cs = new CSession_(); + cs->cpp = std::move(cpp); + return cs; + } catch (const std::exception &e) { + handleException(e); + return nullptr; + } +} + +CSession *ts_session_new_multi_node(const char *const *nodeUrls, int urlCount, + const char *username, + const char *password) { + clearError(); + try { + auto urls = toStringVec(nodeUrls, urlCount); + SessionBuilder builder; + builder.nodeUrls(urls) + ->username(username) + ->password(password) + ->enableAutoFetch(false); + auto cpp = std::make_shared(&builder); + auto *cs = new CSession_(); + cs->cpp = std::move(cpp); + return cs; + } catch (const std::exception &e) { + handleException(e); + return nullptr; + } +} + +void ts_session_destroy(CSession *session) { delete session; } + +TsStatus ts_session_open(CSession *session) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + session->cpp->open(); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_open_with_compression(CSession *session, + bool enableRPCCompression) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + session->cpp->open(enableRPCCompression); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_close(CSession *session) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + session->cpp->close(); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +/* ============================================================ + * Timezone + * ============================================================ */ + +TsStatus ts_session_set_timezone(CSession *session, const char *zoneId) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + session->cpp->setTimeZone(std::string(zoneId)); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_get_timezone(CSession *session, char *buf, int bufLen) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + if (!buf || bufLen <= 0) + return setError(TS_ERR_INVALID_PARAM, "invalid buffer"); + try { + std::string tz = session->cpp->getTimeZone(); + if ((int)tz.size() >= bufLen) { + return setError(TS_ERR_INVALID_PARAM, "buffer too small"); + } + strncpy(buf, tz.c_str(), bufLen); + buf[bufLen - 1] = '\0'; + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +/* ============================================================ + * Database Management (Tree Model) + * ============================================================ */ + +TsStatus ts_session_create_database(CSession *session, const char *database) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + session->cpp->createDatabase(std::string(database)); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_delete_database(CSession *session, const char *database) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + session->cpp->deleteDatabase(std::string(database)); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_delete_databases(CSession *session, + const char *const *databases, int count) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto dbs = toStringVec(databases, count); + session->cpp->deleteDatabases(dbs); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +/* ============================================================ + * Timeseries Management (Tree Model) + * ============================================================ */ + +TsStatus ts_session_create_timeseries(CSession *session, const char *path, + TSDataType_C dataType, + TSEncoding_C encoding, + TSCompressionType_C compressor) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + session->cpp->createTimeseries( + std::string(path), static_cast(dataType), + static_cast(encoding), + static_cast(compressor)); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_create_timeseries_ex( + CSession *session, const char *path, TSDataType_C dataType, + TSEncoding_C encoding, TSCompressionType_C compressor, int propsCount, + const char *const *propKeys, const char *const *propValues, int tagsCount, + const char *const *tagKeys, const char *const *tagValues, int attrsCount, + const char *const *attrKeys, const char *const *attrValues, + const char *measurementAlias) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + std::map props = + propsCount > 0 ? toStringMap(propsCount, propKeys, propValues) + : std::map(); + std::map tags = + tagsCount > 0 ? toStringMap(tagsCount, tagKeys, tagValues) + : std::map(); + std::map attrs = + attrsCount > 0 ? toStringMap(attrsCount, attrKeys, attrValues) + : std::map(); + session->cpp->createTimeseries( + std::string(path), static_cast(dataType), + static_cast(encoding), + static_cast(compressor), &props, + &tags, &attrs, std::string(measurementAlias ? measurementAlias : "")); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_create_multi_timeseries( + CSession *session, int count, const char *const *paths, + const TSDataType_C *dataTypes, const TSEncoding_C *encodings, + const TSCompressionType_C *compressors) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto pathsVec = toStringVec(paths, count); + auto typesVec = toTypeVec(dataTypes, count); + auto encVec = toEncodingVec(encodings, count); + auto compVec = toCompressionVec(compressors, count); + session->cpp->createMultiTimeseries(pathsVec, typesVec, encVec, compVec, + nullptr, nullptr, nullptr, nullptr); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_create_aligned_timeseries( + CSession *session, const char *deviceId, int count, + const char *const *measurements, const TSDataType_C *dataTypes, + const TSEncoding_C *encodings, const TSCompressionType_C *compressors) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto measurementsVec = toStringVec(measurements, count); + auto typesVec = toTypeVec(dataTypes, count); + auto encVec = toEncodingVec(encodings, count); + auto compVec = toCompressionVec(compressors, count); + session->cpp->createAlignedTimeseries( + std::string(deviceId), measurementsVec, typesVec, encVec, compVec); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_check_timeseries_exists(CSession *session, const char *path, + bool *exists) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + if (!exists) + return setError(TS_ERR_INVALID_PARAM, "exists pointer is null"); + try { + *exists = session->cpp->checkTimeseriesExists(std::string(path)); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_delete_timeseries(CSession *session, const char *path) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + session->cpp->deleteTimeseries(std::string(path)); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_delete_timeseries_batch(CSession *session, + const char *const *paths, + int count) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto pathsVec = toStringVec(paths, count); + session->cpp->deleteTimeseries(pathsVec); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +/* ============================================================ + * Tablet Operations + * ============================================================ */ + +CTablet *ts_tablet_new(const char *deviceId, int columnCount, + const char *const *columnNames, + const TSDataType_C *dataTypes, int maxRowNumber) { + try { + std::vector> schemas; + schemas.reserve(columnCount); + for (int i = 0; i < columnCount; i++) { + schemas.emplace_back(std::string(columnNames[i]), + static_cast(dataTypes[i])); + } + Tablet tablet(std::string(deviceId), schemas, maxRowNumber); + auto *ct = new CTablet_(); + ct->cpp = std::move(tablet); + return ct; + } catch (const std::exception &e) { + handleException(e); + return nullptr; + } +} + +void ts_tablet_destroy(CTablet *tablet) { delete tablet; } + +void ts_tablet_reset(CTablet *tablet) { + if (tablet) { + tablet->cpp.reset(); + } +} + +int ts_tablet_get_row_count(CTablet *tablet) { + if (!tablet) + return 0; + return static_cast(tablet->cpp.rowSize); +} + +TsStatus ts_tablet_set_row_count(CTablet *tablet, int rowCount) { + clearError(); + if (!tablet) + return setError(TS_ERR_NULL_PTR, "tablet is null"); + tablet->cpp.rowSize = rowCount; + return TS_OK; +} + +TsStatus ts_tablet_add_timestamp(CTablet *tablet, int rowIndex, + int64_t timestamp) { + clearError(); + if (!tablet) + return setError(TS_ERR_NULL_PTR, "tablet is null"); + try { + tablet->cpp.addTimestamp(static_cast(rowIndex), timestamp); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_tablet_add_value_bool(CTablet *tablet, int colIndex, int rowIndex, + bool value) { + clearError(); + if (!tablet) + return setError(TS_ERR_NULL_PTR, "tablet is null"); + try { + tablet->cpp.addValue(static_cast(colIndex), + static_cast(rowIndex), value); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_tablet_add_value_int32(CTablet *tablet, int colIndex, int rowIndex, + int32_t value) { + clearError(); + if (!tablet) + return setError(TS_ERR_NULL_PTR, "tablet is null"); + try { + tablet->cpp.addValue(static_cast(colIndex), + static_cast(rowIndex), value); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_tablet_add_value_int64(CTablet *tablet, int colIndex, int rowIndex, + int64_t value) { + clearError(); + if (!tablet) + return setError(TS_ERR_NULL_PTR, "tablet is null"); + try { + tablet->cpp.addValue(static_cast(colIndex), + static_cast(rowIndex), value); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_tablet_add_value_float(CTablet *tablet, int colIndex, int rowIndex, + float value) { + clearError(); + if (!tablet) + return setError(TS_ERR_NULL_PTR, "tablet is null"); + try { + tablet->cpp.addValue(static_cast(colIndex), + static_cast(rowIndex), value); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_tablet_add_value_double(CTablet *tablet, int colIndex, int rowIndex, + double value) { + clearError(); + if (!tablet) + return setError(TS_ERR_NULL_PTR, "tablet is null"); + try { + tablet->cpp.addValue(static_cast(colIndex), + static_cast(rowIndex), value); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_tablet_add_value_string(CTablet *tablet, int colIndex, int rowIndex, + const char *value) { + clearError(); + if (!tablet) + return setError(TS_ERR_NULL_PTR, "tablet is null"); + try { + std::string str(value); + tablet->cpp.addValue(static_cast(colIndex), + static_cast(rowIndex), str); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +/* ============================================================ + * Data Insertion — Tree Model (Record, string values) + * ============================================================ */ + +TsStatus ts_session_insert_record_str(CSession *session, const char *deviceId, + int64_t time, int count, + const char *const *measurements, + const char *const *values) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto measurementsVec = toStringVec(measurements, count); + auto valuesVec = toStringVec(values, count); + session->cpp->insertRecord(std::string(deviceId), time, measurementsVec, + valuesVec); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_insert_record(CSession *session, const char *deviceId, + int64_t time, int count, + const char *const *measurements, + const TSDataType_C *types, + const void *const *values) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto measurementsVec = toStringVec(measurements, count); + auto typesVec = toTypeVec(types, count); + auto charVec = toCharPtrVec(types, values, count); + session->cpp->insertRecord(std::string(deviceId), time, measurementsVec, + typesVec, charVec); + freeCharPtrVec(charVec, types, count); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_insert_aligned_record_str(CSession *session, + const char *deviceId, + int64_t time, int count, + const char *const *measurements, + const char *const *values) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto measurementsVec = toStringVec(measurements, count); + auto valuesVec = toStringVec(values, count); + session->cpp->insertAlignedRecord(std::string(deviceId), time, + measurementsVec, valuesVec); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_insert_aligned_record(CSession *session, + const char *deviceId, int64_t time, + int count, + const char *const *measurements, + const TSDataType_C *types, + const void *const *values) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto measurementsVec = toStringVec(measurements, count); + auto typesVec = toTypeVec(types, count); + auto charVec = toCharPtrVec(types, values, count); + session->cpp->insertAlignedRecord(std::string(deviceId), time, + measurementsVec, typesVec, charVec); + freeCharPtrVec(charVec, types, count); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +/* ============================================================ + * Data Insertion — Batch, multiple devices (string values) + * ============================================================ */ + +TsStatus ts_session_insert_records_str( + CSession *session, int deviceCount, const char *const *deviceIds, + const int64_t *times, const int *measurementCounts, + const char *const *const *measurementsList, + const char *const *const *valuesList) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto devVec = toStringVec(deviceIds, deviceCount); + std::vector timesVec(times, times + deviceCount); + std::vector> mList, vList; + mList.reserve(deviceCount); + vList.reserve(deviceCount); + for (int i = 0; i < deviceCount; i++) { + mList.push_back(toStringVec(measurementsList[i], measurementCounts[i])); + vList.push_back(toStringVec(valuesList[i], measurementCounts[i])); + } + session->cpp->insertRecords(devVec, timesVec, mList, vList); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_insert_aligned_records_str( + CSession *session, int deviceCount, const char *const *deviceIds, + const int64_t *times, const int *measurementCounts, + const char *const *const *measurementsList, + const char *const *const *valuesList) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto devVec = toStringVec(deviceIds, deviceCount); + std::vector timesVec(times, times + deviceCount); + std::vector> mList, vList; + mList.reserve(deviceCount); + vList.reserve(deviceCount); + for (int i = 0; i < deviceCount; i++) { + mList.push_back(toStringVec(measurementsList[i], measurementCounts[i])); + vList.push_back(toStringVec(valuesList[i], measurementCounts[i])); + } + session->cpp->insertAlignedRecords(devVec, timesVec, mList, vList); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +/* ============================================================ + * Data Insertion — Batch, multiple devices (typed values) + * ============================================================ */ + +TsStatus ts_session_insert_records(CSession *session, int deviceCount, + const char *const *deviceIds, + const int64_t *times, + const int *measurementCounts, + const char *const *const *measurementsList, + const TSDataType_C *const *typesList, + const void *const *const *valuesList) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto devVec = toStringVec(deviceIds, deviceCount); + std::vector timesVec(times, times + deviceCount); + std::vector> mList; + std::vector> tList; + std::vector> vList; + mList.reserve(deviceCount); + tList.reserve(deviceCount); + vList.reserve(deviceCount); + for (int i = 0; i < deviceCount; i++) { + mList.push_back(toStringVec(measurementsList[i], measurementCounts[i])); + tList.push_back(toTypeVec(typesList[i], measurementCounts[i])); + vList.push_back( + toCharPtrVec(typesList[i], valuesList[i], measurementCounts[i])); + } + session->cpp->insertRecords(devVec, timesVec, mList, tList, vList); + for (int i = 0; i < deviceCount; i++) { + freeCharPtrVec(vList[i], typesList[i], measurementCounts[i]); + } + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_insert_aligned_records( + CSession *session, int deviceCount, const char *const *deviceIds, + const int64_t *times, const int *measurementCounts, + const char *const *const *measurementsList, + const TSDataType_C *const *typesList, + const void *const *const *valuesList) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto devVec = toStringVec(deviceIds, deviceCount); + std::vector timesVec(times, times + deviceCount); + std::vector> mList; + std::vector> tList; + std::vector> vList; + mList.reserve(deviceCount); + tList.reserve(deviceCount); + vList.reserve(deviceCount); + for (int i = 0; i < deviceCount; i++) { + mList.push_back(toStringVec(measurementsList[i], measurementCounts[i])); + tList.push_back(toTypeVec(typesList[i], measurementCounts[i])); + vList.push_back( + toCharPtrVec(typesList[i], valuesList[i], measurementCounts[i])); + } + session->cpp->insertAlignedRecords(devVec, timesVec, mList, tList, vList); + for (int i = 0; i < deviceCount; i++) { + freeCharPtrVec(vList[i], typesList[i], measurementCounts[i]); + } + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +/* ============================================================ + * Data Insertion — Batch, single device (typed values) + * ============================================================ */ + +TsStatus ts_session_insert_records_of_one_device( + CSession *session, const char *deviceId, int rowCount, const int64_t *times, + const int *measurementCounts, const char *const *const *measurementsList, + const TSDataType_C *const *typesList, const void *const *const *valuesList, + bool sorted) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + std::vector timesVec(times, times + rowCount); + std::vector> mList; + std::vector> tList; + std::vector> vList; + mList.reserve(rowCount); + tList.reserve(rowCount); + vList.reserve(rowCount); + for (int i = 0; i < rowCount; i++) { + mList.push_back(toStringVec(measurementsList[i], measurementCounts[i])); + tList.push_back(toTypeVec(typesList[i], measurementCounts[i])); + vList.push_back( + toCharPtrVec(typesList[i], valuesList[i], measurementCounts[i])); + } + session->cpp->insertRecordsOfOneDevice(std::string(deviceId), timesVec, + mList, tList, vList, sorted); + for (int i = 0; i < rowCount; i++) { + freeCharPtrVec(vList[i], typesList[i], measurementCounts[i]); + } + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_insert_aligned_records_of_one_device( + CSession *session, const char *deviceId, int rowCount, const int64_t *times, + const int *measurementCounts, const char *const *const *measurementsList, + const TSDataType_C *const *typesList, const void *const *const *valuesList, + bool sorted) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + std::vector timesVec(times, times + rowCount); + std::vector> mList; + std::vector> tList; + std::vector> vList; + mList.reserve(rowCount); + tList.reserve(rowCount); + vList.reserve(rowCount); + for (int i = 0; i < rowCount; i++) { + mList.push_back(toStringVec(measurementsList[i], measurementCounts[i])); + tList.push_back(toTypeVec(typesList[i], measurementCounts[i])); + vList.push_back( + toCharPtrVec(typesList[i], valuesList[i], measurementCounts[i])); + } + session->cpp->insertAlignedRecordsOfOneDevice( + std::string(deviceId), timesVec, mList, tList, vList, sorted); + for (int i = 0; i < rowCount; i++) { + freeCharPtrVec(vList[i], typesList[i], measurementCounts[i]); + } + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +/* ============================================================ + * Data Insertion — Tree Model (Tablet) + * ============================================================ */ + +TsStatus ts_session_insert_tablet(CSession *session, CTablet *tablet, + bool sorted) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + if (!tablet) + return setError(TS_ERR_NULL_PTR, "tablet is null"); + try { + session->cpp->insertTablet(tablet->cpp, sorted); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_insert_aligned_tablet(CSession *session, CTablet *tablet, + bool sorted) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + if (!tablet) + return setError(TS_ERR_NULL_PTR, "tablet is null"); + try { + session->cpp->insertAlignedTablet(tablet->cpp, sorted); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_insert_tablets(CSession *session, int tabletCount, + const char *const *deviceIds, + CTablet **tablets, bool sorted) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + std::unordered_map tabletMap; + for (int i = 0; i < tabletCount; i++) { + tabletMap[std::string(deviceIds[i])] = &(tablets[i]->cpp); + } + session->cpp->insertTablets(tabletMap, sorted); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_insert_aligned_tablets(CSession *session, int tabletCount, + const char *const *deviceIds, + CTablet **tablets, bool sorted) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + std::unordered_map tabletMap; + for (int i = 0; i < tabletCount; i++) { + tabletMap[std::string(deviceIds[i])] = &(tablets[i]->cpp); + } + session->cpp->insertAlignedTablets(tabletMap, sorted); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +/* ============================================================ + * Query — Tree Model + * ============================================================ */ + +TsStatus ts_session_execute_query(CSession *session, const char *sql, + CSessionDataSet **dataSet) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + if (!dataSet) + return setError(TS_ERR_INVALID_PARAM, "dataSet pointer is null"); + try { + auto ds = session->cpp->executeQueryStatement(std::string(sql)); + CSessionDataSet_ tmp{}; + tmp.cpp = std::move(ds); + auto *cds = new CSessionDataSet_(); + cds->cpp = std::move(tmp.cpp); + *dataSet = cds; + return TS_OK; + } catch (const std::exception &e) { + *dataSet = nullptr; + return handleException(e); + } +} + +TsStatus ts_session_execute_query_with_timeout(CSession *session, + const char *sql, + int64_t timeoutInMs, + CSessionDataSet **dataSet) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + if (!dataSet) + return setError(TS_ERR_INVALID_PARAM, "dataSet pointer is null"); + try { + auto ds = + session->cpp->executeQueryStatement(std::string(sql), timeoutInMs); + CSessionDataSet_ tmp{}; + tmp.cpp = std::move(ds); + auto *cds = new CSessionDataSet_(); + cds->cpp = std::move(tmp.cpp); + *dataSet = cds; + return TS_OK; + } catch (const std::exception &e) { + *dataSet = nullptr; + return handleException(e); + } +} + +TsStatus ts_session_execute_non_query(CSession *session, const char *sql) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + session->cpp->executeNonQueryStatement(std::string(sql)); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_execute_raw_data_query(CSession *session, int pathCount, + const char *const *paths, + int64_t startTime, int64_t endTime, + CSessionDataSet **dataSet) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + if (!dataSet) + return setError(TS_ERR_INVALID_PARAM, "dataSet pointer is null"); + try { + auto pathsVec = toStringVec(paths, pathCount); + auto ds = session->cpp->executeRawDataQuery(pathsVec, startTime, endTime); + CSessionDataSet_ tmp{}; + tmp.cpp = std::move(ds); + auto *cds = new CSessionDataSet_(); + cds->cpp = std::move(tmp.cpp); + *dataSet = cds; + return TS_OK; + } catch (const std::exception &e) { + *dataSet = nullptr; + return handleException(e); + } +} + +TsStatus ts_session_execute_last_data_query(CSession *session, int pathCount, + const char *const *paths, + CSessionDataSet **dataSet) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + if (!dataSet) + return setError(TS_ERR_INVALID_PARAM, "dataSet pointer is null"); + try { + auto pathsVec = toStringVec(paths, pathCount); + auto ds = session->cpp->executeLastDataQuery(pathsVec); + CSessionDataSet_ tmp{}; + tmp.cpp = std::move(ds); + auto *cds = new CSessionDataSet_(); + cds->cpp = std::move(tmp.cpp); + *dataSet = cds; + return TS_OK; + } catch (const std::exception &e) { + *dataSet = nullptr; + return handleException(e); + } +} + +TsStatus ts_session_execute_last_data_query_with_time( + CSession *session, int pathCount, const char *const *paths, + int64_t lastTime, CSessionDataSet **dataSet) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + if (!dataSet) + return setError(TS_ERR_INVALID_PARAM, "dataSet pointer is null"); + try { + auto pathsVec = toStringVec(paths, pathCount); + auto ds = session->cpp->executeLastDataQuery(pathsVec, lastTime); + CSessionDataSet_ tmp{}; + tmp.cpp = std::move(ds); + auto *cds = new CSessionDataSet_(); + cds->cpp = std::move(tmp.cpp); + *dataSet = cds; + return TS_OK; + } catch (const std::exception &e) { + *dataSet = nullptr; + return handleException(e); + } +} + +/* ============================================================ + * SessionDataSet & RowRecord — Result Iteration + * ============================================================ */ + +void ts_dataset_destroy(CSessionDataSet *dataSet) { + if (dataSet) { + if (dataSet->cpp) { + dataSet->cpp->closeOperationHandle(); + } + delete dataSet; + } +} + +bool ts_dataset_has_next(CSessionDataSet *dataSet) { + clearError(); + if (!dataSet) { + (void)setError(TS_ERR_NULL_PTR, "dataSet is null"); + return false; + } + if (!dataSet->cpp) { + (void)setError(TS_ERR_NULL_PTR, "dataSet is not initialized"); + return false; + } + try { + return dataSet->cpp->hasNext(); + } catch (const std::exception &e) { + (void)handleException(e); + return false; + } catch (...) { + (void)setError(TS_ERR_UNKNOWN, "non-standard exception"); + return false; + } +} + +CRowRecord *ts_dataset_next(CSessionDataSet *dataSet) { + clearError(); + if (!dataSet) { + (void)setError(TS_ERR_NULL_PTR, "dataSet is null"); + return nullptr; + } + if (!dataSet->cpp) { + (void)setError(TS_ERR_NULL_PTR, "dataSet is not initialized"); + return nullptr; + } + try { + auto row = dataSet->cpp->next(); + if (!row) + return nullptr; + CRowRecord_ tmp{}; + tmp.cpp = std::move(row); + auto *crr = new CRowRecord_(); + crr->cpp = std::move(tmp.cpp); + return crr; + } catch (const std::exception &e) { + (void)handleException(e); + return nullptr; + } catch (...) { + (void)setError(TS_ERR_UNKNOWN, "non-standard exception"); + return nullptr; + } +} + +int ts_dataset_get_column_count(CSessionDataSet *dataSet) { + if (!dataSet || !dataSet->cpp) + return 0; + return static_cast(dataSet->cpp->getColumnNames().size()); +} + +static thread_local std::string g_colNameBuf; + +const char *ts_dataset_get_column_name(CSessionDataSet *dataSet, int index) { + if (!dataSet || !dataSet->cpp) + return ""; + const auto &names = dataSet->cpp->getColumnNames(); + if (index < 0 || index >= (int)names.size()) + return ""; + g_colNameBuf = names[index]; + return g_colNameBuf.c_str(); +} + +static thread_local std::string g_colTypeBuf; + +const char *ts_dataset_get_column_type(CSessionDataSet *dataSet, int index) { + if (!dataSet || !dataSet->cpp) + return ""; + const auto &types = dataSet->cpp->getColumnTypeList(); + if (index < 0 || index >= (int)types.size()) + return ""; + g_colTypeBuf = types[index]; + return g_colTypeBuf.c_str(); +} + +void ts_dataset_set_fetch_size(CSessionDataSet *dataSet, int fetchSize) { + if (dataSet && dataSet->cpp) { + dataSet->cpp->setFetchSize(fetchSize); + } +} + +void ts_row_record_destroy(CRowRecord *record) { delete record; } + +int64_t ts_row_record_get_timestamp(CRowRecord *record) { + if (!record || !record->cpp) + return -1; + return record->cpp->timestamp; +} + +int ts_row_record_get_field_count(CRowRecord *record) { + if (!record || !record->cpp) + return 0; + return static_cast(record->cpp->fields.size()); +} + +bool ts_row_record_is_null(CRowRecord *record, int index) { + if (!record || !record->cpp) + return true; + if (index < 0 || index >= (int)record->cpp->fields.size()) + return true; + return record->cpp->fields[index].isNull(); +} + +bool ts_row_record_get_bool(CRowRecord *record, int index) { + if (!record || !record->cpp) + return false; + if (index < 0 || index >= (int)record->cpp->fields.size()) + return false; + const Field &f = record->cpp->fields[index]; + return f.boolV.is_initialized() ? f.boolV.value() : false; +} + +int32_t ts_row_record_get_int32(CRowRecord *record, int index) { + if (!record || !record->cpp) + return 0; + if (index < 0 || index >= (int)record->cpp->fields.size()) + return 0; + const Field &f = record->cpp->fields[index]; + return f.intV.is_initialized() ? f.intV.value() : 0; +} + +int64_t ts_row_record_get_int64(CRowRecord *record, int index) { + if (!record || !record->cpp) + return 0; + if (index < 0 || index >= (int)record->cpp->fields.size()) + return 0; + const Field &f = record->cpp->fields[index]; + if (f.longV.is_initialized()) + return f.longV.value(); + if (f.intV.is_initialized()) + return f.intV.value(); + if (f.doubleV.is_initialized()) + return static_cast(f.doubleV.value()); + return 0; +} + +float ts_row_record_get_float(CRowRecord *record, int index) { + if (!record || !record->cpp) + return 0.0f; + if (index < 0 || index >= (int)record->cpp->fields.size()) + return 0.0f; + const Field &f = record->cpp->fields[index]; + return f.floatV.is_initialized() ? f.floatV.value() : 0.0f; +} + +double ts_row_record_get_double(CRowRecord *record, int index) { + if (!record || !record->cpp) + return 0.0; + if (index < 0 || index >= (int)record->cpp->fields.size()) + return 0.0; + const Field &f = record->cpp->fields[index]; + return f.doubleV.is_initialized() ? f.doubleV.value() : 0.0; +} + +static thread_local std::string g_stringBuf; + +const char *ts_row_record_get_string(CRowRecord *record, int index) { + if (!record || !record->cpp) + return ""; + if (index < 0 || index >= (int)record->cpp->fields.size()) + return ""; + const Field &f = record->cpp->fields[index]; + if (f.stringV.is_initialized()) { + g_stringBuf = f.stringV.value(); + return g_stringBuf.c_str(); + } + return ""; +} + +TSDataType_C ts_row_record_get_data_type(CRowRecord *record, int index) { + if (!record || !record->cpp) + return TS_TYPE_INVALID; + if (index < 0 || index >= (int)record->cpp->fields.size()) + return TS_TYPE_INVALID; + return static_cast(record->cpp->fields[index].dataType); +} + +/* ============================================================ + * Data Deletion (Tree Model) + * ============================================================ */ + +TsStatus ts_session_delete_data(CSession *session, const char *path, + int64_t endTime) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + session->cpp->deleteData(std::string(path), endTime); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_delete_data_batch(CSession *session, int pathCount, + const char *const *paths, + int64_t endTime) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto pathsVec = toStringVec(paths, pathCount); + session->cpp->deleteData(pathsVec, endTime); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +TsStatus ts_session_delete_data_range(CSession *session, int pathCount, + const char *const *paths, + int64_t startTime, int64_t endTime) { + clearError(); + if (!session) + return setError(TS_ERR_NULL_PTR, "session is null"); + try { + auto pathsVec = toStringVec(paths, pathCount); + session->cpp->deleteData(pathsVec, startTime, endTime); + return TS_OK; + } catch (const std::exception &e) { + return handleException(e); + } +} + +} /* extern "C" */ diff --git a/iotdb-client/client-cpp/src/session/SessionDataSet.cpp b/iotdb-client/client-cpp/src/session/SessionDataSet.cpp new file mode 100644 index 0000000000000..94a81d3c58d65 --- /dev/null +++ b/iotdb-client/client-cpp/src/session/SessionDataSet.cpp @@ -0,0 +1,341 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "SessionDataSet.h" + +#include + +#include "IoTDBRpcDataSet.h" +#include "SessionDataSetFactory.h" + +using namespace std; + +struct SessionDataSet::Impl { + std::shared_ptr iotdbRpcDataSet_; +}; + +std::unique_ptr createSessionDataSet( + const std::string &sql, const std::vector &columnNameList, + const std::vector &columnTypeList, + const std::map &columnNameIndex, int64_t queryId, + int64_t statementId, std::shared_ptr client, + int64_t sessionId, const std::vector &queryResult, + bool ignoreTimestamp, int64_t timeout, bool moreData, int32_t fetchSize, + const std::string &zoneId) { + auto dataSet = std::unique_ptr(new SessionDataSet()); + dataSet->impl_ = + std::unique_ptr(new SessionDataSet::Impl()); + dataSet->impl_->iotdbRpcDataSet_ = std::make_shared( + sql, columnNameList, columnTypeList, columnNameIndex, ignoreTimestamp, + moreData, queryId, statementId, client, sessionId, queryResult, fetchSize, + timeout, zoneId, IoTDBRpcDataSet::DEFAULT_TIME_FORMAT); + return dataSet; +} + +RowRecord::RowRecord(int64_t timestamp) { this->timestamp = timestamp; } + +RowRecord::RowRecord(int64_t timestamp, const std::vector &fields) + : timestamp(timestamp), fields(fields) {} + +RowRecord::RowRecord(const std::vector &fields) + : timestamp(-1), fields(fields) {} + +RowRecord::RowRecord() { this->timestamp = -1; } + +void RowRecord::addField(const Field &f) { this->fields.push_back(f); } + +std::string RowRecord::toString() { + std::string ret; + if (this->timestamp != -1) { + ret.append(std::to_string(timestamp)); + ret.append("\t"); + } + for (size_t i = 0; i < fields.size(); i++) { + if (i != 0) { + ret.append("\t"); + } + const Field &f = fields[i]; + switch (f.dataType) { + case TSDataType::BOOLEAN: + if (f.isNull()) { + ret.append("null"); + } else { + ret.append(f.boolV.value() ? "true" : "false"); + } + break; + case TSDataType::INT32: + if (f.isNull()) { + ret.append("null"); + } else { + ret.append(std::to_string(f.intV.value())); + } + break; + case TSDataType::DATE: + if (f.isNull()) { + ret.append("null"); + } else { + ret.append(f.dateV.value().toIsoExtendedString()); + } + break; + case TSDataType::TIMESTAMP: + case TSDataType::INT64: + if (f.isNull()) { + ret.append("null"); + } else { + ret.append(std::to_string(f.longV.value())); + } + break; + case TSDataType::FLOAT: + if (f.isNull()) { + ret.append("null"); + } else { + ret.append(std::to_string(f.floatV.value())); + } + break; + case TSDataType::DOUBLE: + if (f.isNull()) { + ret.append("null"); + } else { + ret.append(std::to_string(f.doubleV.value())); + } + break; + case TSDataType::BLOB: + case TSDataType::STRING: + case TSDataType::TEXT: + if (f.isNull()) { + ret.append("null"); + } else { + ret.append(f.stringV.value()); + } + break; + case TSDataType::OBJECT: + if (!f.stringV.is_initialized()) { + ret.append("null"); + } else { + ret.append(f.stringV.value()); + } + break; + default: + break; + } + } + ret.append("\n"); + return ret; +} + +SessionDataSet::~SessionDataSet() = default; + +bool SessionDataSet::hasNext() { + if (impl_->iotdbRpcDataSet_->hasCachedRecord()) { + return true; + } + return impl_->iotdbRpcDataSet_->next(); +} + +shared_ptr SessionDataSet::next() { + if (!impl_->iotdbRpcDataSet_->hasCachedRecord() && !hasNext()) { + return nullptr; + } + impl_->iotdbRpcDataSet_->setHasCachedRecord(false); + return constructRowRecordFromValueArray(); +} + +int SessionDataSet::getFetchSize() { + return impl_->iotdbRpcDataSet_->getFetchSize(); +} + +void SessionDataSet::setFetchSize(int fetchSize) { + impl_->iotdbRpcDataSet_->setFetchSize(fetchSize); +} + +const std::vector &SessionDataSet::getColumnNames() const { + return impl_->iotdbRpcDataSet_->getColumnNameList(); +} + +const std::vector &SessionDataSet::getColumnTypeList() const { + return impl_->iotdbRpcDataSet_->getColumnTypeList(); +} + +void SessionDataSet::closeOperationHandle(bool forceClose) { + impl_->iotdbRpcDataSet_->close(forceClose); +} + +SessionDataSet::DataIterator::DataIterator(std::shared_ptr impl) + : impl_(std::move(impl)) {} + +bool SessionDataSet::DataIterator::next() { + return impl_->iotdbRpcDataSet_->next(); +} + +bool SessionDataSet::DataIterator::isNull(const std::string &columnName) { + return impl_->iotdbRpcDataSet_->isNullByColumnName(columnName); +} + +bool SessionDataSet::DataIterator::isNullByIndex(int32_t columnIndex) { + return impl_->iotdbRpcDataSet_->isNullByIndex(columnIndex); +} + +Optional +SessionDataSet::DataIterator::getBooleanByIndex(int32_t columnIndex) { + return impl_->iotdbRpcDataSet_->getBooleanByIndex(columnIndex); +} + +Optional +SessionDataSet::DataIterator::getBoolean(const std::string &columnName) { + return impl_->iotdbRpcDataSet_->getBoolean(columnName); +} + +Optional +SessionDataSet::DataIterator::getDoubleByIndex(int32_t columnIndex) { + return impl_->iotdbRpcDataSet_->getDoubleByIndex(columnIndex); +} + +Optional +SessionDataSet::DataIterator::getDouble(const std::string &columnName) { + return impl_->iotdbRpcDataSet_->getDouble(columnName); +} + +Optional +SessionDataSet::DataIterator::getFloatByIndex(int32_t columnIndex) { + return impl_->iotdbRpcDataSet_->getFloatByIndex(columnIndex); +} + +Optional +SessionDataSet::DataIterator::getFloat(const std::string &columnName) { + return impl_->iotdbRpcDataSet_->getFloat(columnName); +} + +Optional +SessionDataSet::DataIterator::getIntByIndex(int32_t columnIndex) { + return impl_->iotdbRpcDataSet_->getIntByIndex(columnIndex); +} + +Optional +SessionDataSet::DataIterator::getInt(const std::string &columnName) { + return impl_->iotdbRpcDataSet_->getInt(columnName); +} + +Optional +SessionDataSet::DataIterator::getLongByIndex(int32_t columnIndex) { + return impl_->iotdbRpcDataSet_->getLongByIndex(columnIndex); +} + +Optional +SessionDataSet::DataIterator::getLong(const std::string &columnName) { + return impl_->iotdbRpcDataSet_->getLong(columnName); +} + +Optional +SessionDataSet::DataIterator::getStringByIndex(int32_t columnIndex) { + return impl_->iotdbRpcDataSet_->getStringByIndex(columnIndex); +} + +Optional +SessionDataSet::DataIterator::getString(const std::string &columnName) { + return impl_->iotdbRpcDataSet_->getString(columnName); +} + +Optional +SessionDataSet::DataIterator::getTimestampByIndex(int32_t columnIndex) { + return impl_->iotdbRpcDataSet_->getTimestampByIndex(columnIndex); +} + +Optional +SessionDataSet::DataIterator::getTimestamp(const std::string &columnName) { + return impl_->iotdbRpcDataSet_->getTimestamp(columnName); +} + +Optional +SessionDataSet::DataIterator::getDateByIndex(int32_t columnIndex) { + return impl_->iotdbRpcDataSet_->getDateByIndex(columnIndex); +} + +Optional +SessionDataSet::DataIterator::getDate(const std::string &columnName) { + return impl_->iotdbRpcDataSet_->getDate(columnName); +} + +int32_t +SessionDataSet::DataIterator::findColumn(const std::string &columnName) { + return impl_->iotdbRpcDataSet_->findColumn(columnName); +} + +const std::vector & +SessionDataSet::DataIterator::getColumnNames() const { + return impl_->iotdbRpcDataSet_->getColumnNameList(); +} + +const std::vector & +SessionDataSet::DataIterator::getColumnTypeList() const { + return impl_->iotdbRpcDataSet_->getColumnTypeList(); +} + +SessionDataSet::DataIterator SessionDataSet::getIterator() { + return DataIterator(std::shared_ptr(impl_.get(), [](Impl *) {})); +} + +shared_ptr SessionDataSet::constructRowRecordFromValueArray() { + std::vector outFields; + const auto &dataSet = impl_->iotdbRpcDataSet_; + const int32_t valueColumnCount = dataSet->getServerColumnCount(); + for (int i = 0; i < valueColumnCount; i++) { + const int32_t listIndex = dataSet->getValueColumnNameListIndex(i); + const std::string &columnName = dataSet->getColumnNameList().at(listIndex); + Field field; + if (!dataSet->isNullByColumnName(columnName)) { + TSDataType::TSDataType dataType = dataSet->getDataType(columnName); + field.dataType = dataType; + switch (dataType) { + case TSDataType::BOOLEAN: + field.boolV = dataSet->getBoolean(columnName); + break; + case TSDataType::INT32: + field.intV = dataSet->getInt(columnName); + break; + case TSDataType::DATE: + field.dateV = dataSet->getDate(columnName); + break; + case TSDataType::INT64: + case TSDataType::TIMESTAMP: + field.longV = dataSet->getLong(columnName); + break; + case TSDataType::FLOAT: + field.floatV = dataSet->getFloat(columnName); + break; + case TSDataType::DOUBLE: + field.doubleV = dataSet->getDouble(columnName); + break; + case TSDataType::TEXT: + case TSDataType::BLOB: + case TSDataType::STRING: + case TSDataType::OBJECT: { + auto stringValue = dataSet->getString(columnName); + if (stringValue.is_initialized()) { + field.stringV = stringValue.value(); + } + break; + } + default: + throw UnSupportedDataTypeException("Data type is not supported."); + } + } + outFields.emplace_back(field); + } + return std::make_shared(dataSet->getCurrentRowTime(), outFields); +} diff --git a/iotdb-client/client-cpp/src/session/SessionPool.cpp b/iotdb-client/client-cpp/src/session/SessionPool.cpp new file mode 100644 index 0000000000000..67df6ca1a40e5 --- /dev/null +++ b/iotdb-client/client-cpp/src/session/SessionPool.cpp @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "SessionPool.h" + +void PooledSession::reset() { + if (session_ && pool_ != nullptr) { + pool_->putBack(session_, broken_); + } + pool_ = nullptr; + session_ = nullptr; + broken_ = false; +} + +SessionPool::SessionPool(std::string host, int rpcPort, std::string username, + std::string password, size_t maxSize) + : host_(std::move(host)), rpcPort_(rpcPort), username_(std::move(username)), + password_(std::move(password)), maxSize_(maxSize) { + if (maxSize_ == 0) { + throw IoTDBException("SessionPool maxSize must be greater than 0."); + } +} + +SessionPool::~SessionPool() { + try { + close(); + } catch (const std::exception &e) { + log_debug(std::string("SessionPool::~SessionPool(), ") + e.what()); + } +} + +SessionPool &SessionPool::setFetchSize(int fetchSize) { + fetchSize_ = fetchSize; + return *this; +} + +SessionPool &SessionPool::setZoneId(std::string zoneId) { + zoneId_ = std::move(zoneId); + return *this; +} + +SessionPool &SessionPool::setSqlDialect(std::string sqlDialect) { + sqlDialect_ = std::move(sqlDialect); + return *this; +} + +SessionPool &SessionPool::setDatabase(std::string database) { + database_ = std::move(database); + return *this; +} + +SessionPool &SessionPool::setEnableRedirection(bool enable) { + enableRedirection_ = enable; + return *this; +} + +SessionPool &SessionPool::setEnableAutoFetch(bool enable) { + enableAutoFetch_ = enable; + return *this; +} + +SessionPool &SessionPool::setEnableRPCCompression(bool enable) { + enableRPCCompression_ = enable; + return *this; +} + +SessionPool &SessionPool::setConnectTimeoutMs(int connectTimeoutMs) { + connectTimeoutMs_ = connectTimeoutMs; + return *this; +} + +SessionPool &SessionPool::setWaitToGetSessionTimeoutMs(int64_t timeoutMs) { + waitTimeoutMs_ = timeoutMs; + return *this; +} + +std::shared_ptr SessionPool::constructNewSession() { + AbstractSessionBuilder builder; + builder.host = host_; + builder.rpcPort = rpcPort_; + builder.username = username_; + builder.password = password_; + builder.zoneId = zoneId_; + builder.fetchSize = fetchSize_; + builder.sqlDialect = sqlDialect_; + builder.database = database_; + builder.enableAutoFetch = enableAutoFetch_; + builder.enableRedirections = enableRedirection_; + builder.enableRPCCompression = enableRPCCompression_; + + auto session = std::make_shared(&builder); + session->open(enableRPCCompression_, connectTimeoutMs_); + return session; +} + +std::shared_ptr SessionPool::acquire(int64_t timeoutMs) { + const int64_t effectiveTimeout = timeoutMs > 0 ? timeoutMs : waitTimeoutMs_; + std::unique_lock lock(mutex_); + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(effectiveTimeout); + + while (true) { + if (closed_) { + throw IoTDBException("SessionPool is closed."); + } + if (!idleQueue_.empty()) { + auto session = idleQueue_.front(); + idleQueue_.pop_front(); + return session; + } + if (size_ < maxSize_) { + // Reserve a slot, then build the connection outside the lock so other + // borrowers are not blocked by network/handshake latency. + ++size_; + lock.unlock(); + std::shared_ptr session; + try { + session = constructNewSession(); + } catch (...) { + lock.lock(); + --size_; + cv_.notify_one(); + throw; + } + lock.lock(); + if (closed_) { + // The pool was closed while this session was being built; do not hand + // it out. Release its slot and let it be torn down outside the lock. + --size_; + lock.unlock(); + throw IoTDBException("SessionPool is closed."); + } + return session; + } + + // Pool exhausted: wait for a Session to be returned. + if (effectiveTimeout <= 0) { + cv_.wait(lock); + } else { + if (cv_.wait_until(lock, deadline) == std::cv_status::timeout && + idleQueue_.empty() && size_ >= maxSize_ && !closed_) { + throw IoTDBException( + "Wait to get session timeout in SessionPool, maxSize=" + + std::to_string(maxSize_) + + ", waitTimeoutMs=" + std::to_string(effectiveTimeout) + "."); + } + } + } +} + +void SessionPool::putBack(const std::shared_ptr &session, + bool broken) { + std::lock_guard lock(mutex_); + if (broken || closed_) { + // Drop the Session and free its slot so a healthy replacement can be + // created on demand. The caller (PooledSession::reset) still holds the last + // reference and tears the connection down after we return, i.e. outside + // this lock. + --size_; + } else { + idleQueue_.push_back(session); + } + cv_.notify_one(); +} + +PooledSession SessionPool::getSession() { return getSession(waitTimeoutMs_); } + +PooledSession SessionPool::getSession(int64_t timeoutMs) { + return PooledSession(this, acquire(timeoutMs)); +} + +void SessionPool::insertTablet(Tablet &tablet, bool sorted) { + execute([&](Session &s) { s.insertTablet(tablet, sorted); }); +} + +void SessionPool::insertAlignedTablet(Tablet &tablet, bool sorted) { + execute([&](Session &s) { s.insertAlignedTablet(tablet, sorted); }); +} + +void SessionPool::insertTablets( + std::unordered_map &tablets, bool sorted) { + execute([&](Session &s) { s.insertTablets(tablets, sorted); }); +} + +void SessionPool::insertRecord(const std::string &deviceId, int64_t time, + const std::vector &measurements, + const std::vector &values) { + execute([&](Session &s) { + s.insertRecord(deviceId, time, measurements, values); + }); +} + +void SessionPool::insertRecords( + const std::vector &deviceIds, + const std::vector ×, + const std::vector> &measurementsList, + const std::vector> &valuesList) { + execute([&](Session &s) { + s.insertRecords(deviceIds, times, measurementsList, valuesList); + }); +} + +void SessionPool::executeNonQueryStatement(const std::string &sql) { + execute([&](Session &s) { s.executeNonQueryStatement(sql); }); +} + +PooledSessionDataSet +SessionPool::executeQueryStatement(const std::string &sql) { + PooledSession lease = getSession(); + try { + auto dataSet = lease->executeQueryStatement(sql); + return PooledSessionDataSet(std::move(lease), std::move(dataSet)); + } catch (const IoTDBConnectionException &) { + lease.markBroken(); + throw; + } +} + +PooledSessionDataSet SessionPool::executeQueryStatement(const std::string &sql, + int64_t timeoutInMs) { + PooledSession lease = getSession(); + try { + auto dataSet = lease->executeQueryStatement(sql, timeoutInMs); + return PooledSessionDataSet(std::move(lease), std::move(dataSet)); + } catch (const IoTDBConnectionException &) { + lease.markBroken(); + throw; + } +} + +void SessionPool::close() { + std::deque> toClose; + { + std::lock_guard lock(mutex_); + if (closed_) { + return; + } + closed_ = true; + toClose.swap(idleQueue_); + size_ -= toClose.size(); + } + cv_.notify_all(); + // Sessions destructed here (outside the lock) close their connections. + toClose.clear(); +} + +size_t SessionPool::activeCount() { + std::lock_guard lock(mutex_); + return size_ - idleQueue_.size(); +} diff --git a/iotdb-client/client-cpp/src/session/TsBlock.cpp b/iotdb-client/client-cpp/src/session/TsBlock.cpp new file mode 100644 index 0000000000000..b4c3cc2da82c6 --- /dev/null +++ b/iotdb-client/client-cpp/src/session/TsBlock.cpp @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "TsBlock.h" +#include +#include +#include + +std::shared_ptr +TsBlock::create(int32_t positionCount, std::shared_ptr timeColumn, + std::vector> valueColumns) { + if (valueColumns.empty()) { + throw std::invalid_argument("valueColumns cannot be empty"); + } + return std::shared_ptr(new TsBlock( + positionCount, std::move(timeColumn), std::move(valueColumns))); +} + +std::shared_ptr TsBlock::deserialize(const std::string &data) { + MyStringBuffer buffer(data); + + // Read value column count + int32_t valueColumnCount = buffer.getInt(); + if (valueColumnCount < 0) { + throw IoTDBException("TsBlock::deserialize: negative valueColumnCount"); + } + const int64_t minHeaderBytes = + 9LL + 2LL * static_cast(valueColumnCount); + if (minHeaderBytes > static_cast(data.size())) { + throw IoTDBException("TsBlock::deserialize: truncated header"); + } + + // Read value column data types + std::vector valueColumnDataTypes(valueColumnCount); + for (int32_t i = 0; i < valueColumnCount; i++) { + valueColumnDataTypes[i] = + static_cast(buffer.getChar()); + } + + // Read position count + int32_t positionCount = buffer.getInt(); + if (positionCount < 0) { + throw IoTDBException("TsBlock::deserialize: negative positionCount"); + } + + // Read column encodings + std::vector columnEncodings(valueColumnCount + 1); + for (int32_t i = 0; i < valueColumnCount + 1; i++) { + columnEncodings[i] = static_cast(buffer.getChar()); + } + + // Read time column + auto timeColumnDecoder = getColumnDecoder(columnEncodings[0]); + auto timeColumn = + timeColumnDecoder->readColumn(buffer, TSDataType::INT64, positionCount); + + // Read value columns + std::vector> valueColumns(valueColumnCount); + for (int32_t i = 0; i < valueColumnCount; i++) { + auto valueColumnDecoder = getColumnDecoder(columnEncodings[i + 1]); + valueColumns[i] = valueColumnDecoder->readColumn( + buffer, valueColumnDataTypes[i], positionCount); + } + + return create(positionCount, std::move(timeColumn), std::move(valueColumns)); +} + +TsBlock::TsBlock(int32_t positionCount, std::shared_ptr timeColumn, + std::vector> valueColumns) + : positionCount_(positionCount), timeColumn_(std::move(timeColumn)), + valueColumns_(std::move(valueColumns)) {} + +int32_t TsBlock::getPositionCount() const { return positionCount_; } + +int64_t TsBlock::getStartTime() const { return timeColumn_->getLong(0); } + +int64_t TsBlock::getEndTime() const { + return timeColumn_->getLong(positionCount_ - 1); +} + +bool TsBlock::isEmpty() const { return positionCount_ == 0; } + +int64_t TsBlock::getTimeByIndex(int32_t index) const { + return timeColumn_->getLong(index); +} + +int32_t TsBlock::getValueColumnCount() const { + return static_cast(valueColumns_.size()); +} + +const std::shared_ptr TsBlock::getTimeColumn() const { + return timeColumn_; +} + +const std::vector> &TsBlock::getValueColumns() const { + return valueColumns_; +} + +const std::shared_ptr TsBlock::getColumn(int32_t columnIndex) const { + return valueColumns_[columnIndex]; +} diff --git a/iotdb-client/client-cpp/src/test/CMakeLists.txt b/iotdb-client/client-cpp/src/test/CMakeLists.txt deleted file mode 100644 index d38c975de201c..0000000000000 --- a/iotdb-client/client-cpp/src/test/CMakeLists.txt +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -CMAKE_MINIMUM_REQUIRED(VERSION 3.7) -INCLUDE( CTest ) -SET(CMAKE_CXX_STANDARD 11) -SET(CMAKE_CXX_STANDARD_REQUIRED ON) -SET(TARGET_NAME session_tests) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -g -O2") -ENABLE_TESTING() - -# Add Boost include path for MacOS -INCLUDE_DIRECTORIES(/usr/local/include) -# Add Session related include files -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/../main/generated-sources-cpp) -# Add Thrift include directory -INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/../../thrift/include) - -# Add Boost library headers for MaxOS -FIND_PACKAGE(Boost REQUIRED) -IF (DEFINED BOOST_INCLUDEDIR) - include_directories("${Boost_INCLUDE_DIR}") -ENDIF() - -# Link directories are different for Windows and Linux/Mac -IF(MSVC) - LINK_DIRECTORIES(${CMAKE_SOURCE_DIR}/../main/Release) - SET(THRIFT_STATIC_LIB "${CMAKE_SOURCE_DIR}/../../thrift/lib/Release/thriftmd.lib") -ELSE() - LINK_DIRECTORIES(${CMAKE_SOURCE_DIR}/../main) - SET(THRIFT_STATIC_LIB "${CMAKE_SOURCE_DIR}/../../thrift/lib/libthrift.a") -ENDIF() - -ADD_EXECUTABLE(${TARGET_NAME} main.cpp cpp/sessionIT.cpp) - -# Link with shared library iotdb_session and pthread -IF(MSVC) - TARGET_LINK_LIBRARIES(${TARGET_NAME} iotdb_session ${THRIFT_STATIC_LIB}) -ELSE() - TARGET_LINK_LIBRARIES(${TARGET_NAME} iotdb_session pthread) -ENDIF() -TARGET_INCLUDE_DIRECTORIES(${TARGET_NAME} PUBLIC ./catch2/) - -# Add 'sessionIT' to the project to be run by ctest -IF(MSVC) - ADD_TEST(NAME sessionIT CONFIGURATIONS Release COMMAND ${TARGET_NAME}) -ELSE() - ADD_TEST(NAME sessionIT COMMAND ${TARGET_NAME}) -ENDIF() diff --git a/iotdb-client/client-cpp/src/test/cpp/sessionIT.cpp b/iotdb-client/client-cpp/src/test/cpp/sessionIT.cpp deleted file mode 100644 index 09154b868e47c..0000000000000 --- a/iotdb-client/client-cpp/src/test/cpp/sessionIT.cpp +++ /dev/null @@ -1,730 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "catch.hpp" -#include "Session.h" - -using namespace std; - -extern std::shared_ptr session; - -static vector testTimeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3"}; - -void prepareTimeseries() { - for (const string ×eries: testTimeseries) { - if (session->checkTimeseriesExists(timeseries)) { - session->deleteTimeseries(timeseries); - } - session->createTimeseries(timeseries, TSDataType::INT64, TSEncoding::RLE, CompressionType::SNAPPY); - } -} - -static int global_test_id = 0; -class CaseReporter -{ -public: - CaseReporter(const char *caseNameArg) : caseName(caseNameArg) - { - test_id = global_test_id++; - std::cout << "Test " << test_id << ": " << caseName << std::endl; - } - ~CaseReporter() - { - std::cout << "Test " << test_id << ": " << caseName << " Done"<< std::endl << std::endl; - } -private: - const char *caseName; - int test_id; -}; - -TEST_CASE("Create timeseries success", "[createTimeseries]") { - CaseReporter cr("createTimeseries"); - if (!session->checkTimeseriesExists("root.test.d1.s1")) { - session->createTimeseries("root.test.d1.s1", TSDataType::INT64, TSEncoding::RLE, CompressionType::SNAPPY); - } - REQUIRE(session->checkTimeseriesExists("root.test.d1.s1") == true); - session->deleteTimeseries("root.test.d1.s1"); -} - -TEST_CASE("Delete timeseries success", "[deleteTimeseries]") { - CaseReporter cr("deleteTimeseries"); - if (!session->checkTimeseriesExists("root.test.d1.s1")) { - session->createTimeseries("root.test.d1.s1", TSDataType::INT64, TSEncoding::RLE, CompressionType::SNAPPY); - } - REQUIRE(session->checkTimeseriesExists("root.test.d1.s1") == true); - session->deleteTimeseries("root.test.d1.s1"); - REQUIRE(session->checkTimeseriesExists("root.test.d1.s1") == false); -} - -TEST_CASE("Test insertRecord by string", "[testInsertRecord]") { - CaseReporter cr("testInsertRecord"); - prepareTimeseries(); - string deviceId = "root.test.d1"; - vector measurements = {"s1", "s2", "s3"}; - - for (long time = 0; time < 100; time++) { - vector values = {"1", "2", "3"}; - session->insertRecord(deviceId, time, measurements, values); - } - - session->executeNonQueryStatement("insert into root.test.d1(timestamp,s1, s2, s3) values(100, 1,2,3)"); - - - unique_ptr sessionDataSet = session->executeQueryStatement("select * from root.ln.设备"); - - sessionDataSet->setFetchSize(1024); - while (sessionDataSet->hasNext()) { - auto record = sessionDataSet->next(); - std::cout << record->toString() << std::endl; - } - - // unique_ptr sessionDataSet = session->executeQueryStatement("select s1,s2,s3 from root.test.d1"); - // sessionDataSet->setFetchSize(1024); - // int count = 0; - // while (sessionDataSet->hasNext()) { - // long index = 1; - // count++; - // for (const Field &f: sessionDataSet->next()->fields) { - // REQUIRE(f.longV == index); - // index++; - // } - // } - // REQUIRE(count == 101); -} - -TEST_CASE("Test insertRecords ", "[testInsertRecords]") { - CaseReporter cr("testInsertRecords"); - prepareTimeseries(); - string deviceId = "root.test.d1"; - vector measurements = {"s1", "s2", "s3"}; - vector deviceIds; - vector> measurementsList; - vector> valuesList; - vector timestamps; - - int64_t COUNT = 500; - for (int64_t time = 1; time <= COUNT; time++) { - vector values = {"1", "2", "3"}; - - deviceIds.push_back(deviceId); - measurementsList.push_back(measurements); - valuesList.push_back(values); - timestamps.push_back(time); - if (time != 0 && time % 100 == 0) { - session->insertRecords(deviceIds, timestamps, measurementsList, valuesList); - deviceIds.clear(); - measurementsList.clear(); - valuesList.clear(); - timestamps.clear(); - } - } - - if (timestamps.size() > 0) { - session->insertRecords(deviceIds, timestamps, measurementsList, valuesList); - } - - unique_ptr sessionDataSet = session->executeQueryStatement("select s1,s2,s3 from root.test.d1"); - sessionDataSet->setFetchSize(1024); - int count = 0; - while (sessionDataSet->hasNext()) { - long index = 1; - count++; - for (const Field &f: sessionDataSet->next()->fields) { - REQUIRE(f.longV == index); - index++; - } - } - REQUIRE(count == COUNT); -} - -TEST_CASE("Test insertRecord with types ", "[testTypedInsertRecord]") { - CaseReporter cr("testTypedInsertRecord"); - vector timeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3"}; - vector types = {TSDataType::INT32, TSDataType::DOUBLE, TSDataType::INT64}; - - for (size_t i = 0; i < timeseries.size(); i++) { - if (session->checkTimeseriesExists(timeseries[i])) { - session->deleteTimeseries(timeseries[i]); - } - session->createTimeseries(timeseries[i], types[i], TSEncoding::RLE, CompressionType::SNAPPY); - } - string deviceId = "root.test.d1"; - vector measurements = {"s1", "s2", "s3"}; - vector value1(100, 1); - vector value2(100, 2.2); - vector value3(100, 3); - - for (long time = 0; time < 100; time++) { - vector values = {(char *) (&value1[time]), (char *) (&value2[time]), (char *) (&value3[time])}; - session->insertRecord(deviceId, time, measurements, types, values); - } - - unique_ptr sessionDataSet = session->executeQueryStatement("select s1,s2,s3 from root.test.d1"); - sessionDataSet->setFetchSize(1024); - long count = 0; - while (sessionDataSet->hasNext()) { - sessionDataSet->next(); - count++; - } - REQUIRE(count == 100); -} - - -TEST_CASE("Test insertRecord with new datatypes ", "[testTypedInsertRecordNewDatatype]") { - CaseReporter cr("testTypedInsertRecordNewDatatype"); - vector timeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3", "root.test.d1.s4"}; - std::vector types = {TSDataType::TIMESTAMP, - TSDataType::DATE, TSDataType::BLOB, TSDataType::STRING}; - - for (size_t i = 0; i < timeseries.size(); i++) { - if (session->checkTimeseriesExists(timeseries[i])) { - session->deleteTimeseries(timeseries[i]); - } - session->createTimeseries(timeseries[i], types[i], TSEncoding::PLAIN, CompressionType::SNAPPY); - } - string deviceId = "root.test.d1"; - vector measurements = {"s1", "s2", "s3", "s4"}; - int64_t value1 = 20250507; - boost::gregorian::date value2 = boost::gregorian::date(2025, 5, 7); - string value3 = "20250507"; - string value4 = "20250507"; - - for (long time = 0; time < 100; time++) { - vector values = {(char *) (&value1), (char *) (&value2), - const_cast(value3.c_str()), const_cast(value4.c_str())}; - session->insertRecord(deviceId, time, measurements, types, values); - } - - unique_ptr sessionDataSet = session->executeQueryStatement("select s1,s2,s3,s4 from root.test.d1"); - sessionDataSet->setFetchSize(1024); - long count = 0; - while (sessionDataSet->hasNext()) { - auto record = sessionDataSet->next(); - REQUIRE(record->fields.size() == 4); - for (int i = 0; i < 4; i++) { - REQUIRE(types[i] == record->fields[i].dataType); - } - REQUIRE(record->fields[0].longV == value1); - REQUIRE(record->fields[1].dateV == value2); - REQUIRE(record->fields[2].stringV == value3); - REQUIRE(record->fields[3].stringV == value4); - count++; - } - REQUIRE(count == 100); -} - -TEST_CASE("Test insertRecords with types ", "[testTypedInsertRecords]") { - CaseReporter cr("testTypedInsertRecords"); - vector timeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3"}; - vector types = {TSDataType::INT32, TSDataType::DOUBLE, TSDataType::INT64}; - - for (size_t i = 0; i < timeseries.size(); i++) { - if (session->checkTimeseriesExists(timeseries[i])) { - session->deleteTimeseries(timeseries[i]); - } - session->createTimeseries(timeseries[i], types[i], TSEncoding::RLE, CompressionType::SNAPPY); - } - string deviceId = "root.test.d1"; - vector measurements = {"s1", "s2", "s3"}; - vector deviceIds; - vector> measurementsList; - vector> typesList; - vector> valuesList; - vector timestamps; - vector value1(100, 1); - vector value2(100, 2.2); - vector value3(100, 3); - - for (int64_t time = 0; time < 100; time++) { - vector values = {(char *) (&value1[time]), (char *) (&value2[time]), (char *) (&value3[time])}; - deviceIds.push_back(deviceId); - measurementsList.push_back(measurements); - typesList.push_back(types); - valuesList.push_back(values); - timestamps.push_back(time); - } - - session->insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList); - - unique_ptr sessionDataSet = session->executeQueryStatement("select s1,s2,s3 from root.test.d1"); - sessionDataSet->setFetchSize(1024); - int count = 0; - while (sessionDataSet->hasNext()) { - sessionDataSet->next(); - count++; - } - REQUIRE(count == 100); -} - -TEST_CASE("Test insertRecordsOfOneDevice", "[testInsertRecordsOfOneDevice]") { - CaseReporter cr("testInsertRecordsOfOneDevice"); - vector timeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3"}; - vector types = {TSDataType::INT32, TSDataType::DOUBLE, TSDataType::INT64}; - - for (size_t i = 0; i < timeseries.size(); i++) { - if (session->checkTimeseriesExists(timeseries[i])) { - session->deleteTimeseries(timeseries[i]); - } - session->createTimeseries(timeseries[i], types[i], TSEncoding::RLE, CompressionType::SNAPPY); - } - string deviceId = "root.test.d1"; - vector measurements = {"s1", "s2", "s3"}; - vector> measurementsList; - vector> typesList; - vector> valuesList; - vector timestamps; - vector value1(100, 1); - vector value2(100, 2.2); - vector value3(100, 3); - - for (int64_t time = 0; time < 100; time++) { - vector values = {(char *) (&value1[time]), (char *) (&value2[time]), (char *) (&value3[time])}; - measurementsList.push_back(measurements); - typesList.push_back(types); - valuesList.push_back(values); - timestamps.push_back(time); - } - - session->insertRecordsOfOneDevice(deviceId, timestamps, measurementsList, typesList, valuesList); - - unique_ptr sessionDataSet = session->executeQueryStatement("select * from root.test.d1"); - sessionDataSet->setFetchSize(1024); - int count = 0; - while (sessionDataSet->hasNext()) { - sessionDataSet->next(); - count++; - } - REQUIRE(count == 100); -} - -TEST_CASE("Test insertTablet ", "[testInsertTablet]") { - CaseReporter cr("testInsertTablet"); - prepareTimeseries(); - string deviceId = "root.test.d1"; - vector> schemaList; - schemaList.emplace_back("s1", TSDataType::INT64); - schemaList.emplace_back("s2", TSDataType::INT64); - schemaList.emplace_back("s3", TSDataType::INT64); - - Tablet tablet(deviceId, schemaList, 100); - for (int64_t time = 0; time < 100; time++) { - int row = tablet.rowSize++; - tablet.timestamps[row] = time; - for (int64_t i = 0; i < 3; i++) { - tablet.addValue(i, row, i); - } - if (tablet.rowSize == tablet.maxRowNumber) { - session->insertTablet(tablet); - tablet.reset(); - } - } - - if (tablet.rowSize != 0) { - session->insertTablet(tablet); - tablet.reset(); - } - unique_ptr sessionDataSet = session->executeQueryStatement("select s1,s2,s3 from root.test.d1"); - sessionDataSet->setFetchSize(1024); - int count = 0; - while (sessionDataSet->hasNext()) { - long index = 0; - count++; - for (const Field& f: sessionDataSet->next()->fields) { - REQUIRE(f.longV == index); - index++; - } - } - REQUIRE(count == 100); -} - -TEST_CASE("Test insertTablets ", "[testInsertTablets]") { - CaseReporter cr("testInsertTablets"); - vector testTimeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3", - "root.test.d2.s1", "root.test.d2.s2", "root.test.d2.s3"}; - for (const string ×eries: testTimeseries) { - if (session->checkTimeseriesExists(timeseries)) { - session->deleteTimeseries(timeseries); - } - session->createTimeseries(timeseries, TSDataType::INT64, TSEncoding::RLE, CompressionType::SNAPPY); - } - vector> schemaList; - schemaList.emplace_back("s1", TSDataType::INT64); - schemaList.emplace_back("s2", TSDataType::INT64); - schemaList.emplace_back("s3", TSDataType::INT64); - - int maxRowNumber = 100; - vector deviceIds = {"root.test.d1", "root.test.d2"}; - vector tablets; - for (const auto& deviceId: deviceIds) { - tablets.emplace_back(deviceId, schemaList, maxRowNumber); - } - for (auto& tablet : tablets) { - for (int64_t time = 0; time < maxRowNumber; time++) { - int row = tablet.rowSize++; - tablet.timestamps[row] = time; - for (int64_t i = 0; i < 3; i++) { - tablet.addValue(i, row, i); - } - } - } - unordered_map tabletsMap; - for (auto& tablet : tablets) { - tabletsMap[tablet.deviceId] = &tablet; - } - session->insertTablets(tabletsMap); - - unique_ptr sessionDataSet = session->executeQueryStatement("select s1,s2,s3 from root.test.d2"); - sessionDataSet->setFetchSize(1024); - int count = 0; - while (sessionDataSet->hasNext()) { - long index = 0; - count++; - for (const Field& f: sessionDataSet->next()->fields) { - REQUIRE(f.longV == index); - index++; - } - } - REQUIRE(count == 100); -} - -TEST_CASE("Test insertTablet new datatype", "[testInsertTabletNewDatatype]") { - CaseReporter cr("testInsertTabletNewDatatype"); - string deviceId = "root.test.d2"; - vector> schemaList; - std::vector measurements = {"s1", "s2", "s3", "s4"}; - std::vector dataTypes = {TSDataType::TIMESTAMP, - TSDataType::DATE, TSDataType::BLOB, TSDataType::STRING}; - for (int i = 0; i < 4; i++) { - schemaList.emplace_back(measurements[i], dataTypes[i]); - } - - for (int i = 0; i < 4; i++) { - auto timeseries = deviceId + "." + measurements[i]; - if (session->checkTimeseriesExists(timeseries)) { - session->deleteTimeseries(timeseries); - } - session->createTimeseries(timeseries, dataTypes[i], TSEncoding::PLAIN, CompressionType::UNCOMPRESSED); - } - - int64_t s1Value = 20250507; - boost::gregorian::date s2Value(2025, 5, 7); - std::string s3Value("20250507"); - std::string s4Value("20250507"); - - Tablet tablet(deviceId, schemaList, 100); - for (int64_t time = 0; time < 100; time++) { - int row = tablet.rowSize++; - tablet.timestamps[row] = time; - tablet.addValue(0, row, s1Value); - tablet.addValue(1, row, s2Value); - tablet.addValue(2, row, s3Value); - tablet.addValue(3, row, s4Value); - if (tablet.rowSize == tablet.maxRowNumber) { - session->insertTablet(tablet); - tablet.reset(); - } - } - - if (tablet.rowSize != 0) { - session->insertTablet(tablet); - tablet.reset(); - } - unique_ptr sessionDataSet = session->executeQueryStatement("select s1,s2,s3,s4 from root.test.d2"); - sessionDataSet->setFetchSize(1024); - int count = 0; - while (sessionDataSet->hasNext()) { - auto record = sessionDataSet->next(); - REQUIRE(record->fields.size() == 4); - for (int i = 0; i < 4; i++) { - REQUIRE(dataTypes[i] == record->fields[i].dataType); - } - REQUIRE(record->fields[0].longV == s1Value); - REQUIRE(record->fields[1].dateV == s2Value); - REQUIRE(record->fields[2].stringV == s3Value); - REQUIRE(record->fields[3].stringV == s4Value); - count++; - } - REQUIRE(count == 100); -} - -TEST_CASE("Test Last query ", "[testLastQuery]") { - CaseReporter cr("testLastQuery"); - prepareTimeseries(); - string deviceId = "root.test.d1"; - vector measurements = {"s1", "s2", "s3"}; - - for (long time = 0; time < 100; time++) { - vector values = {"1", "2", "3"}; - session->insertRecord(deviceId, time, measurements, values); - } - - vector measurementValues = {"1", "2", "3"}; - unique_ptr sessionDataSet = session->executeQueryStatement( - "select last s1,s2,s3 from root.test.d1"); - sessionDataSet->setFetchSize(1024); - long index = 0; - while (sessionDataSet->hasNext()) { - vector fields = sessionDataSet->next()->fields; - REQUIRE("1" <= fields[1].stringV); - REQUIRE(fields[1].stringV <= "3"); - index++; - } -} - -TEST_CASE("Test Huge query ", "[testHugeQuery]") { - CaseReporter cr("testHugeQuery"); - prepareTimeseries(); - string deviceId = "root.test.d1"; - vector measurements = {"s1", "s2", "s3"}; - vector types = {TSDataType::INT64, TSDataType::INT64, TSDataType::INT64}; - int64_t value1 = 1, value2 = 2, value3 = 3; - vector values = {(char*)&value1, (char*)&value2, (char*)&value3}; - - long total_count = 500000; - int print_count = 0; - std::cout.width(7); - std::cout << "inserting " << total_count << " rows:" << std::endl; - for (long time = 0; time < total_count; time++) { - session->insertRecord(deviceId, time, measurements, types, values); - if (time != 0 && time % 1000 == 0) { - std::cout << time << "\t" << std::flush; - if (++print_count % 20 == 0) { - std::cout << std::endl; - } - } - } - - unique_ptr sessionDataSet = session->executeQueryStatement("select s1,s2,s3 from root.test.d1"); - sessionDataSet->setFetchSize(1024); - int count = 0; - print_count = 0; - std::cout << "\n\niterating " << total_count << " rows:" << std::endl; - while (sessionDataSet->hasNext()) { - auto rowRecord = sessionDataSet->next(); - REQUIRE(rowRecord->timestamp == count); - REQUIRE(rowRecord->fields[0].longV== 1); - REQUIRE(rowRecord->fields[1].longV == 2); - REQUIRE(rowRecord->fields[2].longV == 3); - count++; - if (count % 1000 == 0) { - std::cout << count << "\t" << std::flush; - if (++print_count % 20 == 0) { - std::cout << std::endl; - } - } - } - - REQUIRE(count == total_count); -} - - -TEST_CASE("Test executeRawDataQuery ", "[executeRawDataQuery]") { - CaseReporter cr("executeRawDataQuery"); - prepareTimeseries(); - - string deviceId = "root.test.d1"; - vector measurements = {"s1", "s2", "s3"}; - vector types = {TSDataType::INT64, TSDataType::INT64, TSDataType::INT64}; - - long total_count = 5000; - vector values; - int64_t valueArray[3]; - for (long time = -total_count; time < total_count; time++) { - valueArray[0] = time; - valueArray[1] = time * 2; - valueArray[2] = time * 3; - values.clear(); - values.push_back((char*)&valueArray[0]); - values.push_back((char*)&valueArray[1]); - values.push_back((char*)&valueArray[2]); - session->insertRecord(deviceId, time, measurements, types, values); - if (time == 100) { //insert 1 big timestamp data for generate un-seq data. - valueArray[0] = 9; - valueArray[2] = 999; - values.clear(); - values.push_back((char*)&valueArray[0]); - values.push_back((char*)&valueArray[2]); - vector measurements2 = {"s1", "s3"}; - vector types2 = {TSDataType::INT64, TSDataType::INT64}; - session->insertRecord(deviceId, 99999, measurements2, types2, values); - } - } - - vector paths; - paths.push_back("root.test.d1.s1"); - paths.push_back("root.test.d1.s2"); - paths.push_back("root.test.d1.s3"); - - //== Test executeRawDataQuery() with negative timestamp - int startTs = -total_count, endTs = total_count; - unique_ptr sessionDataSet = session->executeRawDataQuery(paths, startTs, endTs); - sessionDataSet->setFetchSize(10); - vector columns = sessionDataSet->getColumnNames(); - columns = sessionDataSet->getColumnNames(); - for (const string &column : columns) { - cout << column << " " ; - } - cout << endl; - REQUIRE(columns[0] == "Time"); - REQUIRE(columns[1] == paths[0]); - REQUIRE(columns[2] == paths[1]); - REQUIRE(columns[3] == paths[2]); - - int ts = startTs; - while (sessionDataSet->hasNext()) { - auto rowRecordPtr = sessionDataSet->next(); - //cout << rowRecordPtr->toString(); - - vector fields = rowRecordPtr->fields; - REQUIRE(rowRecordPtr->timestamp == ts); - REQUIRE(fields[0].dataType == TSDataType::INT64); - REQUIRE(fields[0].longV == ts); - REQUIRE(fields[1].dataType == TSDataType::INT64); - REQUIRE(fields[1].longV == ts * 2); - REQUIRE(fields[2].dataType == TSDataType::INT64); - REQUIRE(fields[2].longV == ts *3); - ts++; - } - - - //== Test executeRawDataQuery() with null field - startTs = 99999; - endTs = 99999 + 10; - sessionDataSet = session->executeRawDataQuery(paths, startTs, endTs); - - sessionDataSet->setFetchSize(10); - columns = sessionDataSet->getColumnNames(); - for (const string &column : columns) { - cout << column << " " ; - } - cout << endl; - REQUIRE(columns[0] == "Time"); - REQUIRE(columns[1] == paths[0]); - REQUIRE(columns[2] == paths[1]); - REQUIRE(columns[3] == paths[2]); - ts = startTs; - while (sessionDataSet->hasNext()) { - auto rowRecordPtr = sessionDataSet->next(); - cout << rowRecordPtr->toString(); - - vector fields = rowRecordPtr->fields; - REQUIRE(rowRecordPtr->timestamp == ts); - REQUIRE(fields[0].dataType == TSDataType::INT64); - REQUIRE(fields[0].longV == 9); - REQUIRE(fields[1].dataType == TSDataType::UNKNOWN); - REQUIRE(fields[2].dataType == TSDataType::INT64); - REQUIRE(fields[2].longV == 999); - } - - //== Test executeRawDataQuery() with empty data - sessionDataSet = session->executeRawDataQuery(paths, 100000, 110000); - sessionDataSet->setFetchSize(1); - REQUIRE(sessionDataSet->hasNext() == false); -} - -TEST_CASE("Test executeLastDataQuery ", "[testExecuteLastDataQuery]") { - CaseReporter cr("testExecuteLastDataQuery"); - prepareTimeseries(); - - string deviceId = "root.test.d1"; - vector measurements = {"s1", "s2", "s3"}; - vector types = {TSDataType::INT64, TSDataType::INT64, TSDataType::INT64}; - - long total_count = 5000; - vector values; - int64_t valueArray[3]; - for (long time = -total_count; time < total_count; time++) { - valueArray[0] = time; - valueArray[1] = time * 2; - valueArray[2] = time * 3; - values.clear(); - values.push_back((char*)&valueArray[0]); - values.push_back((char*)&valueArray[1]); - values.push_back((char*)&valueArray[2]); - session->insertRecord(deviceId, time, measurements, types, values); - if (time == 100) { //insert 1 big timestamp data for gen unseq data. - valueArray[0] = 9; - valueArray[2] = 999; - values.clear(); - values.push_back((char*)&valueArray[0]); - values.push_back((char*)&valueArray[2]); - vector measurements2 = {"s1", "s3"}; - vector types2 = {TSDataType::INT64, TSDataType::INT64}; - session->insertRecord(deviceId, 99999, measurements2, types2, values); - } - } - - int64_t tsCheck[3] = {99999, 4999, 99999}; - std::vector valueCheck = {"9", "9998", "999"}; - - vector paths; - paths.push_back("root.test.d1.s1"); - paths.push_back("root.test.d1.s2"); - paths.push_back("root.test.d1.s3"); - - //== Test executeLastDataQuery() without lastTime - unique_ptr sessionDataSet = session->executeLastDataQuery(paths); - sessionDataSet->setFetchSize(1); - - vector columns = sessionDataSet->getColumnNames(); - for (const string &column : columns) { - cout << column << " " ; - } - cout << endl; - - int index = 0; - while (sessionDataSet->hasNext()) { - auto rowRecordPtr = sessionDataSet->next(); - cout << rowRecordPtr->toString(); - - vector fields = rowRecordPtr->fields; - REQUIRE(rowRecordPtr->timestamp == tsCheck[index]); - REQUIRE(fields[0].stringV == paths[index]); - REQUIRE(fields[1].stringV == valueCheck[index]); - REQUIRE(fields[2].stringV == "INT64"); - index++; - } - - //== Test executeLastDataQuery() with negative lastTime - sessionDataSet = session->executeLastDataQuery(paths, -200); - sessionDataSet->setFetchSize(1); - columns = sessionDataSet->getColumnNames(); - for (const string &column : columns) { - cout << column << " " ; - } - cout << endl; - - index = 0; - while (sessionDataSet->hasNext()) { - auto rowRecordPtr = sessionDataSet->next(); - cout << rowRecordPtr->toString(); - - vector fields = rowRecordPtr->fields; - REQUIRE(rowRecordPtr->timestamp == tsCheck[index]); - REQUIRE(fields[0].stringV == paths[index]); - REQUIRE(fields[1].stringV == valueCheck[index]); - REQUIRE(fields[2].stringV == "INT64"); - index++; - } - - //== Test executeLastDataQuery() with the lastTime that is > largest timestamp. - sessionDataSet = session->executeLastDataQuery(paths, 100000); - sessionDataSet->setFetchSize(1024); - REQUIRE(sessionDataSet->hasNext() == false); -} diff --git a/iotdb-client/client-cpp/test/CMakeLists.txt b/iotdb-client/client-cpp/test/CMakeLists.txt new file mode 100644 index 0000000000000..921b9bddd1bd6 --- /dev/null +++ b/iotdb-client/client-cpp/test/CMakeLists.txt @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ============================================================================= +# C++ client integration tests (driven by ctest). +# The parent CMakeLists.txt include()s this folder when BUILD_TESTING=ON; +# it relies on the iotdb_session target (and iotdb_thrift_static) already +# being defined at the parent scope. +# ============================================================================= + +include(CTest) + +set(CATCH2_URL + "https://github.com/catchorg/Catch2/releases/download/v2.13.7/catch.hpp") +if(CATCH2_INCLUDE_DIR) + set(_catch2_include_dir "${CATCH2_INCLUDE_DIR}") +else() + set(_catch2_include_dir "${CMAKE_CURRENT_BINARY_DIR}/catch2") +endif() +set(_catch2_header "${_catch2_include_dir}/catch.hpp") +if(NOT EXISTS "${_catch2_header}") + file(MAKE_DIRECTORY "${_catch2_include_dir}") + message(STATUS "Downloading Catch2 from ${CATCH2_URL}") + file(DOWNLOAD "${CATCH2_URL}" "${_catch2_header}" SHOW_PROGRESS TLS_VERIFY ON) +endif() + +set(_test_targets + session_tests + session_c_tests) + +add_executable(session_tests main.cpp cpp/sessionIT.cpp) +add_executable(session_c_tests main_c.cpp cpp/sessionCIT.cpp) + +foreach(_t IN LISTS _test_targets) + target_include_directories(${_t} PRIVATE + "${_catch2_include_dir}" + "${CMAKE_CURRENT_SOURCE_DIR}/../src/rpc" + "${THRIFT_GEN_CPP_DIR}" + "${THRIFT_INCLUDE_DIR}") + if(BOOST_INCLUDE_DIR) + target_include_directories(${_t} PRIVATE "${BOOST_INCLUDE_DIR}") + endif() + target_link_libraries(${_t} PRIVATE iotdb_session) + if(WITH_SSL) + target_link_libraries(${_t} PRIVATE OpenSSL::SSL OpenSSL::Crypto) + endif() +endforeach() + +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang" AND NOT MSVC) + foreach(_t IN LISTS _test_targets) + target_compile_options(${_t} PRIVATE -fsanitize=address -fno-omit-frame-pointer) + target_link_options(${_t} PRIVATE -fsanitize=address) + endforeach() +endif() + +# Linux: keep iotdb_session in the executable's needed-list even when there +# is no direct symbol reference from main.cpp (the IT helpers pull it in +# transitively); without this, ld may drop the lib at link time. +if(UNIX AND NOT APPLE) + foreach(_t IN LISTS _test_targets) + target_link_options(${_t} PRIVATE -Wl,--no-as-needed) + endforeach() +endif() + +# Register ctest cases (multi-config on MSVC). +if(MSVC) + add_test(NAME sessionIT CONFIGURATIONS Release COMMAND session_tests) + add_test(NAME sessionCIT CONFIGURATIONS Release COMMAND session_c_tests) + foreach(_t IN LISTS _test_targets) + add_custom_command(TARGET ${_t} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ $) + endforeach() +else() + add_test(NAME sessionIT COMMAND session_tests) + add_test(NAME sessionCIT COMMAND session_c_tests) +endif() + +# Run sequentially: parallel ctest overloads the single local IoTDB instance. +set_tests_properties(sessionIT sessionCIT PROPERTIES RUN_SERIAL TRUE) diff --git a/iotdb-client/client-cpp/test/catch2/.gitignore b/iotdb-client/client-cpp/test/catch2/.gitignore new file mode 100644 index 0000000000000..39e8e4e792225 --- /dev/null +++ b/iotdb-client/client-cpp/test/catch2/.gitignore @@ -0,0 +1,4 @@ +# Catch2 single-header (downloaded at build time by Maven or CMake) +catch.hpp +* +!.gitignore diff --git a/iotdb-client/client-cpp/test/cpp/sessionCIT.cpp b/iotdb-client/client-cpp/test/cpp/sessionCIT.cpp new file mode 100644 index 0000000000000..48ce21aeba256 --- /dev/null +++ b/iotdb-client/client-cpp/test/cpp/sessionCIT.cpp @@ -0,0 +1,748 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "catch.hpp" +#include "SessionC.h" +#include +#include +#include +#include +#include + +extern CSession* g_session; + +static int global_test_id = 0; + +class CaseReporter { +public: + CaseReporter(const char* caseNameArg) : caseName(caseNameArg) { + test_id = global_test_id++; + std::cout << "C-API Test " << test_id << ": " << caseName << std::endl; + } + ~CaseReporter() { + std::cout << "C-API Test " << test_id << ": " << caseName << " Done" << std::endl << std::endl; + } + +private: + const char* caseName; + int test_id; +}; + +static const char* testTimeseries[] = {"root.ctest.d1.s1", "root.ctest.d1.s2", "root.ctest.d1.s3"}; +static const int testTimeseriesCount = 3; + +static void dropTimeseriesIfExists(CSession* session, const char* path) { + bool exists = false; + REQUIRE(ts_session_check_timeseries_exists(session, path, &exists) == TS_OK); + if (exists) { + REQUIRE(ts_session_delete_timeseries(session, path) == TS_OK); + } +} + +static void ensureTimeseries(CSession* session, const char* path, TSDataType_C type, + TSEncoding_C encoding, TSCompressionType_C compression) { + dropTimeseriesIfExists(session, path); + REQUIRE(ts_session_create_timeseries(session, path, type, encoding, compression) == TS_OK); +} + +static int queryRowCount(CSession* session, const char* sql, int fetchSize = 1024) { + CSessionDataSet* dataSet = nullptr; + REQUIRE(ts_session_execute_query(session, sql, &dataSet) == TS_OK); + REQUIRE(dataSet != nullptr); + ts_dataset_set_fetch_size(dataSet, fetchSize); + int count = 0; + while (ts_dataset_has_next(dataSet)) { + CRowRecord* record = ts_dataset_next(dataSet); + REQUIRE(record != nullptr); + ++count; + ts_row_record_destroy(record); + } + ts_dataset_destroy(dataSet); + return count; +} + +static void dropDatabaseIfExists(CSession* session, const char* database) { + TsStatus status = ts_session_delete_database(session, database); + (void)status; +} + +static void prepareTimeseries() { + for (int i = 0; i < testTimeseriesCount; i++) { + ensureTimeseries(g_session, testTimeseries[i], TS_TYPE_INT64, TS_ENCODING_RLE, + TS_COMPRESSION_SNAPPY); + } +} + +/* ============================================================ + * Timeseries CRUD + * ============================================================ */ + +TEST_CASE("C API - Create timeseries", "[c_createTimeseries]") { + CaseReporter cr("c_createTimeseries"); + const char* path = "root.ctest.d1.s1"; + ensureTimeseries(g_session, path, TS_TYPE_INT64, TS_ENCODING_RLE, TS_COMPRESSION_SNAPPY); + bool exists = false; + REQUIRE(ts_session_check_timeseries_exists(g_session, path, &exists) == TS_OK); + REQUIRE(exists); + REQUIRE(ts_session_delete_timeseries(g_session, path) == TS_OK); +} + +TEST_CASE("C API - Delete timeseries", "[c_deleteTimeseries]") { + CaseReporter cr("c_deleteTimeseries"); + const char* path = "root.ctest.d1.s1"; + ensureTimeseries(g_session, path, TS_TYPE_INT64, TS_ENCODING_RLE, TS_COMPRESSION_SNAPPY); + REQUIRE(ts_session_delete_timeseries(g_session, path) == TS_OK); + bool exists = true; + REQUIRE(ts_session_check_timeseries_exists(g_session, path, &exists) == TS_OK); + REQUIRE_FALSE(exists); +} + +TEST_CASE("C API - Login failure", "[c_Authentication]") { + CaseReporter cr("c_LoginTest"); + CSession* badSession = ts_session_new("127.0.0.1", 6667, "root", "wrong-password"); + REQUIRE(badSession != nullptr); + TsStatus status = ts_session_open(badSession); + REQUIRE(status != TS_OK); + const char* err = ts_get_last_error(); + REQUIRE((std::string(err).find("801") != std::string::npos || + std::string(err).find("Authentication") != std::string::npos)); + ts_session_destroy(badSession); +} + +/* ============================================================ + * Insert Record (string values) + * ============================================================ */ + +TEST_CASE("C API - Insert record by string", "[c_insertRecordStr]") { + CaseReporter cr("c_insertRecordStr"); + prepareTimeseries(); + const char* deviceId = "root.ctest.d1"; + const char* measurements[] = {"s1", "s2", "s3"}; + + for (int64_t time = 0; time < 100; time++) { + const char* values[] = {"1", "2", "3"}; + TsStatus status = + ts_session_insert_record_str(g_session, deviceId, time, 3, measurements, values); + REQUIRE(status == TS_OK); + } + + CSessionDataSet* dataSet = nullptr; + TsStatus status = + ts_session_execute_query(g_session, "select s1,s2,s3 from root.ctest.d1", &dataSet); + REQUIRE(status == TS_OK); + REQUIRE(dataSet != nullptr); + ts_dataset_set_fetch_size(dataSet, 1024); + int count = 0; + while (ts_dataset_has_next(dataSet)) { + CRowRecord* record = ts_dataset_next(dataSet); + REQUIRE(record != nullptr); + REQUIRE(ts_row_record_get_int64(record, 0) == 1); + REQUIRE(ts_row_record_get_int64(record, 1) == 2); + REQUIRE(ts_row_record_get_int64(record, 2) == 3); + ++count; + ts_row_record_destroy(record); + } + REQUIRE(count == 100); + ts_dataset_destroy(dataSet); +} + +/* ============================================================ + * Insert Record (typed values) + * ============================================================ */ + +TEST_CASE("C API - Insert record with types", "[c_insertRecordTyped]") { + CaseReporter cr("c_insertRecordTyped"); + + const char* timeseries[] = {"root.ctest.d1.s1", "root.ctest.d1.s2", "root.ctest.d1.s3"}; + TSDataType_C types[] = {TS_TYPE_INT32, TS_TYPE_DOUBLE, TS_TYPE_INT64}; + TSEncoding_C encodings[] = {TS_ENCODING_RLE, TS_ENCODING_RLE, TS_ENCODING_RLE}; + TSCompressionType_C compressions[] = {TS_COMPRESSION_SNAPPY, TS_COMPRESSION_SNAPPY, + TS_COMPRESSION_SNAPPY}; + + for (int i = 0; i < 3; i++) { + ensureTimeseries(g_session, timeseries[i], types[i], encodings[i], compressions[i]); + } + + const char* deviceId = "root.ctest.d1"; + const char* measurements[] = {"s1", "s2", "s3"}; + + for (int64_t time = 0; time < 100; time++) { + int32_t v1 = 1; + double v2 = 2.2; + int64_t v3 = 3; + const void* values[] = {&v1, &v2, &v3}; + TsStatus status = + ts_session_insert_record(g_session, deviceId, time, 3, measurements, types, values); + REQUIRE(status == TS_OK); + } + + REQUIRE(queryRowCount(g_session, "select s1,s2,s3 from root.ctest.d1") == 100); +} + +/* ============================================================ + * Insert Records (batch, string values) + * ============================================================ */ + +TEST_CASE("C API - Insert records batch", "[c_insertRecordsBatch]") { + CaseReporter cr("c_insertRecordsBatch"); + prepareTimeseries(); + + const int BATCH = 100; + const char* deviceId = "root.ctest.d1"; + const char* measurements[] = {"s1", "s2", "s3"}; + + const char* deviceIds[BATCH]; + int64_t times[BATCH]; + int measurementCounts[BATCH]; + const char* const* measurementsList[BATCH]; + const char* values[] = {"1", "2", "3"}; + const char* const* valuesList[BATCH]; + + for (int i = 0; i < BATCH; i++) { + deviceIds[i] = deviceId; + times[i] = i; + measurementCounts[i] = 3; + measurementsList[i] = measurements; + valuesList[i] = values; + } + + TsStatus status = ts_session_insert_records_str(g_session, BATCH, deviceIds, times, + measurementCounts, measurementsList, valuesList); + REQUIRE(status == TS_OK); + + REQUIRE(queryRowCount(g_session, "select s1,s2,s3 from root.ctest.d1") == BATCH); +} + +/* ============================================================ + * Insert Tablet + * ============================================================ */ + +TEST_CASE("C API - Insert tablet", "[c_insertTablet]") { + CaseReporter cr("c_insertTablet"); + prepareTimeseries(); + + const char* columnNames[] = {"s1", "s2", "s3"}; + TSDataType_C dataTypes[] = {TS_TYPE_INT64, TS_TYPE_INT64, TS_TYPE_INT64}; + + CTablet* tablet = ts_tablet_new("root.ctest.d1", 3, columnNames, dataTypes, 100); + REQUIRE(tablet != nullptr); + + for (int64_t time = 0; time < 100; time++) { + ts_tablet_add_timestamp(tablet, (int)time, time); + for (int col = 0; col < 3; col++) { + int64_t val = col; + ts_tablet_add_value_int64(tablet, col, (int)time, val); + } + } + ts_tablet_set_row_count(tablet, 100); + + TsStatus status = ts_session_insert_tablet(g_session, tablet, false); + REQUIRE(status == TS_OK); + + CSessionDataSet* dataSet = nullptr; + REQUIRE(ts_session_execute_query(g_session, "select s1,s2,s3 from root.ctest.d1", &dataSet) == + TS_OK); + REQUIRE(dataSet != nullptr); + ts_dataset_set_fetch_size(dataSet, 1024); + int count = 0; + while (ts_dataset_has_next(dataSet)) { + CRowRecord* record = ts_dataset_next(dataSet); + REQUIRE(ts_row_record_get_int64(record, 0) == 0); + REQUIRE(ts_row_record_get_int64(record, 1) == 1); + REQUIRE(ts_row_record_get_int64(record, 2) == 2); + ++count; + ts_row_record_destroy(record); + } + REQUIRE(count == 100); + ts_dataset_destroy(dataSet); + ts_tablet_destroy(tablet); +} + +/* ============================================================ + * Execute SQL directly + * ============================================================ */ + +TEST_CASE("C API - Execute non-query SQL", "[c_executeNonQuery]") { + CaseReporter cr("c_executeNonQuery"); + prepareTimeseries(); + + TsStatus status = ts_session_execute_non_query( + g_session, "insert into root.ctest.d1(timestamp,s1,s2,s3) values(200,10,20,30)"); + REQUIRE(status == TS_OK); + + CSessionDataSet* dataSet = nullptr; + ts_session_execute_query(g_session, "select s1 from root.ctest.d1 where time=200", &dataSet); + REQUIRE(ts_dataset_has_next(dataSet)); + CRowRecord* record = ts_dataset_next(dataSet); + REQUIRE(ts_row_record_get_int64(record, 0) == 10); + ts_row_record_destroy(record); + ts_dataset_destroy(dataSet); +} + +/* ============================================================ + * Raw data query + * ============================================================ */ + +TEST_CASE("C API - Execute raw data query", "[c_executeRawDataQuery]") { + CaseReporter cr("c_executeRawDataQuery"); + prepareTimeseries(); + + const char* deviceId = "root.ctest.d1"; + const char* measurements[] = {"s1", "s2", "s3"}; + + for (int64_t time = 0; time < 50; time++) { + const char* values[] = {"1", "2", "3"}; + ts_session_insert_record_str(g_session, deviceId, time, 3, measurements, values); + } + + const char* paths[] = {"root.ctest.d1.s1", "root.ctest.d1.s2", "root.ctest.d1.s3"}; + CSessionDataSet* dataSet = nullptr; + TsStatus status = ts_session_execute_raw_data_query(g_session, 3, paths, 0, 50, &dataSet); + REQUIRE(status == TS_OK); + + int count = 0; + while (ts_dataset_has_next(dataSet)) { + CRowRecord* record = ts_dataset_next(dataSet); + count++; + ts_row_record_destroy(record); + } + REQUIRE(count == 50); + ts_dataset_destroy(dataSet); +} + +/* ============================================================ + * Data deletion + * ============================================================ */ + +TEST_CASE("C API - Delete data", "[c_deleteData]") { + CaseReporter cr("c_deleteData"); + prepareTimeseries(); + + const char* deviceId = "root.ctest.d1"; + const char* measurements[] = {"s1", "s2", "s3"}; + for (int64_t time = 0; time < 100; time++) { + const char* values[] = {"1", "2", "3"}; + ts_session_insert_record_str(g_session, deviceId, time, 3, measurements, values); + } + + const char* paths[] = {"root.ctest.d1.s1", "root.ctest.d1.s2", "root.ctest.d1.s3"}; + TsStatus status = ts_session_delete_data_batch(g_session, 3, paths, 49); + REQUIRE(status == TS_OK); + + REQUIRE(queryRowCount(g_session, "select s1,s2,s3 from root.ctest.d1") == 50); +} + +/* ============================================================ + * Timezone + * ============================================================ */ + +TEST_CASE("C API - Timezone", "[c_timezone]") { + CaseReporter cr("c_timezone"); + char buf[64] = {0}; + TsStatus status = ts_session_get_timezone(g_session, buf, sizeof(buf)); + REQUIRE(status == TS_OK); + REQUIRE(strlen(buf) > 0); + + status = ts_session_set_timezone(g_session, "Asia/Shanghai"); + REQUIRE(status == TS_OK); + + memset(buf, 0, sizeof(buf)); + ts_session_get_timezone(g_session, buf, sizeof(buf)); + REQUIRE(std::string(buf) == "Asia/Shanghai"); +} + +/* ============================================================ + * Multi-node constructor + * ============================================================ */ + +TEST_CASE("C API - Multi-node session", "[c_multiNode]") { + CaseReporter cr("c_multiNode"); + const char* urls[] = {"127.0.0.1:6667"}; + CSession* localSession = ts_session_new_multi_node(urls, 1, "root", "root"); + REQUIRE(localSession != nullptr); + + TsStatus status = ts_session_open(localSession); + REQUIRE(status == TS_OK); + + const char* path = "root.ctest.d1.s1"; + ensureTimeseries(localSession, path, TS_TYPE_INT64, TS_ENCODING_RLE, TS_COMPRESSION_SNAPPY); + bool exists = false; + REQUIRE(ts_session_check_timeseries_exists(localSession, path, &exists) == TS_OK); + REQUIRE(exists); + REQUIRE(ts_session_delete_timeseries(localSession, path) == TS_OK); + + ts_session_close(localSession); + ts_session_destroy(localSession); +} + +/* ============================================================ + * Dataset column info + * ============================================================ */ + +TEST_CASE("C API - Dataset column info", "[c_datasetColumns]") { + CaseReporter cr("c_datasetColumns"); + prepareTimeseries(); + + const char* deviceId = "root.ctest.d1"; + const char* measurements[] = {"s1", "s2", "s3"}; + const char* values[] = {"1", "2", "3"}; + ts_session_insert_record_str(g_session, deviceId, 0, 3, measurements, values); + + CSessionDataSet* dataSet = nullptr; + ts_session_execute_query(g_session, "select s1,s2,s3 from root.ctest.d1", &dataSet); + REQUIRE(dataSet != nullptr); + + int colCount = ts_dataset_get_column_count(dataSet); + REQUIRE(colCount == 4); // Time + s1 + s2 + s3 + + const char* col0 = ts_dataset_get_column_name(dataSet, 0); + REQUIRE(std::string(col0) == "Time"); + + int n = ts_dataset_get_column_count(dataSet); + for (int i = 0; i < n; i++) { + const char* ct = ts_dataset_get_column_type(dataSet, i); + REQUIRE(ct != nullptr); + REQUIRE(strlen(ct) > 0); + } + + ts_dataset_destroy(dataSet); +} + +/* ============================================================ + * SessionC.h API coverage (tree model) — additional smoke tests + * ============================================================ */ + +TEST_CASE("C API - Session lifecycle variants", "[c_sessionLifecycle]") { + CaseReporter cr("c_sessionLifecycle"); + + CSession* s1 = ts_session_new_with_zone("127.0.0.1", 6667, "root", "root", "Asia/Shanghai", 1024); + REQUIRE(s1 != nullptr); + REQUIRE(ts_session_open(s1) == TS_OK); + ts_session_close(s1); + ts_session_destroy(s1); + + CSession* s2 = ts_session_new("127.0.0.1", 6667, "root", "root"); + REQUIRE(s2 != nullptr); + REQUIRE(ts_session_open_with_compression(s2, true) == TS_OK); + ts_session_close(s2); + ts_session_destroy(s2); +} + +TEST_CASE("C API - Database and extended timeseries APIs", "[c_dbTimeseries]") { + CaseReporter cr("c_dbTimeseries"); + + const char* sg1 = "root.cov_sg_a"; + const char* sg2 = "root.cov_sg_b"; + dropDatabaseIfExists(g_session, sg1); + dropDatabaseIfExists(g_session, sg2); + REQUIRE(ts_session_create_database(g_session, sg1) == TS_OK); + REQUIRE(ts_session_create_database(g_session, sg2) == TS_OK); + const char* dbs[] = {sg1, sg2}; + REQUIRE(ts_session_delete_databases(g_session, dbs, 2) == TS_OK); + + const char* sg3 = "root.cov_sg_c"; + dropDatabaseIfExists(g_session, sg3); + REQUIRE(ts_session_create_database(g_session, sg3) == TS_OK); + REQUIRE(ts_session_delete_database(g_session, sg3) == TS_OK); + + const char* sgEx = "root.cov_sg_ex"; + dropDatabaseIfExists(g_session, sgEx); + REQUIRE(ts_session_create_database(g_session, sgEx) == TS_OK); + + const char* pathEx = "root.cov_sg_ex.d1.s_ex"; + dropTimeseriesIfExists(g_session, pathEx); + REQUIRE(ts_session_create_timeseries_ex(g_session, pathEx, TS_TYPE_INT64, TS_ENCODING_RLE, + TS_COMPRESSION_SNAPPY, 0, nullptr, nullptr, 0, nullptr, + nullptr, 0, nullptr, nullptr, nullptr) == TS_OK); + + const char* pathsM[] = {"root.cov_sg_ex.d1.s_m1", "root.cov_sg_ex.d1.s_m2"}; + TSDataType_C tsM[] = {TS_TYPE_INT64, TS_TYPE_DOUBLE}; + TSEncoding_C encM[] = {TS_ENCODING_RLE, TS_ENCODING_RLE}; + TSCompressionType_C compM[] = {TS_COMPRESSION_SNAPPY, TS_COMPRESSION_SNAPPY}; + for (int i = 0; i < 2; i++) { + dropTimeseriesIfExists(g_session, pathsM[i]); + } + REQUIRE(ts_session_create_multi_timeseries(g_session, 2, pathsM, tsM, encM, compM) == TS_OK); + REQUIRE(ts_session_delete_timeseries_batch(g_session, pathsM, 2) == TS_OK); + REQUIRE(ts_session_delete_timeseries(g_session, pathEx) == TS_OK); + REQUIRE(ts_session_delete_database(g_session, sgEx) == TS_OK); +} + +TEST_CASE("C API - Tablet row count and reset", "[c_tabletReset]") { + CaseReporter cr("c_tabletReset"); + const char* colNames[] = {"s1"}; + TSDataType_C dts[] = {TS_TYPE_INT64}; + CTablet* tablet = ts_tablet_new("root.ctest.d1", 1, colNames, dts, 10); + REQUIRE(tablet != nullptr); + REQUIRE(ts_tablet_get_row_count(tablet) == 0); + REQUIRE(ts_tablet_set_row_count(tablet, 1) == TS_OK); + REQUIRE(ts_tablet_get_row_count(tablet) == 1); + ts_tablet_reset(tablet); + REQUIRE(ts_tablet_get_row_count(tablet) == 0); + ts_tablet_destroy(tablet); +} + +TEST_CASE("C API - Aligned timeseries and aligned writes", "[c_aligned]") { + CaseReporter cr("c_aligned"); + + const char* sg = "root.cov_al"; + dropDatabaseIfExists(g_session, sg); + REQUIRE(ts_session_create_database(g_session, sg) == TS_OK); + + const char* alDev = "root.cov_al.dev"; + const char* meas[] = {"m1", "m2"}; + TSDataType_C adt[] = {TS_TYPE_INT64, TS_TYPE_INT64}; + TSEncoding_C aenc[] = {TS_ENCODING_RLE, TS_ENCODING_RLE}; + TSCompressionType_C acomp[] = {TS_COMPRESSION_SNAPPY, TS_COMPRESSION_SNAPPY}; + REQUIRE(ts_session_create_aligned_timeseries(g_session, alDev, 2, meas, adt, aenc, acomp) == + TS_OK); + + const char* mstr[] = {"m1", "m2"}; + const char* vstr[] = {"1", "2"}; + REQUIRE(ts_session_insert_aligned_record_str(g_session, alDev, 100LL, 2, mstr, vstr) == TS_OK); + + int64_t v1 = 3; + int64_t v2 = 4; + const void* vals[] = {&v1, &v2}; + REQUIRE(ts_session_insert_aligned_record(g_session, alDev, 101LL, 2, mstr, adt, vals) == TS_OK); + + const char* devs1[] = {alDev}; + int64_t times1[] = {102LL}; + int mc1[] = {2}; + const char* const* mlist1[] = {mstr}; + const char* const* vlist1[] = {vstr}; + REQUIRE(ts_session_insert_aligned_records_str(g_session, 1, devs1, times1, mc1, mlist1, vlist1) == + TS_OK); + + const TSDataType_C* trows[] = {adt}; + const void* const* vrows[] = {vals}; + REQUIRE(ts_session_insert_aligned_records(g_session, 1, devs1, times1, mc1, mlist1, trows, + vrows) == TS_OK); + + int64_t tRows[] = {104LL, 105LL}; + int mcRows[] = {2, 2}; + const char* const* mRows[] = {mstr, mstr}; + const TSDataType_C* tRowsList[] = {adt, adt}; + int64_t v1a = 5, v1b = 6; + int64_t v2a = 7, v2b = 8; + const void* row0[] = {&v1a, &v2a}; + const void* row1[] = {&v1b, &v2b}; + const void* const* vRowsList[] = {row0, row1}; + REQUIRE(ts_session_insert_aligned_records_of_one_device(g_session, alDev, 2, tRows, mcRows, mRows, + tRowsList, vRowsList, true) == TS_OK); + + const char* alDev2 = "root.cov_al.dev2"; + REQUIRE(ts_session_create_aligned_timeseries(g_session, alDev2, 2, meas, adt, aenc, acomp) == + TS_OK); + CTablet* tab = ts_tablet_new(alDev, 2, meas, adt, 10); + CTablet* tab2 = ts_tablet_new(alDev2, 2, meas, adt, 5); + REQUIRE(tab != nullptr); + REQUIRE(tab2 != nullptr); + ts_tablet_add_timestamp(tab, 0, 106LL); + ts_tablet_add_value_int64(tab, 0, 0, 9); + ts_tablet_add_value_int64(tab, 1, 0, 10); + ts_tablet_set_row_count(tab, 1); + ts_tablet_add_timestamp(tab2, 0, 107LL); + ts_tablet_add_value_int64(tab2, 0, 0, 11); + ts_tablet_add_value_int64(tab2, 1, 0, 12); + ts_tablet_set_row_count(tab2, 1); + const char* devIds[] = {alDev, alDev2}; + CTablet* tabs[] = {tab, tab2}; + REQUIRE(ts_session_insert_aligned_tablets(g_session, 2, devIds, tabs, false) == TS_OK); + + ts_tablet_reset(tab); + ts_tablet_add_timestamp(tab, 0, 200LL); + ts_tablet_add_value_int64(tab, 0, 0, 13); + ts_tablet_add_value_int64(tab, 1, 0, 14); + ts_tablet_set_row_count(tab, 1); + REQUIRE(ts_session_insert_aligned_tablet(g_session, tab, false) == TS_OK); + + ts_tablet_destroy(tab2); + ts_tablet_destroy(tab); + + REQUIRE(ts_session_delete_database(g_session, sg) == TS_OK); +} + +TEST_CASE("C API - Typed batch inserts and insert_tablets", "[c_batchTablet]") { + CaseReporter cr("c_batchTablet"); + + const char* sg = "root.cov_batch"; + dropDatabaseIfExists(g_session, sg); + REQUIRE(ts_session_create_database(g_session, sg) == TS_OK); + + const char* p1 = "root.cov_batch.da.s1"; + const char* p2 = "root.cov_batch.db.s1"; + ensureTimeseries(g_session, p1, TS_TYPE_INT64, TS_ENCODING_RLE, TS_COMPRESSION_SNAPPY); + ensureTimeseries(g_session, p2, TS_TYPE_INT64, TS_ENCODING_RLE, TS_COMPRESSION_SNAPPY); + + const char* devIds[] = {"root.cov_batch.da", "root.cov_batch.db"}; + int64_t tt[] = {1LL, 2LL}; + int mmc[] = {1, 1}; + const char* mda[] = {"s1"}; + const char* mdb[] = {"s1"}; + const char* const* mlist[] = {mda, mdb}; + int64_t va = 11; + int64_t vb = 22; + const void* vva[] = {&va}; + const void* vvb[] = {&vb}; + const void* const* vlist[] = {vva, vvb}; + TSDataType_C ta[] = {TS_TYPE_INT64}; + TSDataType_C tb[] = {TS_TYPE_INT64}; + const TSDataType_C* tlist[] = {ta, tb}; + REQUIRE(ts_session_insert_records(g_session, 2, devIds, tt, mmc, mlist, tlist, vlist) == TS_OK); + + const char* dc = "root.cov_batch.dc"; + const char* p3 = "root.cov_batch.dc.s1"; + ensureTimeseries(g_session, p3, TS_TYPE_INT64, TS_ENCODING_RLE, TS_COMPRESSION_SNAPPY); + int64_t tdc[] = {3LL, 4LL}; + int mcdc[] = {1, 1}; + const char* const* mdcList[] = {mda, mda}; + const TSDataType_C* tdcList[] = {ta, ta}; + int64_t vc = 30, vd = 40; + const void* rv0[] = {&vc}; + const void* rv1[] = {&vd}; + const void* const* vdcList[] = {rv0, rv1}; + REQUIRE(ts_session_insert_records_of_one_device(g_session, dc, 2, tdc, mcdc, mdcList, tdcList, + vdcList, true) == TS_OK); + + const char* col1[] = {"s1"}; + TSDataType_C dt1[] = {TS_TYPE_INT64}; + CTablet* tb1 = ts_tablet_new("root.cov_batch.ta", 1, col1, dt1, 5); + CTablet* tb2 = ts_tablet_new("root.cov_batch.tb", 1, col1, dt1, 5); + REQUIRE(tb1 != nullptr); + REQUIRE(tb2 != nullptr); + const char* pta = "root.cov_batch.ta.s1"; + const char* ptb = "root.cov_batch.tb.s1"; + ensureTimeseries(g_session, pta, TS_TYPE_INT64, TS_ENCODING_RLE, TS_COMPRESSION_SNAPPY); + ensureTimeseries(g_session, ptb, TS_TYPE_INT64, TS_ENCODING_RLE, TS_COMPRESSION_SNAPPY); + ts_tablet_add_timestamp(tb1, 0, 1LL); + ts_tablet_add_value_int64(tb1, 0, 0, 100); + ts_tablet_set_row_count(tb1, 1); + ts_tablet_add_timestamp(tb2, 0, 2LL); + ts_tablet_add_value_int64(tb2, 0, 0, 200); + ts_tablet_set_row_count(tb2, 1); + const char* tabDevs[] = {"root.cov_batch.ta", "root.cov_batch.tb"}; + CTablet* tbs[] = {tb1, tb2}; + REQUIRE(ts_session_insert_tablets(g_session, 2, tabDevs, tbs, false) == TS_OK); + ts_tablet_destroy(tb2); + ts_tablet_destroy(tb1); + + REQUIRE(ts_session_delete_database(g_session, sg) == TS_OK); +} + +TEST_CASE("C API - Query timeout and last data queries", "[c_queryLast]") { + CaseReporter cr("c_queryLast"); + prepareTimeseries(); + + const char* deviceId = "root.ctest.d1"; + const char* measurements[] = {"s1", "s2", "s3"}; + for (int64_t time = 300; time < 310; time++) { + const char* values[] = {"7", "8", "9"}; + REQUIRE(ts_session_insert_record_str(g_session, deviceId, time, 3, measurements, values) == + TS_OK); + } + + const char* paths[] = {"root.ctest.d1.s1", "root.ctest.d1.s2"}; + CSessionDataSet* ds = nullptr; + REQUIRE(ts_session_execute_query_with_timeout( + g_session, "select s1 from root.ctest.d1 where time>=300", 60000, &ds) == TS_OK); + REQUIRE(ds != nullptr); + ts_dataset_destroy(ds); + + ds = nullptr; + REQUIRE(ts_session_execute_last_data_query(g_session, 2, paths, &ds) == TS_OK); + REQUIRE(ds != nullptr); + ts_dataset_destroy(ds); + + ds = nullptr; + REQUIRE(ts_session_execute_last_data_query_with_time(g_session, 2, paths, 305LL, &ds) == TS_OK); + REQUIRE(ds != nullptr); + ts_dataset_destroy(ds); +} + +TEST_CASE("C API - RowRecord and delete data APIs", "[c_rowDelete]") { + CaseReporter cr("c_rowDelete"); + + const char* sg = "root.cov_types"; + dropDatabaseIfExists(g_session, sg); + REQUIRE(ts_session_create_database(g_session, sg) == TS_OK); + + const char* pb = "root.cov_types.d1.sb"; + const char* pi = "root.cov_types.d1.si"; + const char* pf = "root.cov_types.d1.sf"; + const char* pd = "root.cov_types.d1.sd"; + const char* pt = "root.cov_types.d1.st"; + const char* tpaths[] = {pb, pi, pf, pd, pt}; + for (const char* tp : tpaths) { + dropTimeseriesIfExists(g_session, tp); + } + REQUIRE(ts_session_create_timeseries(g_session, pb, TS_TYPE_BOOLEAN, TS_ENCODING_RLE, + TS_COMPRESSION_SNAPPY) == TS_OK); + REQUIRE(ts_session_create_timeseries(g_session, pi, TS_TYPE_INT32, TS_ENCODING_RLE, + TS_COMPRESSION_SNAPPY) == TS_OK); + REQUIRE(ts_session_create_timeseries(g_session, pf, TS_TYPE_FLOAT, TS_ENCODING_RLE, + TS_COMPRESSION_SNAPPY) == TS_OK); + REQUIRE(ts_session_create_timeseries(g_session, pd, TS_TYPE_DOUBLE, TS_ENCODING_RLE, + TS_COMPRESSION_SNAPPY) == TS_OK); + REQUIRE(ts_session_create_timeseries(g_session, pt, TS_TYPE_TEXT, TS_ENCODING_PLAIN, + TS_COMPRESSION_SNAPPY) == TS_OK); + + const char* dev = "root.cov_types.d1"; + const char* names[] = {"sb", "si", "sf", "sd", "st"}; + TSDataType_C types[] = {TS_TYPE_BOOLEAN, TS_TYPE_INT32, TS_TYPE_FLOAT, TS_TYPE_DOUBLE, + TS_TYPE_TEXT}; + bool bv = true; + int32_t iv = 42; + float fv = 2.5f; + double dv = 3.25; + const char* tv = "hi"; + const void* vals[] = {&bv, &iv, &fv, &dv, tv}; + REQUIRE(ts_session_insert_record(g_session, dev, 500LL, 5, names, types, vals) == TS_OK); + + CSessionDataSet* dataSet = nullptr; + REQUIRE(ts_session_execute_query(g_session, + "select sb,si,sf,sd,st from root.cov_types.d1 where time=500", + &dataSet) == TS_OK); + REQUIRE(dataSet != nullptr); + REQUIRE(ts_dataset_has_next(dataSet)); + CRowRecord* record = ts_dataset_next(dataSet); + REQUIRE(record != nullptr); + REQUIRE(ts_row_record_get_timestamp(record) == 500LL); + REQUIRE(ts_row_record_get_field_count(record) == 5); + REQUIRE_FALSE(ts_row_record_is_null(record, 0)); + REQUIRE(ts_row_record_get_bool(record, 0) == true); + REQUIRE(ts_row_record_get_int32(record, 1) == 42); + REQUIRE(std::fabs(ts_row_record_get_float(record, 2) - 2.5f) < 1e-4f); + REQUIRE(std::fabs(ts_row_record_get_double(record, 3) - 3.25) < 1e-9); + REQUIRE(std::string(ts_row_record_get_string(record, 4)) == "hi"); + REQUIRE(ts_row_record_get_data_type(record, 0) == TS_TYPE_BOOLEAN); + ts_row_record_destroy(record); + ts_dataset_destroy(dataSet); + + REQUIRE(ts_session_delete_data(g_session, pb, 500LL) == TS_OK); + const char* delPaths[] = {pi, pf}; + REQUIRE(ts_session_delete_data_range(g_session, 2, delPaths, 400LL, 600LL) == TS_OK); + + REQUIRE(ts_session_delete_timeseries(g_session, pb) == TS_OK); + REQUIRE(ts_session_delete_timeseries(g_session, pi) == TS_OK); + REQUIRE(ts_session_delete_timeseries(g_session, pf) == TS_OK); + REQUIRE(ts_session_delete_timeseries(g_session, pd) == TS_OK); + REQUIRE(ts_session_delete_timeseries(g_session, pt) == TS_OK); + REQUIRE(ts_session_delete_database(g_session, sg) == TS_OK); +} diff --git a/iotdb-client/client-cpp/test/cpp/sessionIT.cpp b/iotdb-client/client-cpp/test/cpp/sessionIT.cpp new file mode 100644 index 0000000000000..fdea5a8d07d8a --- /dev/null +++ b/iotdb-client/client-cpp/test/cpp/sessionIT.cpp @@ -0,0 +1,1042 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "catch.hpp" +#include "Column.h" +#include "Date.h" +#include "RpcCommon.h" +#include "Session.h" +#include "SessionBuilder.h" +#include "SessionPool.h" +#include "TsBlock.h" +#include "common_types.h" +#include +#include + +using namespace std; + +extern std::shared_ptr session; + +static vector testTimeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3"}; + +void prepareTimeseries() { + for (const string& timeseries : testTimeseries) { + if (session->checkTimeseriesExists(timeseries)) { + session->deleteTimeseries(timeseries); + } + session->createTimeseries(timeseries, TSDataType::INT64, TSEncoding::RLE, + CompressionType::SNAPPY); + } +} + +static int global_test_id = 0; +class CaseReporter { +public: + CaseReporter(const char* caseNameArg) : caseName(caseNameArg) { + test_id = global_test_id++; + std::cout << "Test " << test_id << ": " << caseName << std::endl; + } + ~CaseReporter() { + std::cout << "Test " << test_id << ": " << caseName << " Done" << std::endl << std::endl; + } + +private: + const char* caseName; + int test_id; +}; + +TEST_CASE("Create timeseries success", "[createTimeseries]") { + CaseReporter cr("createTimeseries"); + if (!session->checkTimeseriesExists("root.test.d1.s1")) { + session->createTimeseries("root.test.d1.s1", TSDataType::INT64, TSEncoding::RLE, + CompressionType::SNAPPY); + } + REQUIRE(session->checkTimeseriesExists("root.test.d1.s1") == true); + session->deleteTimeseries("root.test.d1.s1"); +} + +TEST_CASE("Login Test - Authentication failed with error code 801", "[Authentication]") { + CaseReporter cr("Login Test"); + + try { + Session session("127.0.0.1", 6667, "root", "wrong-password"); + session.open(false); + FAIL("Expected authentication exception"); // Test fails if no exception + } catch (const std::exception& e) { + // Verify exception contains error code 801 + REQUIRE(std::string(e.what()).find("801") != std::string::npos); + } +} + +TEST_CASE("Test Session constructor with nodeUrls", "[SessionInitAndOperate]") { + CaseReporter cr("SessionInitWithNodeUrls"); + + std::vector nodeUrls = {"127.0.0.1:6667"}; + std::shared_ptr localSession = std::make_shared(nodeUrls, "root", "root"); + localSession->open(); + if (!localSession->checkTimeseriesExists("root.test.d1.s1")) { + localSession->createTimeseries("root.test.d1.s1", TSDataType::INT64, TSEncoding::RLE, + CompressionType::SNAPPY); + } + REQUIRE(localSession->checkTimeseriesExists("root.test.d1.s1") == true); + localSession->deleteTimeseries("root.test.d1.s1"); + localSession->close(); +} + +TEST_CASE("Test Session builder with nodeUrls", "[SessionBuilderInit]") { + CaseReporter cr("SessionInitWithNodeUrls"); + + std::vector nodeUrls = {"127.0.0.1:6667"}; + auto builder = std::unique_ptr(new SessionBuilder()); + std::shared_ptr session = std::shared_ptr( + builder->username("root")->password("root")->nodeUrls(nodeUrls)->build()); + session->open(); + if (!session->checkTimeseriesExists("root.test.d1.s1")) { + session->createTimeseries("root.test.d1.s1", TSDataType::INT64, TSEncoding::RLE, + CompressionType::SNAPPY); + } + REQUIRE(session->checkTimeseriesExists("root.test.d1.s1") == true); + session->deleteTimeseries("root.test.d1.s1"); + session->close(); +} + +TEST_CASE("Delete timeseries success", "[deleteTimeseries]") { + CaseReporter cr("deleteTimeseries"); + if (!session->checkTimeseriesExists("root.test.d1.s1")) { + session->createTimeseries("root.test.d1.s1", TSDataType::INT64, TSEncoding::RLE, + CompressionType::SNAPPY); + } + REQUIRE(session->checkTimeseriesExists("root.test.d1.s1") == true); + session->deleteTimeseries("root.test.d1.s1"); + REQUIRE(session->checkTimeseriesExists("root.test.d1.s1") == false); +} + +TEST_CASE("Test insertRecord by string", "[testInsertRecord]") { + CaseReporter cr("testInsertRecord"); + prepareTimeseries(); + string deviceId = "root.test.d1"; + vector measurements = {"s1", "s2", "s3"}; + + for (long time = 0; time < 100; time++) { + vector values = {"1", "2", "3"}; + session->insertRecord(deviceId, time, measurements, values); + } + + session->executeNonQueryStatement( + "insert into root.test.d1(timestamp,s1, s2, s3) values(100, 1,2,3)"); + + unique_ptr sessionDataSet = + session->executeQueryStatement("select s1,s2,s3 from root.test.d1"); + sessionDataSet->setFetchSize(1024); + int count = 0; + while (sessionDataSet->hasNext()) { + long index = 1; + count++; + auto fields = sessionDataSet->next()->fields; + for (const Field& f : fields) { + REQUIRE(f.longV.value() == index); + index++; + } + } + REQUIRE(count == 101); +} + +TEST_CASE("Test insertRecords ", "[testInsertRecords]") { + CaseReporter cr("testInsertRecords"); + prepareTimeseries(); + string deviceId = "root.test.d1"; + vector measurements = {"s1", "s2", "s3"}; + vector deviceIds; + vector> measurementsList; + vector> valuesList; + vector timestamps; + + int64_t COUNT = 500; + for (int64_t time = 1; time <= COUNT; time++) { + vector values = {"1", "2", "3"}; + + deviceIds.push_back(deviceId); + measurementsList.push_back(measurements); + valuesList.push_back(values); + timestamps.push_back(time); + if (time != 0 && time % 100 == 0) { + session->insertRecords(deviceIds, timestamps, measurementsList, valuesList); + deviceIds.clear(); + measurementsList.clear(); + valuesList.clear(); + timestamps.clear(); + } + } + + if (timestamps.size() > 0) { + session->insertRecords(deviceIds, timestamps, measurementsList, valuesList); + } + + unique_ptr sessionDataSet = + session->executeQueryStatement("select s1,s2,s3 from root.test.d1"); + sessionDataSet->setFetchSize(1024); + int count = 0; + while (sessionDataSet->hasNext()) { + long index = 1; + count++; + auto fields = sessionDataSet->next()->fields; + for (const Field& f : fields) { + REQUIRE(f.longV.value() == index); + index++; + } + } + REQUIRE(count == COUNT); +} + +TEST_CASE("Test insertRecord with types ", "[testTypedInsertRecord]") { + CaseReporter cr("testTypedInsertRecord"); + vector timeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3"}; + vector types = {TSDataType::INT32, TSDataType::DOUBLE, TSDataType::INT64}; + + for (size_t i = 0; i < timeseries.size(); i++) { + if (session->checkTimeseriesExists(timeseries[i])) { + session->deleteTimeseries(timeseries[i]); + } + session->createTimeseries(timeseries[i], types[i], TSEncoding::RLE, CompressionType::SNAPPY); + } + string deviceId = "root.test.d1"; + vector measurements = {"s1", "s2", "s3"}; + vector value1(100, 1); + vector value2(100, 2.2); + vector value3(100, 3); + + for (long time = 0; time < 100; time++) { + vector values = {(char*)(&value1[time]), (char*)(&value2[time]), (char*)(&value3[time])}; + session->insertRecord(deviceId, time, measurements, types, values); + } + + unique_ptr sessionDataSet = + session->executeQueryStatement("select s1,s2,s3 from root.test.d1"); + sessionDataSet->setFetchSize(1024); + long count = 0; + while (sessionDataSet->hasNext()) { + sessionDataSet->next(); + count++; + } + REQUIRE(count == 100); +} + +TEST_CASE("Test insertRecord with new datatypes ", "[testTypedInsertRecordNewDatatype]") { + CaseReporter cr("testTypedInsertRecordNewDatatype"); + vector timeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3", + "root.test.d1.s4"}; + std::vector types = {TSDataType::TIMESTAMP, TSDataType::DATE, + TSDataType::BLOB, TSDataType::STRING}; + + for (size_t i = 0; i < timeseries.size(); i++) { + if (session->checkTimeseriesExists(timeseries[i])) { + session->deleteTimeseries(timeseries[i]); + } + session->createTimeseries(timeseries[i], types[i], TSEncoding::PLAIN, CompressionType::SNAPPY); + } + string deviceId = "root.test.d1"; + vector measurements = {"s1", "s2", "s3", "s4"}; + int64_t value1 = 20250507; + IoTDBDate value2 = IoTDBDate(2025, 5, 7); + string value3 = "20250507"; + string value4 = "20250507"; + + for (long time = 0; time < 100; time++) { + vector values = {(char*)(&value1), (char*)(&value2), const_cast(value3.c_str()), + const_cast(value4.c_str())}; + session->insertRecord(deviceId, time, measurements, types, values); + } + + unique_ptr sessionDataSet = + session->executeQueryStatement("select s1,s2,s3,s4 from root.test.d1"); + sessionDataSet->setFetchSize(1024); + long count = 0; + while (sessionDataSet->hasNext()) { + auto record = sessionDataSet->next(); + REQUIRE(record->fields.size() == 4); + for (int i = 0; i < 4; i++) { + REQUIRE(types[i] == record->fields[i].dataType); + } + REQUIRE(record->fields[0].longV.value() == value1); + REQUIRE(record->fields[1].dateV.value() == value2); + REQUIRE(record->fields[2].stringV.value() == value3); + REQUIRE(record->fields[3].stringV.value() == value4); + count++; + } + REQUIRE(count == 100); +} + +TEST_CASE("Test insertRecords with types ", "[testTypedInsertRecords]") { + CaseReporter cr("testTypedInsertRecords"); + vector timeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3"}; + vector types = {TSDataType::INT32, TSDataType::DOUBLE, TSDataType::INT64}; + + for (size_t i = 0; i < timeseries.size(); i++) { + if (session->checkTimeseriesExists(timeseries[i])) { + session->deleteTimeseries(timeseries[i]); + } + session->createTimeseries(timeseries[i], types[i], TSEncoding::RLE, CompressionType::SNAPPY); + } + string deviceId = "root.test.d1"; + vector measurements = {"s1", "s2", "s3"}; + vector deviceIds; + vector> measurementsList; + vector> typesList; + vector> valuesList; + vector timestamps; + vector value1(100, 1); + vector value2(100, 2.2); + vector value3(100, 3); + + for (int64_t time = 0; time < 100; time++) { + vector values = {(char*)(&value1[time]), (char*)(&value2[time]), (char*)(&value3[time])}; + deviceIds.push_back(deviceId); + measurementsList.push_back(measurements); + typesList.push_back(types); + valuesList.push_back(values); + timestamps.push_back(time); + } + + session->insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList); + + unique_ptr sessionDataSet = + session->executeQueryStatement("select s1,s2,s3 from root.test.d1"); + sessionDataSet->setFetchSize(1024); + int count = 0; + while (sessionDataSet->hasNext()) { + sessionDataSet->next(); + count++; + } + REQUIRE(count == 100); +} + +TEST_CASE("Test insertRecordsOfOneDevice", "[testInsertRecordsOfOneDevice]") { + CaseReporter cr("testInsertRecordsOfOneDevice"); + vector timeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3"}; + vector types = {TSDataType::INT32, TSDataType::DOUBLE, TSDataType::INT64}; + + for (size_t i = 0; i < timeseries.size(); i++) { + if (session->checkTimeseriesExists(timeseries[i])) { + session->deleteTimeseries(timeseries[i]); + } + session->createTimeseries(timeseries[i], types[i], TSEncoding::RLE, CompressionType::SNAPPY); + } + string deviceId = "root.test.d1"; + vector measurements = {"s1", "s2", "s3"}; + vector> measurementsList; + vector> typesList; + vector> valuesList; + vector timestamps; + vector value1(100, 1); + vector value2(100, 2.2); + vector value3(100, 3); + + for (int64_t time = 0; time < 100; time++) { + vector values = {(char*)(&value1[time]), (char*)(&value2[time]), (char*)(&value3[time])}; + measurementsList.push_back(measurements); + typesList.push_back(types); + valuesList.push_back(values); + timestamps.push_back(time); + } + + session->insertRecordsOfOneDevice(deviceId, timestamps, measurementsList, typesList, valuesList); + + unique_ptr sessionDataSet = + session->executeQueryStatement("select * from root.test.d1"); + sessionDataSet->setFetchSize(1024); + int count = 0; + while (sessionDataSet->hasNext()) { + sessionDataSet->next(); + count++; + } + REQUIRE(count == 100); +} + +TEST_CASE("Test insertTablet ", "[testInsertTablet]") { + CaseReporter cr("testInsertTablet"); + prepareTimeseries(); + string deviceId = "root.test.d1"; + vector> schemaList; + schemaList.emplace_back("s1", TSDataType::INT64); + schemaList.emplace_back("s2", TSDataType::INT64); + schemaList.emplace_back("s3", TSDataType::INT64); + + Tablet tablet(deviceId, schemaList, 100); + for (int64_t time = 0; time < 100; time++) { + int row = tablet.rowSize++; + tablet.timestamps[row] = time; + for (int64_t i = 0; i < 3; i++) { + tablet.addValue(i, row, i); + } + if (tablet.rowSize == tablet.maxRowNumber) { + session->insertTablet(tablet); + tablet.reset(); + } + } + + if (tablet.rowSize != 0) { + session->insertTablet(tablet); + tablet.reset(); + } + unique_ptr sessionDataSet = + session->executeQueryStatement("select s1,s2,s3 from root.test.d1"); + sessionDataSet->setFetchSize(1024); + int count = 0; + while (sessionDataSet->hasNext()) { + long index = 0; + count++; + auto fields = sessionDataSet->next()->fields; + for (const Field& f : fields) { + REQUIRE(f.longV.value() == index); + index++; + } + } + REQUIRE(count == 100); +} + +TEST_CASE("Test insertTablets ", "[testInsertTablets]") { + CaseReporter cr("testInsertTablets"); + vector testTimeseries = {"root.test.d1.s1", "root.test.d1.s2", "root.test.d1.s3", + "root.test.d2.s1", "root.test.d2.s2", "root.test.d2.s3"}; + for (const string& timeseries : testTimeseries) { + if (session->checkTimeseriesExists(timeseries)) { + session->deleteTimeseries(timeseries); + } + session->createTimeseries(timeseries, TSDataType::INT64, TSEncoding::RLE, + CompressionType::SNAPPY); + } + vector> schemaList; + schemaList.emplace_back("s1", TSDataType::INT64); + schemaList.emplace_back("s2", TSDataType::INT64); + schemaList.emplace_back("s3", TSDataType::INT64); + + int maxRowNumber = 100; + vector deviceIds = {"root.test.d1", "root.test.d2"}; + vector tablets; + for (const auto& deviceId : deviceIds) { + tablets.emplace_back(deviceId, schemaList, maxRowNumber); + } + for (auto& tablet : tablets) { + for (int64_t time = 0; time < maxRowNumber; time++) { + int row = tablet.rowSize++; + tablet.timestamps[row] = time; + for (int64_t i = 0; i < 3; i++) { + tablet.addValue(i, row, i); + } + } + } + unordered_map tabletsMap; + for (auto& tablet : tablets) { + tabletsMap[tablet.deviceId] = &tablet; + } + session->insertTablets(tabletsMap); + + unique_ptr sessionDataSet = + session->executeQueryStatement("select s1,s2,s3 from root.test.d2"); + sessionDataSet->setFetchSize(1024); + int count = 0; + while (sessionDataSet->hasNext()) { + long index = 0; + count++; + auto fields = sessionDataSet->next()->fields; + for (const Field& f : fields) { + REQUIRE(f.longV.value() == index); + index++; + } + } + REQUIRE(count == 100); +} + +TEST_CASE("Test insertTablet multi datatype", "[testInsertTabletMultiDatatype]") { + CaseReporter cr("testInsertTabletNewDatatype"); + string deviceId = "root.test.d2"; + vector> schemaList; + std::vector measurements = {"s1", "s2", "s3", "s4"}; + std::vector dataTypes = {TSDataType::TIMESTAMP, TSDataType::DATE, + TSDataType::BLOB, TSDataType::STRING}; + for (int i = 0; i < 4; i++) { + schemaList.emplace_back(measurements[i], dataTypes[i]); + } + + for (int i = 0; i < 4; i++) { + auto timeseries = deviceId + "." + measurements[i]; + if (session->checkTimeseriesExists(timeseries)) { + session->deleteTimeseries(timeseries); + } + session->createTimeseries(timeseries, dataTypes[i], TSEncoding::PLAIN, + CompressionType::UNCOMPRESSED); + } + + int64_t s1Value = 20250507; + IoTDBDate s2Value(2025, 5, 7); + std::string s3Value("20250507"); + std::string s4Value("20250507"); + + Tablet tablet(deviceId, schemaList, 100); + for (int64_t time = 0; time < 100; time++) { + int row = tablet.rowSize++; + tablet.timestamps[row] = time; + tablet.addValue(0, row, s1Value); + tablet.addValue(1, row, s2Value); + tablet.addValue(2, row, s3Value); + tablet.addValue(3, row, s4Value); + if (tablet.rowSize == tablet.maxRowNumber) { + session->insertTablet(tablet); + tablet.reset(); + } + } + + if (tablet.rowSize != 0) { + session->insertTablet(tablet); + tablet.reset(); + } + unique_ptr sessionDataSet = + session->executeQueryStatement("select s1,s2,s3,s4 from root.test.d2"); + auto dataIter = sessionDataSet->getIterator(); + sessionDataSet->setFetchSize(1024); + int count = 0; + while (dataIter.next()) { + REQUIRE(dataIter.getLongByIndex(2).value() == s1Value); + REQUIRE(dataIter.getDateByIndex(3).value() == s2Value); + REQUIRE(dataIter.getStringByIndex(4).value() == s3Value); + REQUIRE(dataIter.getStringByIndex(5).value() == s4Value); + count++; + } + REQUIRE(count == 100); +} + +TEST_CASE("Test boolean column via DataIterator", "[testBooleanColumnDataIterator]") { + CaseReporter cr("testBooleanColumnDataIterator"); + string deviceId = "root.test.d1"; + string timeseries = "root.test.d1.s1"; + + if (session->checkTimeseriesExists(timeseries)) { + session->deleteTimeseries(timeseries); + } + session->createTimeseries(timeseries, TSDataType::BOOLEAN, TSEncoding::PLAIN, + CompressionType::SNAPPY); + + // Insert boolean values: even timestamps get true, odd get false + vector measurements = {"s1"}; + vector types = {TSDataType::BOOLEAN}; + for (int64_t time = 0; time < 10; time++) { + bool val = (time % 2 == 0); + vector values = {(char*)&val}; + session->insertRecord(deviceId, time, measurements, types, values); + } + + unique_ptr sessionDataSet = + session->executeQueryStatement("select s1 from root.test.d1"); + auto dataIter = sessionDataSet->getIterator(); + sessionDataSet->setFetchSize(1024); + int count = 0; + while (dataIter.next()) { + bool expected = (count % 2 == 0); + // Column 1 is Time, column 2 is s1 + REQUIRE(dataIter.getBooleanByIndex(2).value() == expected); + // Accessing time column (index 1) as boolean should throw + REQUIRE_THROWS_AS(dataIter.getBooleanByIndex(1), IoTDBException); + count++; + } + REQUIRE(count == 10); + session->deleteTimeseries(timeseries); +} + +TEST_CASE("Test Last query ", "[testLastQuery]") { + CaseReporter cr("testLastQuery"); + prepareTimeseries(); + string deviceId = "root.test.d1"; + vector measurements = {"s1", "s2", "s3"}; + + for (long time = 0; time < 100; time++) { + vector values = {"1", "2", "3"}; + session->insertRecord(deviceId, time, measurements, values); + } + + vector measurementValues = {"1", "2", "3"}; + unique_ptr sessionDataSet = + session->executeQueryStatement("select last s1,s2,s3 from root.test.d1"); + sessionDataSet->setFetchSize(1024); + long index = 0; + while (sessionDataSet->hasNext()) { + vector fields = sessionDataSet->next()->fields; + REQUIRE("1" <= fields[1].stringV.value()); + REQUIRE(fields[1].stringV.value() <= "3"); + index++; + } +} + +TEST_CASE("Test Huge query ", "[testHugeQuery]") { + CaseReporter cr("testHugeQuery"); + prepareTimeseries(); + string deviceId = "root.test.d1"; + vector measurements = {"s1", "s2", "s3"}; + vector types = {TSDataType::INT64, TSDataType::INT64, TSDataType::INT64}; + int64_t value1 = 1, value2 = 2, value3 = 3; + vector values = {(char*)&value1, (char*)&value2, (char*)&value3}; + + long total_count = 500000; + int print_count = 0; + std::cout.width(7); + std::cout << "inserting " << total_count << " rows:" << std::endl; + for (long time = 0; time < total_count; time++) { + session->insertRecord(deviceId, time, measurements, types, values); + if (time != 0 && time % 1000 == 0) { + std::cout << time << "\t" << std::flush; + if (++print_count % 20 == 0) { + std::cout << std::endl; + } + } + } + + unique_ptr sessionDataSet = + session->executeQueryStatement("select s1,s2,s3 from root.test.d1"); + sessionDataSet->setFetchSize(1024); + int count = 0; + print_count = 0; + std::cout << "\n\niterating " << total_count << " rows:" << std::endl; + while (sessionDataSet->hasNext()) { + auto rowRecord = sessionDataSet->next(); + REQUIRE(rowRecord->timestamp == count); + REQUIRE(rowRecord->fields[0].longV.value() == 1); + REQUIRE(rowRecord->fields[1].longV.value() == 2); + REQUIRE(rowRecord->fields[2].longV.value() == 3); + count++; + if (count % 1000 == 0) { + std::cout << count << "\t" << std::flush; + if (++print_count % 20 == 0) { + std::cout << std::endl; + } + } + } + + REQUIRE(count == total_count); +} + +TEST_CASE("Test executeRawDataQuery ", "[executeRawDataQuery]") { + CaseReporter cr("executeRawDataQuery"); + prepareTimeseries(); + + string deviceId = "root.test.d1"; + vector measurements = {"s1", "s2", "s3"}; + vector types = {TSDataType::INT64, TSDataType::INT64, TSDataType::INT64}; + + long total_count = 5000; + vector values; + int64_t valueArray[3]; + for (long time = -total_count; time < total_count; time++) { + valueArray[0] = time; + valueArray[1] = time * 2; + valueArray[2] = time * 3; + values.clear(); + values.push_back((char*)&valueArray[0]); + values.push_back((char*)&valueArray[1]); + values.push_back((char*)&valueArray[2]); + session->insertRecord(deviceId, time, measurements, types, values); + if (time == 100) { //insert 1 big timestamp data for generate un-seq data. + valueArray[0] = 9; + valueArray[2] = 999; + values.clear(); + values.push_back((char*)&valueArray[0]); + values.push_back((char*)&valueArray[2]); + vector measurements2 = {"s1", "s3"}; + vector types2 = {TSDataType::INT64, TSDataType::INT64}; + session->insertRecord(deviceId, 99999, measurements2, types2, values); + } + } + + vector paths; + paths.push_back("root.test.d1.s1"); + paths.push_back("root.test.d1.s2"); + paths.push_back("root.test.d1.s3"); + + //== Test executeRawDataQuery() with negative timestamp + int startTs = -total_count, endTs = total_count; + unique_ptr sessionDataSet = session->executeRawDataQuery(paths, startTs, endTs); + sessionDataSet->setFetchSize(10); + vector columns = sessionDataSet->getColumnNames(); + columns = sessionDataSet->getColumnNames(); + REQUIRE(columns[0] == "Time"); + REQUIRE(columns[1] == paths[0]); + REQUIRE(columns[2] == paths[1]); + REQUIRE(columns[3] == paths[2]); + + int ts = startTs; + while (sessionDataSet->hasNext()) { + auto rowRecordPtr = sessionDataSet->next(); + //cout << rowRecordPtr->toString(); + + vector fields = rowRecordPtr->fields; + REQUIRE(rowRecordPtr->timestamp == ts); + REQUIRE(fields[0].dataType == TSDataType::INT64); + REQUIRE(fields[0].longV.value() == ts); + REQUIRE(fields[1].dataType == TSDataType::INT64); + REQUIRE(fields[1].longV.value() == ts * 2); + REQUIRE(fields[2].dataType == TSDataType::INT64); + REQUIRE(fields[2].longV.value() == ts * 3); + ts++; + } + + //== Test executeRawDataQuery() with null field + startTs = 99999; + endTs = 99999 + 10; + sessionDataSet = session->executeRawDataQuery(paths, startTs, endTs); + + sessionDataSet->setFetchSize(10); + columns = sessionDataSet->getColumnNames(); + for (const string& column : columns) { + cout << column << " "; + } + cout << endl; + REQUIRE(columns[0] == "Time"); + REQUIRE(columns[1] == paths[0]); + REQUIRE(columns[2] == paths[1]); + REQUIRE(columns[3] == paths[2]); + ts = startTs; + while (sessionDataSet->hasNext()) { + auto rowRecordPtr = sessionDataSet->next(); + cout << rowRecordPtr->toString(); + + vector fields = rowRecordPtr->fields; + REQUIRE(rowRecordPtr->timestamp == ts); + REQUIRE(fields[0].dataType == TSDataType::INT64); + REQUIRE(fields[0].longV.value() == 9); + REQUIRE(fields[1].dataType == TSDataType::UNKNOWN); + REQUIRE(fields[2].dataType == TSDataType::INT64); + REQUIRE(fields[2].longV.value() == 999); + } + + //== Test executeRawDataQuery() with empty data + sessionDataSet = session->executeRawDataQuery(paths, 100000, 110000); + sessionDataSet->setFetchSize(1); + REQUIRE(sessionDataSet->hasNext() == false); +} + +TEST_CASE("Test executeLastDataQuery ", "[testExecuteLastDataQuery]") { + CaseReporter cr("testExecuteLastDataQuery"); + prepareTimeseries(); + + string deviceId = "root.test.d1"; + vector measurements = {"s1", "s2", "s3"}; + vector types = {TSDataType::INT64, TSDataType::INT64, TSDataType::INT64}; + + long total_count = 5000; + vector values; + int64_t valueArray[3]; + for (long time = -total_count; time < total_count; time++) { + valueArray[0] = time; + valueArray[1] = time * 2; + valueArray[2] = time * 3; + values.clear(); + values.push_back((char*)&valueArray[0]); + values.push_back((char*)&valueArray[1]); + values.push_back((char*)&valueArray[2]); + session->insertRecord(deviceId, time, measurements, types, values); + if (time == 100) { //insert 1 big timestamp data for gen unseq data. + valueArray[0] = 9; + valueArray[2] = 999; + values.clear(); + values.push_back((char*)&valueArray[0]); + values.push_back((char*)&valueArray[2]); + vector measurements2 = {"s1", "s3"}; + vector types2 = {TSDataType::INT64, TSDataType::INT64}; + session->insertRecord(deviceId, 99999, measurements2, types2, values); + } + } + + int64_t tsCheck[3] = {99999, 4999, 99999}; + std::vector valueCheck = {"9", "9998", "999"}; + + vector paths; + paths.push_back("root.test.d1.s1"); + paths.push_back("root.test.d1.s2"); + paths.push_back("root.test.d1.s3"); + + //== Test executeLastDataQuery() without lastTime + unique_ptr sessionDataSet = session->executeLastDataQuery(paths); + sessionDataSet->setFetchSize(1); + + vector columns = sessionDataSet->getColumnNames(); + for (const string& column : columns) { + cout << column << " "; + } + cout << endl; + + int index = 0; + while (sessionDataSet->hasNext()) { + auto rowRecordPtr = sessionDataSet->next(); + cout << rowRecordPtr->toString(); + + vector fields = rowRecordPtr->fields; + REQUIRE(rowRecordPtr->timestamp == tsCheck[index]); + REQUIRE(fields[0].stringV.value() == paths[index]); + REQUIRE(fields[1].stringV.value() == valueCheck[index]); + REQUIRE(fields[2].stringV.value() == "INT64"); + index++; + } + + //== Test executeLastDataQuery() with negative lastTime + sessionDataSet = session->executeLastDataQuery(paths, -200); + sessionDataSet->setFetchSize(1); + columns = sessionDataSet->getColumnNames(); + for (const string& column : columns) { + cout << column << " "; + } + cout << endl; + + index = 0; + while (sessionDataSet->hasNext()) { + auto rowRecordPtr = sessionDataSet->next(); + cout << rowRecordPtr->toString(); + + vector fields = rowRecordPtr->fields; + REQUIRE(rowRecordPtr->timestamp == tsCheck[index]); + REQUIRE(fields[0].stringV.value() == paths[index]); + REQUIRE(fields[1].stringV.value() == valueCheck[index]); + REQUIRE(fields[2].stringV.value() == "INT64"); + index++; + } + + //== Test executeLastDataQuery() with the lastTime that is > largest timestamp. + sessionDataSet = session->executeLastDataQuery(paths, 100000); + sessionDataSet->setFetchSize(1024); + REQUIRE(sessionDataSet->hasNext() == false); +} + +// Helper function for comparing TEndPoint with detailed error message +void assertTEndPointEqual(const TEndPoint& actual, const std::string& expectedIp, int expectedPort, + const char* file, int line) { + if (actual.ip != expectedIp || actual.port != expectedPort) { + std::stringstream ss; + ss << "\nTEndPoint mismatch:\nExpected: " << expectedIp << ":" << expectedPort + << "\nActual: " << actual.ip << ":" << actual.port; + Catch::SourceLineInfo location(file, line); + Catch::AssertionHandler handler("TEndPoint comparison", location, ss.str(), + Catch::ResultDisposition::Normal); + handler.handleMessage(Catch::ResultWas::ExplicitFailure, ss.str()); + handler.complete(); + } +} + +// Macro to simplify test assertions +#define REQUIRE_TENDPOINT(actual, expectedIp, expectedPort) \ + assertTEndPointEqual(actual, expectedIp, expectedPort, __FILE__, __LINE__) + +TEST_CASE("UrlUtils - parseTEndPointIpv4AndIpv6Url", "[UrlUtils]") { + // Test valid IPv4 addresses + SECTION("Valid IPv4") { + REQUIRE_TENDPOINT(UrlUtils::parseTEndPointIpv4AndIpv6Url("192.168.1.1:8080"), "192.168.1.1", + 8080); + REQUIRE_TENDPOINT(UrlUtils::parseTEndPointIpv4AndIpv6Url("10.0.0.1:80"), "10.0.0.1", 80); + } + + // Test valid IPv6 addresses + SECTION("Valid IPv6") { + REQUIRE_TENDPOINT(UrlUtils::parseTEndPointIpv4AndIpv6Url("[2001:db8::1]:8080"), "2001:db8::1", + 8080); + REQUIRE_TENDPOINT(UrlUtils::parseTEndPointIpv4AndIpv6Url("[::1]:80"), "::1", 80); + } + + // Test hostnames + SECTION("Hostnames") { + REQUIRE_TENDPOINT(UrlUtils::parseTEndPointIpv4AndIpv6Url("localhost:8080"), "localhost", 8080); + REQUIRE_TENDPOINT(UrlUtils::parseTEndPointIpv4AndIpv6Url("example.com:443"), "example.com", + 443); + } + + // Test edge cases + SECTION("Edge cases") { + REQUIRE_TENDPOINT(UrlUtils::parseTEndPointIpv4AndIpv6Url(""), "", 0); + REQUIRE_TENDPOINT(UrlUtils::parseTEndPointIpv4AndIpv6Url("127.0.0.1"), "127.0.0.1", 0); + } + + // Test invalid inputs + SECTION("Invalid inputs") { + REQUIRE_TENDPOINT(UrlUtils::parseTEndPointIpv4AndIpv6Url("192.168.1.1:abc"), "192.168.1.1:abc", + 0); + REQUIRE_TENDPOINT(UrlUtils::parseTEndPointIpv4AndIpv6Url("]invalid[:80"), "]invalid[", 80); + } + + // Test port ranges + SECTION("Port ranges") { + REQUIRE_TENDPOINT(UrlUtils::parseTEndPointIpv4AndIpv6Url("localhost:0"), "localhost", 0); + REQUIRE_TENDPOINT(UrlUtils::parseTEndPointIpv4AndIpv6Url("127.0.0.1:65535"), "127.0.0.1", + 65535); + } +} + +TEST_CASE("TsBlock deserialize rejects truncated malicious payload", "[TsBlockDeserialize]") { + std::string data(18, '\0'); + data[3] = '\x10'; + REQUIRE_THROWS_AS(TsBlock::deserialize(data), IoTDBException); +} + +TEST_CASE("Numeric column widening getters align with Java TsFile", "[column]") { + std::vector valueIsNull(1, false); + + std::vector floatValues = {120.00000762939453f}; + auto floatColumn = std::make_shared(0, 1, valueIsNull, floatValues); + auto rleColumn = std::make_shared(floatColumn, 20); + REQUIRE(floatColumn->getDouble(0) == Approx(120.0).margin(0.01)); + REQUIRE(rleColumn->getDouble(0) == Approx(120.0).margin(0.01)); + + std::vector intValues = {42}; + auto intColumn = std::make_shared(0, 1, valueIsNull, intValues); + REQUIRE(intColumn->getLong(0) == 42); + REQUIRE(intColumn->getDouble(0) == Approx(42.0)); + + std::vector longValues = {1000}; + auto longColumn = std::make_shared(0, 1, valueIsNull, longValues); + REQUIRE(longColumn->getDouble(0) == Approx(1000.0)); +} + +TEST_CASE("SessionPool basic borrow/insert/query via RAII lease", "[sessionPool]") { + CaseReporter cr("SessionPool basic"); + auto pool = SessionPoolBuilder() + .host("127.0.0.1") + ->rpcPort(6667) + ->username("root") + ->password("root") + ->maxSize(3) + ->build(); + + { + PooledSession s = pool->getSession(); + try { + s->executeNonQueryStatement("delete timeseries root.test.pool.d1.*"); + } catch (const std::exception &) { + // Ignore: the timeseries may not exist yet on a fresh database. + } + } + + const int rows = 50; + for (int i = 0; i < rows; i++) { + PooledSession s = pool->getSession(); + s->insertRecord("root.test.pool.d1", i, {"s1"}, {to_string(i)}); + } + + int count = 0; + { + PooledSessionDataSet ds = + pool->executeQueryStatement("select s1 from root.test.pool.d1"); + while (ds->hasNext()) { + ds->next(); + count++; + } + } + REQUIRE(count == rows); + + { + PooledSession s = pool->getSession(); + s->executeNonQueryStatement("delete timeseries root.test.pool.d1.*"); + } + pool->close(); +} + +TEST_CASE("SessionPool is safe under concurrent writers", "[sessionPool]") { + CaseReporter cr("SessionPool concurrency"); + auto pool = SessionPoolBuilder() + .host("127.0.0.1") + ->rpcPort(6667) + ->username("root") + ->password("root") + ->maxSize(4) + ->build(); + + { + PooledSession s = pool->getSession(); + try { + s->executeNonQueryStatement("delete timeseries root.test.pool.d2.*"); + } catch (const std::exception &) { + // Ignore: the timeseries may not exist yet on a fresh database. + } + } + + const int threadCount = 8; + const int rowsPerThread = 100; + std::atomic failures(0); + std::vector threads; + for (int t = 0; t < threadCount; t++) { + threads.emplace_back([&pool, t, rowsPerThread, &failures]() { + try { + for (int i = 0; i < rowsPerThread; i++) { + int64_t ts = static_cast(t) * rowsPerThread + i; + // Mix RAII and convenience APIs to exercise both borrow paths. + if (i % 2 == 0) { + pool->insertRecord("root.test.pool.d2", ts, {"s1"}, + {to_string(ts)}); + } else { + PooledSession s = pool->getSession(); + s->insertRecord("root.test.pool.d2", ts, {"s1"}, {to_string(ts)}); + } + } + } catch (const std::exception &e) { + std::cerr << "writer thread failed: " << e.what() << std::endl; + failures++; + } + }); + } + for (auto &th : threads) { + th.join(); + } + REQUIRE(failures.load() == 0); + REQUIRE(pool->getMaxSize() == 4); + + int count = 0; + { + PooledSessionDataSet ds = + pool->executeQueryStatement("select s1 from root.test.pool.d2"); + while (ds->hasNext()) { + ds->next(); + count++; + } + } + REQUIRE(count == threadCount * rowsPerThread); + + { + PooledSession s = pool->getSession(); + s->executeNonQueryStatement("delete timeseries root.test.pool.d2.*"); + } + pool->close(); +} + +TEST_CASE("SessionPool getSession times out when exhausted", "[sessionPool]") { + CaseReporter cr("SessionPool exhaustion timeout"); + auto pool = SessionPoolBuilder() + .host("127.0.0.1") + ->rpcPort(6667) + ->username("root") + ->password("root") + ->maxSize(1) + ->waitToGetSessionTimeoutMs(200) + ->build(); + + PooledSession held = pool->getSession(); + REQUIRE(static_cast(held)); + REQUIRE(pool->activeCount() == 1); + // The only Session is checked out, so a second borrow must time out. + REQUIRE_THROWS_AS(pool->getSession(), IoTDBException); + + held.release(); + // After returning it, a borrow succeeds again. + PooledSession reused = pool->getSession(); + REQUIRE(static_cast(reused)); + reused.release(); + pool->close(); +} diff --git a/iotdb-client/client-cpp/test/main.cpp b/iotdb-client/client-cpp/test/main.cpp new file mode 100644 index 0000000000000..256b733b291f5 --- /dev/null +++ b/iotdb-client/client-cpp/test/main.cpp @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#define CATCH_CONFIG_MAIN + +#include +#include "Session.h" +#include "SessionBuilder.h" + +std::shared_ptr session; + +struct SessionListener : Catch::TestEventListenerBase { + + using TestEventListenerBase::TestEventListenerBase; + + void testCaseStarting(Catch::TestCaseInfo const& testInfo) override { + if (!session) { + SessionBuilder builder; + session = builder.host("127.0.0.1") + ->rpcPort(6667) + ->username("root") + ->password("root") + ->useSSL(false) + ->enableAutoFetch(false) + ->build(); + } else { + session->open(false); + } + } + + void testCaseEnded(Catch::TestCaseStats const& testCaseStats) override { + if (session) { + session->close(); + } + } + + void testRunEnded(Catch::TestRunStats const& testRunStats) override { + // Release session before static/global teardown on Windows. + session.reset(); + } +}; + +CATCH_REGISTER_LISTENER(SessionListener) \ No newline at end of file diff --git a/iotdb-client/client-cpp/test/main_c.cpp b/iotdb-client/client-cpp/test/main_c.cpp new file mode 100644 index 0000000000000..056b1f49b4ca1 --- /dev/null +++ b/iotdb-client/client-cpp/test/main_c.cpp @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#define CATCH_CONFIG_MAIN + +#include +#include "SessionC.h" + +// Global session handle used by the C API tree-model tests +CSession* g_session = nullptr; + +struct CSessionListener : Catch::TestEventListenerBase { + using TestEventListenerBase::TestEventListenerBase; + + void testCaseStarting(Catch::TestCaseInfo const& testInfo) override { + g_session = ts_session_new("127.0.0.1", 6667, "root", "root"); + REQUIRE(g_session != nullptr); + TsStatus st = ts_session_open(g_session); + if (st != TS_OK) { + ts_session_destroy(g_session); + g_session = nullptr; + FAIL("ts_session_open failed; ensure distribution is built and IoTDB listens on " + "127.0.0.1:6667"); + } + } + + void testCaseEnded(Catch::TestCaseStats const& testCaseStats) override { + if (g_session) { + ts_session_close(g_session); + ts_session_destroy(g_session); + g_session = nullptr; + } + } +}; + +CATCH_REGISTER_LISTENER(CSessionListener) diff --git a/iotdb-client/client-cpp/third-party/.gitignore b/iotdb-client/client-cpp/third-party/.gitignore new file mode 100644 index 0000000000000..c8badffab8dc0 --- /dev/null +++ b/iotdb-client/client-cpp/third-party/.gitignore @@ -0,0 +1,7 @@ +# Ignore all cached archives; keep directory structure and README tracked. +* +!.gitignore +!README.md +!linux/ +!mac/ +!windows/ diff --git a/iotdb-client/client-cpp/third-party/README.md b/iotdb-client/client-cpp/third-party/README.md new file mode 100644 index 0000000000000..313a6fb79a1d6 --- /dev/null +++ b/iotdb-client/client-cpp/third-party/README.md @@ -0,0 +1,75 @@ + +# Third-party dependency cache (`client-cpp/third-party`) + +CMake downloads (or reuses) build-time tarballs and archives here. The +directory ships with the source tree so you can **stage dependencies on a +networked machine, copy the whole IoTDB checkout to an offline host, and +build `client-cpp` with `-DIOTDB_OFFLINE=ON`**. + +Tarballs themselves are **not** committed to Git (see per-platform +`.gitignore` files). Only this README and the empty platform folders are +tracked. Each `linux/`, `mac/`, and `windows/` sub-folder ships a minimal +`.gitignore` (`*` with `!.gitignore`) so Git keeps the directory in the +tree while ignoring downloaded archives. + +## Layout + +``` +third-party/ +├── linux/ # tarballs for Linux offline builds +├── mac/ # tarballs for macOS offline builds +└── windows/ # tarballs / zips for Windows offline builds +``` + +Override the root with `-DIOTDB_DEPS_DIR=` (Maven: `-Diotdb.deps.dir=...`). +The platform sub-folder (`linux/`, `mac/`, `windows/`) is selected automatically. + +## Staging dependencies (online machine) + +Run a normal **online** configure once; CMake caches everything under the +matching `/` folder: + +```bash +cmake -S iotdb-client/client-cpp -B build +# or: mvn -P with-cpp -pl iotdb-client/client-cpp -am -DskipTests package +``` + +Alternatively copy files manually from the URLs listed in +[`README.md`](../README.md) (*Offline build* section). + +## Offline machine (copy whole IoTDB tree) + +1. Copy the entire IoTDB repository (including `iotdb-client/client-cpp/third-party//`). +2. Configure with offline mode: + + ```bash + cmake -S iotdb-client/client-cpp -B build -DIOTDB_OFFLINE=ON + cmake --build build --config Release --target install + ``` + +## Per-platform files (offline minimum) + +| Platform | Typical files | +|------------|---------------| +| `linux/` | `thrift-0.21.0.tar.gz`, `boost_1_60_0.tar.gz`, `m4-1.4.19.tar.gz`, `flex-2.6.4.tar.gz`, `bison-3.8.tar.gz` (+ `openssl-3.5.0.tar.gz` when `WITH_SSL=ON`) | +| `mac/` | `thrift-0.21.0.tar.gz`, `boost_1_60_0.tar.gz` (Xcode CLT usually provides m4/flex/bison) | +| `windows/` | `thrift-0.21.0.tar.gz`, `boost_1_60_0.tar.gz`, `win_flex_bison-2.5.25.zip` (or any `win_flex_bison*.zip`; skip if flex/bison already on `PATH`) | + +Download URLs: see the *Offline build* table in [`README.md`](../README.md). diff --git a/iotdb-client/client-cpp/third-party/linux/.gitignore b/iotdb-client/client-cpp/third-party/linux/.gitignore new file mode 100644 index 0000000000000..d6b7ef32c8478 --- /dev/null +++ b/iotdb-client/client-cpp/third-party/linux/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/iotdb-client/client-cpp/third-party/mac/.gitignore b/iotdb-client/client-cpp/third-party/mac/.gitignore new file mode 100644 index 0000000000000..d6b7ef32c8478 --- /dev/null +++ b/iotdb-client/client-cpp/third-party/mac/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/iotdb-client/client-cpp/third-party/windows/.gitignore b/iotdb-client/client-cpp/third-party/windows/.gitignore new file mode 100644 index 0000000000000..d6b7ef32c8478 --- /dev/null +++ b/iotdb-client/client-cpp/third-party/windows/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java index 822c464ddcadf..fc671b83926dd 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java @@ -40,8 +40,6 @@ import java.util.List; import java.util.Map; -import static org.apache.iotdb.rpc.IoTDBRpcDataSet.START_INDEX; - public class SessionDataSet implements ISessionDataSet { private final IoTDBRpcDataSet ioTDBRpcDataSet; @@ -176,56 +174,45 @@ public boolean hasNext() throws StatementExecutionException, IoTDBConnectionExce } private RowRecord constructRowRecordFromValueArray() throws StatementExecutionException { - List outFields = new ArrayList<>(); - for (int i = 0; i < ioTDBRpcDataSet.columnSize; i++) { + int valueColumnStartIndex = ioTDBRpcDataSet.getValueColumnStartIndex(); + int columnSize = ioTDBRpcDataSet.getColumnSize(); + List outFields = new ArrayList<>(columnSize - valueColumnStartIndex); + for (int columnIndex = valueColumnStartIndex + 1; columnIndex <= columnSize; columnIndex++) { Field field; - - int index = i + 1; - int datasetColumnIndex = i + START_INDEX; - if (ioTDBRpcDataSet.ignoreTimeStamp) { - index--; - datasetColumnIndex--; - } - int loc = - ioTDBRpcDataSet.columnOrdinalMap.get(ioTDBRpcDataSet.columnNameList.get(index)) - - START_INDEX; - - if (!ioTDBRpcDataSet.isNull(datasetColumnIndex)) { - TSDataType dataType = ioTDBRpcDataSet.columnTypeDeduplicatedList.get(loc); + if (!ioTDBRpcDataSet.isNull(columnIndex)) { + TSDataType dataType = ioTDBRpcDataSet.getDataType(columnIndex); field = new Field(dataType); switch (dataType) { case BOOLEAN: - boolean booleanValue = ioTDBRpcDataSet.getBoolean(datasetColumnIndex); + boolean booleanValue = ioTDBRpcDataSet.getBoolean(columnIndex); field.setBoolV(booleanValue); break; case INT32: case DATE: - int intValue = ioTDBRpcDataSet.getInt(datasetColumnIndex); + int intValue = ioTDBRpcDataSet.getInt(columnIndex); field.setIntV(intValue); break; case INT64: case TIMESTAMP: - long longValue = ioTDBRpcDataSet.getLong(datasetColumnIndex); + long longValue = ioTDBRpcDataSet.getLong(columnIndex); field.setLongV(longValue); break; case FLOAT: - float floatValue = ioTDBRpcDataSet.getFloat(datasetColumnIndex); + float floatValue = ioTDBRpcDataSet.getFloat(columnIndex); field.setFloatV(floatValue); break; case DOUBLE: - double doubleValue = ioTDBRpcDataSet.getDouble(datasetColumnIndex); + double doubleValue = ioTDBRpcDataSet.getDouble(columnIndex); field.setDoubleV(doubleValue); break; case TEXT: case BLOB: case STRING: - field.setBinaryV(ioTDBRpcDataSet.getBinary(datasetColumnIndex)); + field.setBinaryV(ioTDBRpcDataSet.getBinary(columnIndex)); break; default: throw new UnSupportedDataTypeException( - String.format( - "Data type %s is not supported.", - ioTDBRpcDataSet.columnTypeDeduplicatedList.get(i))); + String.format("Data type %s is not supported.", dataType)); } } else { field = new Field(null); diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java index 5bba6d0ea718d..254c6d685982f 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java @@ -127,20 +127,15 @@ public IoTDBJDBCDataSet( // deduplicate and map if (columnNameIndex != null) { - int deduplicatedColumnSize = (int) columnNameIndex.values().stream().distinct().count(); - this.columnTypeDeduplicatedList = new ArrayList<>(deduplicatedColumnSize); - for (int i = 0; i < deduplicatedColumnSize; i++) { - columnTypeDeduplicatedList.add(null); - } + this.columnTypeDeduplicatedList = + initDeduplicatedColumnTypes(getDeduplicatedColumnSize(columnNameIndex)); for (int i = 0; i < columnNameList.size(); i++) { String name = columnNameList.get(i); this.columnNameList.add(name); this.columnTypeList.add(columnTypeList.get(i)); if (!columnOrdinalMap.containsKey(name)) { int index = columnNameIndex.get(name); - if (!columnOrdinalMap.containsValue(index + START_INDEX)) { - columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnTypeList.get(i))); - } + setColumnTypeIfAbsent(columnTypeDeduplicatedList, index, columnTypeList.get(i)); columnOrdinalMap.put(name, index + START_INDEX); } } @@ -241,11 +236,8 @@ public IoTDBJDBCDataSet( // deduplicate and map if (columnNameIndex != null) { - int deduplicatedColumnSize = (int) columnNameIndex.values().stream().distinct().count(); - this.columnTypeDeduplicatedList = new ArrayList<>(deduplicatedColumnSize); - for (int i = 0; i < deduplicatedColumnSize; i++) { - columnTypeDeduplicatedList.add(null); - } + this.columnTypeDeduplicatedList = + initDeduplicatedColumnTypes(getDeduplicatedColumnSize(columnNameIndex)); for (int i = 0; i < columnNameList.size(); i++) { String name = ""; if (sgList != null @@ -261,9 +253,7 @@ public IoTDBJDBCDataSet( // "Time".equals(name) -> to allow the Time column appear in value columns if (!columnOrdinalMap.containsKey(name) || "Time".equals(name)) { int index = columnNameIndex.get(name); - if (!columnOrdinalMap.containsValue(index + START_INDEX)) { - columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnTypeList.get(i))); - } + setColumnTypeIfAbsent(columnTypeDeduplicatedList, index, columnTypeList.get(i)); columnOrdinalMap.put(name, index + START_INDEX); } } @@ -318,6 +308,31 @@ public IoTDBJDBCDataSet( this.emptyResultSet = (queryDataSet == null || !queryDataSet.time.hasRemaining()); } + private static int getDeduplicatedColumnSize(Map columnNameIndex) { + int deduplicatedColumnSize = 0; + for (Integer index : columnNameIndex.values()) { + if (index != null && index + 1 > deduplicatedColumnSize) { + deduplicatedColumnSize = index + 1; + } + } + return deduplicatedColumnSize; + } + + private static List initDeduplicatedColumnTypes(int deduplicatedColumnSize) { + List columnTypes = new ArrayList<>(deduplicatedColumnSize); + for (int i = 0; i < deduplicatedColumnSize; i++) { + columnTypes.add(null); + } + return columnTypes; + } + + private static void setColumnTypeIfAbsent( + List columnTypeDeduplicatedList, int index, String columnType) { + if (columnTypeDeduplicatedList.get(index) == null) { + columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnType)); + } + } + public void close() throws StatementExecutionException, TException { if (isClosed) { return; diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java index 8fc484fd00c0a..2ff01502fb311 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java @@ -634,4 +634,12 @@ public void checkRecord() throws StatementExecutionException { throw new StatementExecutionException("No record remains"); } } + + public int getValueColumnStartIndex() { + return ignoreTimeStamp ? 0 : 1; + } + + public int getColumnSize() { + return columnNameList.size(); + } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TCompressedElasticFramedTransport.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TCompressedElasticFramedTransport.java index a3b4f38064a88..07768dd98bdd8 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TCompressedElasticFramedTransport.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TCompressedElasticFramedTransport.java @@ -44,13 +44,7 @@ protected TCompressedElasticFramedTransport( protected void readFrame() throws TTransportException { underlying.readAll(i32buf, 0, 4); int size = TFramedTransport.decodeFrameSize(i32buf); - - if (size < 0) { - close(); - throw new TTransportException( - TTransportException.CORRUPTED_DATA, "Read a negative frame size (" + size + ")!"); - } - + checkFrameSize(size); readBuffer.fill(underlying, size); RpcStat.readCompressedBytes.addAndGet(size); try { @@ -75,6 +69,7 @@ public void flush() throws TTransportException { writeCompressBuffer.resizeIfNecessary(maxCompressedLength); int compressedLength = compress(writeBuffer.getBuffer(), 0, length, writeCompressBuffer.getBuffer(), 0); + checkWriteFrameSize(compressedLength); RpcStat.writeCompressedBytes.addAndGet(compressedLength); TFramedTransport.encodeFrameSize(compressedLength, i32buf); underlying.write(i32buf, 0, 4); diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java index 67948a19f049d..40811a00f41df 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java @@ -174,34 +174,88 @@ public int read(byte[] buf, int off, int len) throws TTransportException { protected void readFrame() throws TTransportException { underlying.readAll(i32buf, 0, 4); int size = TFramedTransport.decodeFrameSize(i32buf); + checkFrameSize(size); + readBuffer.fill(underlying, size); + } - if (size < 0) { - close(); - throw new TTransportException( - TTransportException.CORRUPTED_DATA, "Read a negative frame size (" + size + ")!"); + protected void checkFrameSize(int size) throws TTransportException { + final int HTTP_GET_SIGNATURE = 0x47455420; // "GET " + final int HTTP_POST_SIGNATURE = 0x504F5354; // "POST" + final int TLS_MIN_VERSION = 0x160300; + final int TLS_MAX_VERSION = 0x160303; + final int TLS_LENGTH_HIGH_MAX = 0x02; + + FrameError error = null; + if (size == HTTP_GET_SIGNATURE || size == HTTP_POST_SIGNATURE) { + error = FrameError.HTTP_REQUEST; + } else { + int high24 = size >>> 8; + if (high24 >= TLS_MIN_VERSION + && high24 <= TLS_MAX_VERSION + && (i32buf[3] & 0xFF) <= TLS_LENGTH_HIGH_MAX) { + error = FrameError.TLS_REQUEST; + } else if (size < 0) { + error = FrameError.NEGATIVE_FRAME_SIZE; + } else if (size > thriftMaxFrameSize) { + error = FrameError.FRAME_SIZE_EXCEEDED; + } } - if (size > thriftMaxFrameSize) { - close(); - if (size == 1195725856L || size == 1347375956L) { - // if someone sends HTTP GET/POST to this port, the size will be read as the following - throw new TTransportException( - TTransportException.CORRUPTED_DATA, - "Singular frame size (" - + size - + ") detected, you may be sending HTTP GET/POST requests to the Thrift-RPC port, please confirm that you are using the right port"); - } else { - throw new TTransportException( - TTransportException.CORRUPTED_DATA, - "Frame size (" + size + ") larger than protect max size (" + thriftMaxFrameSize + ")!"); - } + if (error == null) { + return; } - readBuffer.fill(underlying, size); + + SocketAddress remoteAddress = null; + if (underlying instanceof TSocket) { + remoteAddress = ((TSocket) underlying).getSocket().getRemoteSocketAddress(); + } + String remoteInfo = (remoteAddress == null) ? "" : " from " + remoteAddress; + close(); + + error.throwException(size, remoteInfo, thriftMaxFrameSize); + } + + private enum FrameError { + HTTP_REQUEST( + "Singular frame size (%d) detected, you may be sending HTTP GET/POST%s " + + "requests to the Thrift-RPC port, please confirm that you are using the right port"), + TLS_REQUEST( + "Singular frame size (%d) detected, you may be sending TLS ClientHello " + + "requests%s to the Non-SSL Thrift-RPC port, please confirm that you are using " + + "the right configuration"), + NEGATIVE_FRAME_SIZE("Read a negative frame size (%d)%s!"), + FRAME_SIZE_EXCEEDED("Frame size (%d) larger than protect max size (%d)%s!"), + STRING_LENGTH_EXCEEDED("String length (%d) larger than protect max size (%d)%s!"); + + private final String messageFormat; + + FrameError(String messageFormat) { + this.messageFormat = messageFormat; + } + + void throwException(long size, String remoteInfo, int maxSize) throws TTransportException { + String message = + (this == FRAME_SIZE_EXCEEDED || this == STRING_LENGTH_EXCEEDED) + ? String.format(messageFormat, size, maxSize, remoteInfo) + : String.format(messageFormat, size, remoteInfo); + throw new TTransportException(TTransportException.CORRUPTED_DATA, message); + } + } + + protected void checkWriteFrameSize(int size) throws TTransportException { + if (size <= thriftMaxFrameSize) { + return; + } + close(); + throw new TTransportException( + TTransportException.CORRUPTED_DATA, + "Frame size (" + size + ") larger than protect max size (" + thriftMaxFrameSize + ")!"); } @Override public void flush() throws TTransportException { int length = writeBuffer.getPos(); + checkWriteFrameSize(length); TFramedTransport.encodeFrameSize(length, i32buf); underlying.write(i32buf, 0, 4); underlying.write(writeBuffer.getBuffer(), 0, length); @@ -224,8 +278,15 @@ public void updateKnownMessageSize(long size) throws TTransportException { @Override public void checkReadBytesAvailable(long numBytes) throws TTransportException { - // do nothing now. - // here we can do some checkm, e.g., see whether the memory is enough. + if (numBytes >= thriftMaxFrameSize) { + SocketAddress remoteAddress = null; + if (underlying instanceof TSocket) { + remoteAddress = ((TSocket) underlying).getSocket().getRemoteSocketAddress(); + } + String remoteInfo = (remoteAddress == null) ? "" : " from " + remoteAddress; + close(); + FrameError.STRING_LENGTH_EXCEEDED.throwException(numBytes, remoteInfo, thriftMaxFrameSize); + } } @Override diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index c2f76d294847c..e2855ed0a5032 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -47,6 +47,7 @@ public enum TSStatusCode { INTERNAL_SERVER_ERROR(305), DISPATCH_ERROR(306), LICENSE_ERROR(307), + THRIFT_FRAME_OVERSIZE(308), // Client, REDIRECTION_RECOMMEND(400), diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index ff7c927871d94..c8d26f0cefe68 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -788,13 +788,12 @@ private TSCreateAlignedTimeseriesReq getTSCreateAlignedTimeseriesReq( TSCreateAlignedTimeseriesReq request = new TSCreateAlignedTimeseriesReq(); request.setPrefixPath(prefixPath); request.setMeasurements(measurements); - request.setDataTypes(dataTypes.stream().map(TSDataType::ordinal).collect(Collectors.toList())); - request.setEncodings(encodings.stream().map(TSEncoding::ordinal).collect(Collectors.toList())); - request.setCompressors( - compressors.stream().map(i -> (int) i.serialize()).collect(Collectors.toList())); - request.setMeasurementAlias(measurementAliasList); - request.setTagsList(tagsList); - request.setAttributesList(attributesList); + request.setDataTypes(toDataTypeOrdinals(dataTypes)); + request.setEncodings(toEncodingOrdinals(encodings)); + request.setCompressors(toCompressionOrdinals(compressors)); + request.setMeasurementAlias(replaceNullStrings(measurementAliasList)); + request.setTagsList(replaceNullMaps(tagsList)); + request.setAttributesList(replaceNullMaps(attributesList)); return request; } @@ -834,29 +833,14 @@ private TSCreateMultiTimeseriesReq genTSCreateMultiTimeseriesReq( TSCreateMultiTimeseriesReq request = new TSCreateMultiTimeseriesReq(); request.setPaths(paths); - - List dataTypeOrdinals = new ArrayList<>(dataTypes.size()); - for (TSDataType dataType : dataTypes) { - dataTypeOrdinals.add(dataType.ordinal()); - } - request.setDataTypes(dataTypeOrdinals); - - List encodingOrdinals = new ArrayList<>(dataTypes.size()); - for (TSEncoding encoding : encodings) { - encodingOrdinals.add(encoding.ordinal()); - } - request.setEncodings(encodingOrdinals); - - List compressionOrdinals = new ArrayList<>(paths.size()); - for (CompressionType compression : compressors) { - compressionOrdinals.add((int) compression.serialize()); - } - request.setCompressors(compressionOrdinals); + request.setDataTypes(toDataTypeOrdinals(dataTypes)); + request.setEncodings(toEncodingOrdinals(encodings)); + request.setCompressors(toCompressionOrdinals(compressors)); request.setPropsList(propsList); - request.setTagsList(tagsList); - request.setAttributesList(attributesList); - request.setMeasurementAliasList(measurementAliasList); + request.setTagsList(replaceNullMaps(tagsList)); + request.setAttributesList(replaceNullMaps(attributesList)); + request.setMeasurementAliasList(replaceNullStrings(measurementAliasList)); return request; } @@ -1042,8 +1026,7 @@ public SessionDataSet executeLastDataQuery(List paths, long lastTime, lo @Override public SessionDataSet executeLastDataQuery(List paths) throws StatementExecutionException, IoTDBConnectionException { - long time = 0L; - return executeLastDataQuery(paths, time, queryTimeoutInMs); + return executeLastDataQuery(paths, Long.MIN_VALUE, queryTimeoutInMs); } @Override @@ -1676,19 +1659,23 @@ private boolean filterNullValueAndMeasurement( List measurementsList, List types, List valuesList) { - Map nullMap = new HashMap<>(); + Map nullMap = logger.isInfoEnabled() ? new HashMap<>() : null; for (int i = valuesList.size() - 1; i >= 0; i--) { if (valuesList.get(i) == null) { - nullMap.put(measurementsList.get(i), valuesList.get(i)); + if (nullMap != null) { + nullMap.put(measurementsList.get(i), valuesList.get(i)); + } valuesList.remove(i); measurementsList.remove(i); types.remove(i); } } if (valuesList.isEmpty()) { - logger.info("All values of the {} are null,null values are {}", deviceId, nullMap); + if (nullMap != null) { + logger.info("All values of the {} are null,null values are {}", deviceId, nullMap); + } return true; - } else { + } else if (nullMap != null) { logger.info("Some values of {} are null,null values are {}", deviceId, nullMap); } return false; @@ -1734,18 +1721,22 @@ private void filterNullValueAndMeasurementWithStringType( */ private boolean filterNullValueAndMeasurementWithStringType( List valuesList, String deviceId, List measurementsList) { - Map nullMap = new HashMap<>(); + Map nullMap = logger.isInfoEnabled() ? new HashMap<>() : null; for (int i = valuesList.size() - 1; i >= 0; i--) { if (valuesList.get(i) == null) { - nullMap.put(measurementsList.get(i), valuesList.get(i)); + if (nullMap != null) { + nullMap.put(measurementsList.get(i), valuesList.get(i)); + } valuesList.remove(i); measurementsList.remove(i); } } if (valuesList.isEmpty()) { - logger.info("All values of the {} are null,null values are {}", deviceId, nullMap); + if (nullMap != null) { + logger.info("All values of the {} are null,null values are {}", deviceId, nullMap); + } return true; - } else { + } else if (nullMap != null) { logger.info("Some values of {} are null,null values are {}", deviceId, nullMap); } return false; @@ -2482,7 +2473,57 @@ private TSInsertStringRecordsOfOneDeviceReq genTSInsertStringRecordsOfOneDeviceR * @return ordered list */ private static List sortList(List source, Integer[] index) { - return Arrays.stream(index).map(source::get).collect(Collectors.toList()); + List sortedList = new ArrayList<>(index.length); + for (int position : index) { + sortedList.add(source.get(position)); + } + return sortedList; + } + + private static List toDataTypeOrdinals(List dataTypes) { + List ordinals = new ArrayList<>(dataTypes.size()); + for (TSDataType dataType : dataTypes) { + ordinals.add(dataType.ordinal()); + } + return ordinals; + } + + private static List toEncodingOrdinals(List encodings) { + List ordinals = new ArrayList<>(encodings.size()); + for (TSEncoding encoding : encodings) { + ordinals.add(encoding.ordinal()); + } + return ordinals; + } + + private static List toCompressionOrdinals(List compressors) { + List ordinals = new ArrayList<>(compressors.size()); + for (CompressionType compression : compressors) { + ordinals.add((int) compression.serialize()); + } + return ordinals; + } + + private static List replaceNullStrings(List values) { + if (values == null) { + return null; + } + List replacedValues = new ArrayList<>(values.size()); + for (String value : values) { + replacedValues.add(value != null ? value : ""); + } + return replacedValues; + } + + private static List> replaceNullMaps(List> values) { + if (values == null) { + return null; + } + List> replacedValues = new ArrayList<>(values.size()); + for (Map value : values) { + replacedValues.add(value != null ? value : new HashMap<>()); + } + return replacedValues; } private List objectValuesListToByteBufferList( @@ -3524,10 +3565,9 @@ public void addAlignedMeasurementsInTemplate( TSAppendSchemaTemplateReq req = new TSAppendSchemaTemplateReq(); req.setName(templateName); req.setMeasurements(measurementsPath); - req.setDataTypes(dataTypes.stream().map(TSDataType::ordinal).collect(Collectors.toList())); - req.setEncodings(encodings.stream().map(TSEncoding::ordinal).collect(Collectors.toList())); - req.setCompressors( - compressors.stream().map(i -> (int) i.serialize()).collect(Collectors.toList())); + req.setDataTypes(toDataTypeOrdinals(dataTypes)); + req.setEncodings(toEncodingOrdinals(encodings)); + req.setCompressors(toCompressionOrdinals(compressors)); req.setIsAligned(true); defaultSessionConnection.appendSchemaTemplate(req); } @@ -3570,10 +3610,9 @@ public void addUnalignedMeasurementsInTemplate( TSAppendSchemaTemplateReq req = new TSAppendSchemaTemplateReq(); req.setName(templateName); req.setMeasurements(measurementsPath); - req.setDataTypes(dataTypes.stream().map(TSDataType::ordinal).collect(Collectors.toList())); - req.setEncodings(encodings.stream().map(TSEncoding::ordinal).collect(Collectors.toList())); - req.setCompressors( - compressors.stream().map(i -> (int) i.serialize()).collect(Collectors.toList())); + req.setDataTypes(toDataTypeOrdinals(dataTypes)); + req.setEncodings(toEncodingOrdinals(encodings)); + req.setCompressors(toCompressionOrdinals(compressors)); req.setIsAligned(false); defaultSessionConnection.appendSchemaTemplate(req); } diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConfig.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConfig.java index 5094ae7eea42e..97c35baf8f4a3 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConfig.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConfig.java @@ -68,6 +68,18 @@ public String getConsumerGroupId() { return getString(ConsumerConstant.CONSUMER_GROUP_ID_KEY); } + public String getUsername() { + return getString(ConsumerConstant.USERNAME_KEY); + } + + public String getPassword() { + return getString(ConsumerConstant.PASSWORD_KEY); + } + + public String getEncryptedPassword() { + return getString(ConsumerConstant.ENCRYPTED_PASSWORD_KEY); + } + public long getHeartbeatIntervalMs() { return getLongOrDefault( ConsumerConstant.HEARTBEAT_INTERVAL_MS_KEY, diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConstant.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConstant.java index 504893b80ed55..21b13e5d3f128 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConstant.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/rpc/subscription/config/ConsumerConstant.java @@ -31,6 +31,7 @@ public class ConsumerConstant { public static final String USERNAME_KEY = "username"; public static final String PASSWORD_KEY = "password"; + public static final String ENCRYPTED_PASSWORD_KEY = "encrypted-password"; public static final String CONSUMER_ID_KEY = "consumer-id"; public static final String CONSUMER_GROUP_ID_KEY = "group-id"; diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java index 92c6b2c9a3e29..56f1c2e9b9b02 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionConsumer.java @@ -100,6 +100,7 @@ abstract class SubscriptionConsumer implements AutoCloseable { private final String username; private final String password; + private final String encryptedPassword; protected String consumerId; protected String consumerGroupId; @@ -177,6 +178,7 @@ protected SubscriptionConsumer(final Builder builder) { this.username = builder.username; this.password = builder.password; + this.encryptedPassword = builder.encryptedPassword; this.consumerId = builder.consumerId; this.consumerGroupId = builder.consumerGroupId; @@ -206,6 +208,7 @@ protected SubscriptionConsumer(final Builder builder, final Properties propertie (String) properties.getOrDefault( ConsumerConstant.PASSWORD_KEY, SessionConfig.DEFAULT_PASSWORD)) + .encryptedPassword((String) properties.get(ConsumerConstant.ENCRYPTED_PASSWORD_KEY)) .consumerId((String) properties.get(ConsumerConstant.CONSUMER_ID_KEY)) .consumerGroupId((String) properties.get(ConsumerConstant.CONSUMER_GROUP_ID_KEY)) .heartbeatIntervalMs( @@ -386,6 +389,7 @@ SubscriptionProvider constructProviderAndHandshake(final TEndPoint endPoint) endPoint, this.username, this.password, + this.encryptedPassword, this.consumerId, this.consumerGroupId, this.thriftMaxFrameSize, @@ -1401,6 +1405,7 @@ public abstract static class Builder { protected String username = SessionConfig.DEFAULT_USER; protected String password = SessionConfig.DEFAULT_PASSWORD; + protected String encryptedPassword; protected String consumerId; protected String consumerGroupId; @@ -1437,10 +1442,27 @@ public Builder username(final String username) { } public Builder password(final String password) { + if (!Objects.equals(password, SessionConfig.DEFAULT_PASSWORD) + && Objects.nonNull(this.encryptedPassword)) { + throw new IllegalStateException( + "password and encryptedPassword are mutually exclusive; encryptedPassword is already set"); + } this.password = password; return this; } + public Builder encryptedPassword(final String encryptedPassword) { + if (Objects.isNull(encryptedPassword)) { + return this; + } + if (!Objects.equals(this.password, SessionConfig.DEFAULT_PASSWORD)) { + throw new IllegalStateException( + "password and encryptedPassword are mutually exclusive; password is already set"); + } + this.encryptedPassword = encryptedPassword; + return this; + } + public Builder consumerId(@Nullable final String consumerId) { if (Objects.isNull(consumerId)) { return this; diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java index 9f153efeb106c..25d9c5da09e69 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionProvider.java @@ -86,11 +86,15 @@ final class SubscriptionProvider extends SubscriptionSession { private final long heartbeatIntervalMs; private final int connectionTimeoutInMs; private int dataNodeId; + private final String username; + private final String password; + private final String encryptedPassword; SubscriptionProvider( final TEndPoint endPoint, final String username, final String password, + final String encryptedPassword, final String consumerId, final String consumerGroupId, final int thriftMaxFrameSize, @@ -101,6 +105,9 @@ final class SubscriptionProvider extends SubscriptionSession { this.endPoint = endPoint; this.consumerId = consumerId; this.consumerGroupId = consumerGroupId; + this.username = username; + this.password = password; + this.encryptedPassword = encryptedPassword; this.heartbeatIntervalMs = heartbeatIntervalMs; this.connectionTimeoutInMs = connectionTimeoutInMs; } @@ -149,6 +156,11 @@ synchronized void handshake() throws SubscriptionException, IoTDBConnectionExcep final Map consumerAttributes = new HashMap<>(); consumerAttributes.put(ConsumerConstant.CONSUMER_GROUP_ID_KEY, consumerGroupId); consumerAttributes.put(ConsumerConstant.CONSUMER_ID_KEY, consumerId); + consumerAttributes.put(ConsumerConstant.USERNAME_KEY, username); + consumerAttributes.put(ConsumerConstant.PASSWORD_KEY, password); + if (encryptedPassword != null) { + consumerAttributes.put(ConsumerConstant.ENCRYPTED_PASSWORD_KEY, encryptedPassword); + } consumerAttributes.put( ConsumerConstant.HEARTBEAT_INTERVAL_MS_KEY, String.valueOf(heartbeatIntervalMs)); consumerAttributes.put( diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPullConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPullConsumer.java index a77716fe02cdf..227843811b15b 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPullConsumer.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPullConsumer.java @@ -329,6 +329,12 @@ public Builder password(final String password) { return this; } + @Override + public Builder encryptedPassword(final String encryptedPassword) { + super.encryptedPassword(encryptedPassword); + return this; + } + @Override public Builder consumerId(final String consumerId) { super.consumerId(consumerId); diff --git a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java index 2a41327809025..b70633bef35a8 100644 --- a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java +++ b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/SubscriptionPushConsumer.java @@ -257,6 +257,12 @@ public Builder password(final String password) { return this; } + @Override + public Builder encryptedPassword(final String encryptedPassword) { + super.encryptedPassword(encryptedPassword); + return this; + } + @Override public Builder consumerId(final String consumerId) { super.consumerId(consumerId); diff --git a/iotdb-core/confignode/pom.xml b/iotdb-core/confignode/pom.xml index fb22fd5b091f7..a7cd4b6298b5f 100644 --- a/iotdb-core/confignode/pom.xml +++ b/iotdb-core/confignode/pom.xml @@ -137,10 +137,6 @@ org.apache.commons commons-lang3 - - commons-codec - commons-codec - org.apache.thrift libthrift diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncAINodeHeartbeatClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncAINodeHeartbeatClientPool.java index e09ccc79becbf..8ec455142b8ba 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncAINodeHeartbeatClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncAINodeHeartbeatClientPool.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.ainode.AsyncAINodeServiceClient; import org.apache.iotdb.confignode.client.async.handlers.heartbeat.AINodeHeartbeatHandler; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; public class AsyncAINodeHeartbeatClientPool { @@ -34,7 +35,8 @@ private AsyncAINodeHeartbeatClientPool() { clientManager = new IClientManager.Factory() .createClientManager( - new ClientPoolFactory.AsyncAINodeHeartbeatServiceClientPoolFactory()); + new ClientPoolFactory.AsyncAINodeHeartbeatServiceClientPoolFactory( + ConfigNodeDescriptor.getInstance().getConf().getSelectorNumOfClientManager())); } public void getAINodeHeartBeat( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncConfigNodeHeartbeatClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncConfigNodeHeartbeatClientPool.java index 7b6bca5d0d9e7..a6dffbe0eef63 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncConfigNodeHeartbeatClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncConfigNodeHeartbeatClientPool.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.async.AsyncConfigNodeInternalServiceClient; import org.apache.iotdb.confignode.client.async.handlers.heartbeat.ConfigNodeHeartbeatHandler; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeHeartbeatReq; public class AsyncConfigNodeHeartbeatClientPool { @@ -34,7 +35,8 @@ private AsyncConfigNodeHeartbeatClientPool() { clientManager = new IClientManager.Factory() .createClientManager( - new ClientPoolFactory.AsyncConfigNodeHeartbeatServiceClientPoolFactory()); + new ClientPoolFactory.AsyncConfigNodeHeartbeatServiceClientPoolFactory( + ConfigNodeDescriptor.getInstance().getConf().getSelectorNumOfClientManager())); } /** diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeHeartbeatClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeHeartbeatClientPool.java index 8d67d150efbbb..18a8120a9e459 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeHeartbeatClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeHeartbeatClientPool.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.async.AsyncDataNodeInternalServiceClient; import org.apache.iotdb.confignode.client.async.handlers.heartbeat.DataNodeHeartbeatHandler; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatReq; /** Asynchronously send RPC requests to DataNodes. See queryengine.thrift for more details. */ @@ -35,7 +36,8 @@ private AsyncDataNodeHeartbeatClientPool() { clientManager = new IClientManager.Factory() .createClientManager( - new ClientPoolFactory.AsyncDataNodeHeartbeatServiceClientPoolFactory()); + new ClientPoolFactory.AsyncDataNodeHeartbeatServiceClientPoolFactory( + ConfigNodeDescriptor.getInstance().getConf().getSelectorNumOfClientManager())); } /** diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToCnInternalServiceAsyncRequestManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToCnInternalServiceAsyncRequestManager.java index 19eaf9d9a40d2..00267e5a8ccef 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToCnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToCnInternalServiceAsyncRequestManager.java @@ -30,6 +30,7 @@ import org.apache.iotdb.confignode.client.async.handlers.rpc.ConfigNodeAsyncRequestRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.ConfigNodeTSStatusRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.SubmitTestConnectionTaskToConfigNodeRPCHandler; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +41,10 @@ public class CnToCnInternalServiceAsyncRequestManager private static final Logger LOGGER = LoggerFactory.getLogger(CnToCnInternalServiceAsyncRequestManager.class); + public CnToCnInternalServiceAsyncRequestManager(int selectorNumOfAsyncClientManager) { + super(selectorNumOfAsyncClientManager); + } + @Override protected void initActionMapBuilder() { actionMapBuilder.put( @@ -71,7 +76,8 @@ protected void adjustClientTimeoutIfNecessary( private static class ClientPoolHolder { private static final CnToCnInternalServiceAsyncRequestManager INSTANCE = - new CnToCnInternalServiceAsyncRequestManager(); + new CnToCnInternalServiceAsyncRequestManager( + ConfigNodeDescriptor.getInstance().getConf().getSelectorNumOfClientManager()); private ClientPoolHolder() { // Empty constructor diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java index b9aac92a16504..e4dcaad2bfb96 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java @@ -46,6 +46,7 @@ import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.CheckSchemaRegionUsingTemplateRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.ConsumerGroupPushMetaRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.TopicPushMetaRPCHandler; +import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.mpp.rpc.thrift.TActiveTriggerInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TAlterViewReq; import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateReq; @@ -105,6 +106,10 @@ public class CnToDnInternalServiceAsyncRequestManager private static final Logger LOGGER = LoggerFactory.getLogger(CnToDnInternalServiceAsyncRequestManager.class); + private CnToDnInternalServiceAsyncRequestManager() { + super(ConfigNodeDescriptor.getInstance().getConf().getSelectorNumOfClientManager()); + } + @SuppressWarnings("unchecked") @Override protected void initActionMapBuilder() { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/PipePushMetaRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/PipePushMetaRPCHandler.java index ffe15dd6466c5..0928c76cdc908 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/PipePushMetaRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/PipePushMetaRPCHandler.java @@ -67,15 +67,15 @@ public void onComplete(TPushPipeMetaResp response) { } @Override - public void onError(Exception e) { - String errorMsg = + public void onError(final Exception e) { + final String errorMsg = "Failed to " + requestType + " on DataNode: " + formattedTargetLocation + ", exception: " + e.getMessage(); - LOGGER.error(errorMsg, e); + LOGGER.warn(errorMsg); responseMap.put( requestId, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 03d12ee46a696..77b353ea59d4d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -136,6 +136,17 @@ public class ConfigNodeConfig { */ private int maxClientNumForEachNode = DefaultProperty.MAX_CLIENT_NUM_FOR_EACH_NODE; + private int maxIdleClientNumForEachNode = DefaultProperty.MAX_IDLE_CLIENT_NUM_FOR_EACH_NODE; + + /** + * ClientManager will have so many selector threads (TAsyncClientManager) to distribute to its + * clients. + */ + private int selectorNumOfClientManager = + Runtime.getRuntime().availableProcessors() / 4 > 0 + ? Runtime.getRuntime().availableProcessors() / 4 + : 1; + /** System directory, including version file for each database and metadata. */ private String systemDir = IoTDBConstant.CN_DEFAULT_DATA_DIR + File.separator + IoTDBConstant.SYSTEM_FOLDER_NAME; @@ -448,6 +459,23 @@ public ConfigNodeConfig setMaxClientNumForEachNode(int maxClientNumForEachNode) return this; } + public int getMaxIdleClientNumForEachNode() { + return maxIdleClientNumForEachNode; + } + + public ConfigNodeConfig setMaxIdleClientNumForEachNode(int maxIdleClientNumForEachNode) { + this.maxIdleClientNumForEachNode = maxIdleClientNumForEachNode; + return this; + } + + public int getSelectorNumOfClientManager() { + return selectorNumOfClientManager; + } + + public void setSelectorNumOfClientManager(int selectorNumOfClientManager) { + this.selectorNumOfClientManager = selectorNumOfClientManager; + } + public String getConsensusDir() { return consensusDir; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java index 614bae210dfa1..83cf1b612d01d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java @@ -299,6 +299,24 @@ private void loadProperties(TrimProperties properties) throws BadNodeUrlExceptio String.valueOf(conf.getMaxClientNumForEachNode())) .trim())); + int cnMaxIdleClientNumForEachNode = + Integer.parseInt( + properties.getProperty( + "cn_max_idle_client_count_for_each_node_in_client_manager", + String.valueOf(conf.getMaxIdleClientNumForEachNode()))); + if (cnMaxIdleClientNumForEachNode >= 0) { + conf.setMaxIdleClientNumForEachNode(cnMaxIdleClientNumForEachNode); + } + + int cnSelectorNumOfClientManager = + Integer.parseInt( + properties.getProperty( + "cn_selector_thread_nums_of_client_manager", + String.valueOf(conf.getSelectorNumOfClientManager()))); + if (cnSelectorNumOfClientManager > 0) { + conf.setSelectorNumOfClientManager(cnSelectorNumOfClientManager); + } + conf.setSystemDir(properties.getProperty("cn_system_dir", conf.getSystemDir()).trim()); conf.setConsensusDir(properties.getProperty("cn_consensus_dir", conf.getConsensusDir()).trim()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index b7a0936ca450f..06c10c3972395 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -73,6 +73,7 @@ import org.apache.iotdb.confignode.consensus.request.write.pipe.task.DropPipePlanV2; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.OperateMultiplePipesPlanV2; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusPlanV2; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusWithStoppedByRuntimeExceptionPlanV2; import org.apache.iotdb.confignode.consensus.request.write.procedure.DeleteProcedurePlan; import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan; import org.apache.iotdb.confignode.consensus.request.write.quota.SetSpaceQuotaPlan; @@ -354,6 +355,9 @@ public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOExcept case SetPipeStatusV2: plan = new SetPipeStatusPlanV2(); break; + case SetPipeStatusWithStoppedByRuntimeExceptionV2: + plan = new SetPipeStatusWithStoppedByRuntimeExceptionPlanV2(); + break; case DropPipeV2: plan = new DropPipePlanV2(); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 24e5b9ba9c631..b986299596675 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -217,6 +217,7 @@ public enum ConfigPhysicalPlanType { ShowPipeV2((short) 1503), AlterPipeV2((short) 1504), OperateMultiplePipesV2((short) 1505), + SetPipeStatusWithStoppedByRuntimeExceptionV2((short) 1506), /** Pipe Runtime. */ PipeHandleLeaderChange((short) 1600), diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/cq/ShowCQPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/cq/ShowCQPlan.java index 5217849deb488..c28838d556b48 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/cq/ShowCQPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/cq/ShowCQPlan.java @@ -21,11 +21,24 @@ import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; +import java.util.Optional; + import static org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType.SHOW_CQ; public class ShowCQPlan extends ConfigPhysicalReadPlan { + private final String cqId; + public ShowCQPlan() { + this(null); + } + + public ShowCQPlan(String cqId) { super(SHOW_CQ); + this.cqId = cqId; + } + + public Optional getCqId() { + return Optional.ofNullable(cqId); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/ActiveCQPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/ActiveCQPlan.java index e488ac2566921..4ca33b054a93f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/ActiveCQPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/ActiveCQPlan.java @@ -35,39 +35,39 @@ public class ActiveCQPlan extends ConfigPhysicalPlan { private String cqId; - private String md5; + private String cqToken; public ActiveCQPlan() { super(ACTIVE_CQ); } - public ActiveCQPlan(String cqId, String md5) { + public ActiveCQPlan(String cqId, String cqToken) { super(ACTIVE_CQ); Validate.notNull(cqId); - Validate.notNull(md5); + Validate.notNull(cqToken); this.cqId = cqId; - this.md5 = md5; + this.cqToken = cqToken; } public String getCqId() { return cqId; } - public String getMd5() { - return md5; + public String getCqToken() { + return cqToken; } @Override protected void serializeImpl(DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); ReadWriteIOUtils.write(cqId, stream); - ReadWriteIOUtils.write(md5, stream); + ReadWriteIOUtils.write(cqToken, stream); } @Override protected void deserializeImpl(ByteBuffer buffer) throws IOException { cqId = ReadWriteIOUtils.readString(buffer); - md5 = ReadWriteIOUtils.readString(buffer); + cqToken = ReadWriteIOUtils.readString(buffer); } @Override @@ -82,11 +82,11 @@ public boolean equals(Object o) { return false; } ActiveCQPlan that = (ActiveCQPlan) o; - return cqId.equals(that.cqId) && md5.equals(that.md5); + return cqId.equals(that.cqId) && cqToken.equals(that.cqToken); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), cqId, md5); + return Objects.hash(super.hashCode(), cqId, cqToken); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/AddCQPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/AddCQPlan.java index 0aae9e2e974ca..721c83d3d2f76 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/AddCQPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/AddCQPlan.java @@ -37,7 +37,7 @@ public class AddCQPlan extends ConfigPhysicalPlan { private TCreateCQReq req; - private String md5; + private String cqToken; private long firstExecutionTime; @@ -45,12 +45,12 @@ public AddCQPlan() { super(ADD_CQ); } - public AddCQPlan(TCreateCQReq req, String md5, long firstExecutionTime) { + public AddCQPlan(TCreateCQReq req, String cqToken, long firstExecutionTime) { super(ADD_CQ); Validate.notNull(req); - Validate.notNull(md5); + Validate.notNull(cqToken); this.req = req; - this.md5 = md5; + this.cqToken = cqToken; this.firstExecutionTime = firstExecutionTime; } @@ -58,8 +58,8 @@ public TCreateCQReq getReq() { return req; } - public String getMd5() { - return md5; + public String getCqToken() { + return cqToken; } public long getFirstExecutionTime() { @@ -70,14 +70,14 @@ public long getFirstExecutionTime() { protected void serializeImpl(DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); ThriftCommonsSerDeUtils.serializeTCreateCQReq(req, stream); - ReadWriteIOUtils.write(md5, stream); + ReadWriteIOUtils.write(cqToken, stream); ReadWriteIOUtils.write(firstExecutionTime, stream); } @Override protected void deserializeImpl(ByteBuffer buffer) throws IOException { req = ThriftCommonsSerDeUtils.deserializeTCreateCQReq(buffer); - md5 = ReadWriteIOUtils.readString(buffer); + cqToken = ReadWriteIOUtils.readString(buffer); firstExecutionTime = ReadWriteIOUtils.readLong(buffer); } @@ -95,11 +95,11 @@ public boolean equals(Object o) { AddCQPlan addCQPlan = (AddCQPlan) o; return firstExecutionTime == addCQPlan.firstExecutionTime && Objects.equals(req, addCQPlan.req) - && Objects.equals(md5, addCQPlan.md5); + && Objects.equals(cqToken, addCQPlan.cqToken); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), req, md5, firstExecutionTime); + return Objects.hash(super.hashCode(), req, cqToken, firstExecutionTime); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/DropCQPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/DropCQPlan.java index 5c901362997f8..f22561c0b807d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/DropCQPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/DropCQPlan.java @@ -37,7 +37,7 @@ public class DropCQPlan extends ConfigPhysicalPlan { private String cqId; // may be null in user call of drop CQ - private String md5; + private String cqToken; public DropCQPlan() { super(DROP_CQ); @@ -49,33 +49,33 @@ public DropCQPlan(String cqId) { this.cqId = cqId; } - public DropCQPlan(String cqId, String md5) { + public DropCQPlan(String cqId, String cqToken) { super(DROP_CQ); Validate.notNull(cqId); - Validate.notNull(md5); + Validate.notNull(cqToken); this.cqId = cqId; - this.md5 = md5; + this.cqToken = cqToken; } public String getCqId() { return cqId; } - public Optional getMd5() { - return Optional.ofNullable(md5); + public Optional getCqToken() { + return Optional.ofNullable(cqToken); } @Override protected void serializeImpl(DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); ReadWriteIOUtils.write(cqId, stream); - ReadWriteIOUtils.write(md5, stream); + ReadWriteIOUtils.write(cqToken, stream); } @Override protected void deserializeImpl(ByteBuffer buffer) throws IOException { cqId = ReadWriteIOUtils.readString(buffer); - md5 = ReadWriteIOUtils.readString(buffer); + cqToken = ReadWriteIOUtils.readString(buffer); } @Override @@ -90,11 +90,11 @@ public boolean equals(Object o) { return false; } DropCQPlan that = (DropCQPlan) o; - return cqId.equals(that.cqId) && Objects.equals(md5, that.md5); + return cqId.equals(that.cqId) && Objects.equals(cqToken, that.cqToken); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), cqId, md5); + return Objects.hash(super.hashCode(), cqId, cqToken); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/UpdateCQLastExecTimePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/UpdateCQLastExecTimePlan.java index 51a8358a04c28..392f6006ba56e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/UpdateCQLastExecTimePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/cq/UpdateCQLastExecTimePlan.java @@ -37,20 +37,19 @@ public class UpdateCQLastExecTimePlan extends ConfigPhysicalPlan { private long executionTime; - // may be null in user call of drop CQ - private String md5; + private String cqToken; public UpdateCQLastExecTimePlan() { super(UPDATE_CQ_LAST_EXEC_TIME); } - public UpdateCQLastExecTimePlan(String cqId, long executionTime, String md5) { + public UpdateCQLastExecTimePlan(String cqId, long executionTime, String cqToken) { super(UPDATE_CQ_LAST_EXEC_TIME); Validate.notNull(cqId); - Validate.notNull(md5); + Validate.notNull(cqToken); this.cqId = cqId; this.executionTime = executionTime; - this.md5 = md5; + this.cqToken = cqToken; } public String getCqId() { @@ -61,8 +60,8 @@ public long getExecutionTime() { return executionTime; } - public String getMd5() { - return md5; + public String getCqToken() { + return cqToken; } @Override @@ -70,14 +69,14 @@ protected void serializeImpl(DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); ReadWriteIOUtils.write(cqId, stream); ReadWriteIOUtils.write(executionTime, stream); - ReadWriteIOUtils.write(md5, stream); + ReadWriteIOUtils.write(cqToken, stream); } @Override protected void deserializeImpl(ByteBuffer buffer) throws IOException { cqId = ReadWriteIOUtils.readString(buffer); executionTime = ReadWriteIOUtils.readLong(buffer); - md5 = ReadWriteIOUtils.readString(buffer); + cqToken = ReadWriteIOUtils.readString(buffer); } @Override @@ -92,11 +91,13 @@ public boolean equals(Object o) { return false; } UpdateCQLastExecTimePlan that = (UpdateCQLastExecTimePlan) o; - return executionTime == that.executionTime && cqId.equals(that.cqId) && md5.equals(that.md5); + return executionTime == that.executionTime + && cqId.equals(that.cqId) + && cqToken.equals(that.cqToken); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), cqId, executionTime, md5); + return Objects.hash(super.hashCode(), cqId, executionTime, cqToken); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/SetPipeStatusWithStoppedByRuntimeExceptionPlanV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/SetPipeStatusWithStoppedByRuntimeExceptionPlanV2.java new file mode 100644 index 0000000000000..35ee503d5c5fc --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/task/SetPipeStatusWithStoppedByRuntimeExceptionPlanV2.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.pipe.task; + +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class SetPipeStatusWithStoppedByRuntimeExceptionPlanV2 extends ConfigPhysicalPlan { + + private String pipeName; + private PipeStatus status; + private boolean stoppedByRuntimeException; + + public SetPipeStatusWithStoppedByRuntimeExceptionPlanV2() { + super(ConfigPhysicalPlanType.SetPipeStatusWithStoppedByRuntimeExceptionV2); + } + + public SetPipeStatusWithStoppedByRuntimeExceptionPlanV2( + final String pipeName, final PipeStatus status, final boolean stoppedByRuntimeException) { + super(ConfigPhysicalPlanType.SetPipeStatusWithStoppedByRuntimeExceptionV2); + this.pipeName = pipeName; + this.status = status; + this.stoppedByRuntimeException = stoppedByRuntimeException; + } + + public String getPipeName() { + return pipeName; + } + + public PipeStatus getPipeStatus() { + return status; + } + + public boolean isStoppedByRuntimeException() { + return stoppedByRuntimeException; + } + + @Override + protected void serializeImpl(final DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(pipeName, stream); + ReadWriteIOUtils.write(status.getType(), stream); + ReadWriteIOUtils.write(stoppedByRuntimeException, stream); + } + + @Override + protected void deserializeImpl(final ByteBuffer buffer) throws IOException { + pipeName = ReadWriteIOUtils.readString(buffer); + status = PipeStatus.getPipeStatus(ReadWriteIOUtils.readByte(buffer)); + stoppedByRuntimeException = ReadWriteIOUtils.readBool(buffer); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final SetPipeStatusWithStoppedByRuntimeExceptionPlanV2 that = + (SetPipeStatusWithStoppedByRuntimeExceptionPlanV2) obj; + return stoppedByRuntimeException == that.stoppedByRuntimeException + && pipeName.equals(that.pipeName) + && status.equals(that.status); + } + + @Override + public int hashCode() { + return Objects.hash(pipeName, status, stoppedByRuntimeException); + } + + @Override + public String toString() { + return "SetPipeStatusWithStoppedByRuntimeExceptionPlanV2{" + + "pipeName='" + + pipeName + + "', status='" + + status + + "', stoppedByRuntimeException='" + + stoppedByRuntimeException + + "'}"; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java index 9592692d1bc45..b4f31d84023fa 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java @@ -110,6 +110,15 @@ public TPermissionInfoResp login(String username, String password) { return authorInfo.login(username, password); } + public TPermissionInfoResp login( + final String username, final String password, final boolean useEncryptedPassword) { + return authorInfo.login(username, password, useEncryptedPassword); + } + + public String login4Pipe(final String userName, final String password) { + return authorInfo.login4Pipe(userName, password); + } + public TPermissionInfoResp checkUserPrivileges( String username, List paths, int permission) { return authorInfo.checkUserPrivileges(username, paths, permission); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index f0f1b9e6bfbff..4912f2dad5c8b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -1443,7 +1443,7 @@ public void pipeHandleLeaderChange( } } - public void pipeHandleMetaChange( + public boolean pipeHandleMetaChange( boolean needWriteConsensusOnConfigNodes, boolean needPushPipeMetaToDataNodes) { try { final long procedureId = @@ -1451,8 +1451,10 @@ public void pipeHandleMetaChange( new PipeHandleMetaChangeProcedure( needWriteConsensusOnConfigNodes, needPushPipeMetaToDataNodes)); LOGGER.info("PipeHandleMetaChangeProcedure was submitted, procedureId: {}.", procedureId); + return true; } catch (Exception e) { LOGGER.warn("PipeHandleMetaChangeProcedure was failed to submit.", e); + return false; } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java index 4cf14ac4cba75..c4d57c2f85e30 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java @@ -125,6 +125,10 @@ public Map getAllTTL() { return ((ShowTTLResp) showTTL(new ShowTTLPlan())).getPathTTLMap(); } + public long getTTL(final String[] pathPattern) { + return ttlInfo.getTTL(pathPattern); + } + public int getTTLCount() { return ttlInfo.getTTLCount(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java index 5726b3ce82698..a90a21f285ff2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQManager.java @@ -42,7 +42,10 @@ import java.util.Collections; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -56,11 +59,15 @@ public class CQManager { private final ReadWriteLock lock; + // Key: CQ id. Value: the local task and the metadata token it owns. + private final ConcurrentMap locallyScheduledCQs; + private ScheduledExecutorService executor; public CQManager(ConfigManager configManager) { this.configManager = configManager; this.lock = new ReentrantReadWriteLock(); + this.locallyScheduledCQs = new ConcurrentHashMap<>(); this.executor = IoTDBThreadPoolFactory.newScheduledThreadPool( CONF.getCqSubmitThread(), ThreadName.CQ_SCHEDULER.getName()); @@ -77,14 +84,21 @@ public TSStatus createCQ(TCreateCQReq req) { } public TSStatus dropCQ(TDropCQReq req) { + lock.readLock().lock(); try { - return configManager.getConsensusManager().write(new DropCQPlan(req.cqId)); + TSStatus status = configManager.getConsensusManager().write(new DropCQPlan(req.cqId)); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + cancelLocallyScheduledCQ(req.cqId); + } + return status; } catch (ConsensusException e) { LOGGER.warn("Unexpected error happened while dropping cq {}: ", req.cqId, e); // consensus layer related errors TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); res.setMessage(e.getMessage()); return res; + } finally { + lock.readLock().unlock(); } } @@ -117,6 +131,7 @@ public void startCQScheduler() { try { // 1. shutdown previous cq schedule thread pool try { + cancelAllLocallyScheduledCQs(); if (executor != null) { executor.shutdown(); } @@ -154,7 +169,15 @@ public void startCQScheduler() { for (CQInfo.CQEntry entry : allCQs) { if (entry.getState() == CQState.ACTIVE) { CQScheduleTask cqScheduleTask = new CQScheduleTask(entry, executor, configManager); - cqScheduleTask.submitSelf(); + if (!markCQLocallyScheduled(entry.getCqId(), entry.getCqToken(), cqScheduleTask)) { + continue; + } + try { + cqScheduleTask.submitSelf(); + } catch (RuntimeException e) { + unmarkCQLocallyScheduled(entry.getCqId(), entry.getCqToken()); + throw e; + } } } } @@ -174,6 +197,7 @@ public void stopCQScheduler() { try { previous = executor; executor = null; + cancelAllLocallyScheduledCQs(); } finally { lock.writeLock().unlock(); } @@ -181,4 +205,78 @@ public void stopCQScheduler() { previous.shutdown(); } } + + public boolean markCQLocallyScheduled(String cqId, String cqToken, CQScheduleTask task) { + AtomicBoolean shouldSchedule = new AtomicBoolean(false); + LocallyScheduledCQ schedule = new LocallyScheduledCQ(cqToken, task); + lock.readLock().lock(); + try { + locallyScheduledCQs.compute( + cqId, + (ignored, previousSchedule) -> { + if (previousSchedule != null && previousSchedule.hasToken(cqToken)) { + return previousSchedule; + } + if (previousSchedule != null) { + previousSchedule.cancel(); + } + shouldSchedule.set(true); + return schedule; + }); + if (!shouldSchedule.get()) { + task.cancel(); + } + return shouldSchedule.get(); + } finally { + lock.readLock().unlock(); + } + } + + public void unmarkCQLocallyScheduled(String cqId, String cqToken) { + lock.readLock().lock(); + try { + locallyScheduledCQs.computeIfPresent( + cqId, + (ignored, schedule) -> { + if (schedule.hasToken(cqToken)) { + schedule.cancel(); + return null; + } + return schedule; + }); + } finally { + lock.readLock().unlock(); + } + } + + private void cancelLocallyScheduledCQ(String cqId) { + LocallyScheduledCQ schedule = locallyScheduledCQs.remove(cqId); + if (schedule != null) { + schedule.cancel(); + } + } + + private void cancelAllLocallyScheduledCQs() { + locallyScheduledCQs.values().forEach(LocallyScheduledCQ::cancel); + locallyScheduledCQs.clear(); + } + + private static class LocallyScheduledCQ { + + private final String cqToken; + private final CQScheduleTask task; + + private LocallyScheduledCQ(String cqToken, CQScheduleTask task) { + this.cqToken = cqToken; + this.task = task; + } + + private boolean hasToken(String cqToken) { + return this.cqToken.equals(cqToken); + } + + private void cancel() { + task.cancel(); + } + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java index 6125edc1ef7bd..d85d0f12a3040 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/cq/CQScheduleTask.java @@ -39,7 +39,10 @@ import java.util.Optional; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; public class CQScheduleTask implements Runnable { @@ -69,7 +72,7 @@ public class CQScheduleTask implements Runnable { private final long endTimeOffset; private final TimeoutPolicy timeoutPolicy; private final String queryBody; - private final String md5; + private final String cqToken; private final String zoneId; @@ -81,12 +84,15 @@ public class CQScheduleTask implements Runnable { private final long retryWaitTimeInMS; + private final AtomicBoolean cancelled; + private final AtomicReference> scheduledFuture; + private long executionTime; public CQScheduleTask( TCreateCQReq req, long firstExecutionTime, - String md5, + String cqToken, ScheduledExecutorService executor, ConfigManager configManager) { this( @@ -96,7 +102,7 @@ public CQScheduleTask( req.endTimeOffset, TimeoutPolicy.deserialize(req.timeoutPolicy), req.queryBody, - md5, + cqToken, req.zoneId, req.username, executor, @@ -113,7 +119,7 @@ public CQScheduleTask( entry.getEndTimeOffset(), entry.getTimeoutPolicy(), entry.getQueryBody(), - entry.getMd5(), + entry.getCqToken(), entry.getZoneId(), entry.getUsername(), executor, @@ -129,7 +135,7 @@ public CQScheduleTask( long endTimeOffset, TimeoutPolicy timeoutPolicy, String queryBody, - String md5, + String cqToken, String zoneId, String username, ScheduledExecutorService executor, @@ -141,12 +147,14 @@ public CQScheduleTask( this.endTimeOffset = endTimeOffset; this.timeoutPolicy = timeoutPolicy; this.queryBody = queryBody; - this.md5 = md5; + this.cqToken = cqToken; this.zoneId = zoneId; this.username = username; this.executor = executor; this.configManager = configManager; this.retryWaitTimeInMS = Math.min(DEFAULT_RETRY_WAIT_TIME_IN_MS, everyInterval / FACTOR); + this.cancelled = new AtomicBoolean(false); + this.scheduledFuture = new AtomicReference<>(); this.executionTime = executionTime; } @@ -165,6 +173,9 @@ public static long getFirstExecutionTime(long boundaryTime, long everyInterval, @Override public void run() { + if (cancelled.get()) { + return; + } long startTime = executionTime - startTimeOffset; long endTime = executionTime - endTimeOffset; @@ -177,6 +188,9 @@ public void run() { submitSelf(retryWaitTimeInMS, TimeUnit.MILLISECONDS); } } else { + if (cancelled.get()) { + return; + } LOGGER.info( "[StartExecuteCQ] execute CQ {} on DataNode[{}], time range is [{}, {}), current time is {}", cqId, @@ -206,12 +220,32 @@ public void submitSelf() { } private void submitSelf(long delay, TimeUnit unit) { - executor.schedule(this, delay, unit); + if (cancelled.get()) { + return; + } + ScheduledFuture newFuture = executor.schedule(this, delay, unit); + ScheduledFuture previousFuture = scheduledFuture.getAndSet(newFuture); + if (previousFuture != null) { + previousFuture.cancel(false); + } + if (cancelled.get() && scheduledFuture.compareAndSet(newFuture, null)) { + newFuture.cancel(false); + } + } + + public void cancel() { + cancelled.set(true); + ScheduledFuture currentFuture = scheduledFuture.getAndSet(null); + if (currentFuture != null) { + currentFuture.cancel(false); + } } private boolean needSubmit() { // current node is still leader and thread pool is not shut down. - return configManager.getConsensusManager().isLeader() && !executor.isShutdown(); + return !cancelled.get() + && configManager.getConsensusManager().isLeader() + && !executor.isShutdown(); } private class AsyncExecuteCQCallback implements AsyncMethodCallback { @@ -238,6 +272,9 @@ private void updateExecutionTime() { @Override public void onComplete(TSStatus response) { + if (cancelled.get()) { + return; + } if (response.code == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.info( @@ -251,7 +288,7 @@ public void onComplete(TSStatus response) { result = configManager .getConsensusManager() - .write(new UpdateCQLastExecTimePlan(cqId, executionTime, md5)); + .write(new UpdateCQLastExecTimePlan(cqId, executionTime, cqToken)); } catch (ConsensusException e) { result = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); result.setMessage(e.getMessage()); @@ -291,6 +328,9 @@ public void onComplete(TSStatus response) { @Override public void onError(Exception exception) { + if (cancelled.get()) { + return; + } LOGGER.warn("Execute CQ {} failed", cqId, exception); if (needSubmit()) { submitSelf(retryWaitTimeInMS, TimeUnit.MILLISECONDS); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/service/EventService.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/service/EventService.java index 5f6035f0bcab9..8267227c7ce41 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/service/EventService.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/service/EventService.java @@ -114,6 +114,11 @@ public void stopEventService() { currentEventServiceFuture = null; LOGGER.info("Event service is stopped successfully."); } + synchronized (this) { + previousNodeStatisticsMap.clear(); + previousRegionGroupStatisticsMap.clear(); + previousConsensusGroupStatisticsMap.clear(); + } } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 2e0ac35d5df98..81cad2f3befdf 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -930,7 +930,7 @@ public List setConfiguration(TSetConfigurationReq req) { if (!targetDataNodes.isEmpty()) { DataNodeAsyncRequestContext clientHandler = new DataNodeAsyncRequestContext<>( - CnToDnAsyncRequestType.SET_CONFIGURATION, req, dataNodeLocationMap); + CnToDnAsyncRequestType.SET_CONFIGURATION, req, targetDataNodes); CnToDnInternalServiceAsyncRequestManager.getInstance() .sendAsyncRequestWithRetry(clientHandler); responseList.addAll(clientHandler.getResponseList()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java index ace07f5e2d3d3..8cafa2094c446 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParser.java @@ -59,7 +59,7 @@ public class PipeHeartbeatParser { this.configManager = configManager; heartbeatCounter = 0; - registeredNodeNumber = 1; + registeredNodeNumber = getExpectedHeartbeatNodeCount(); needWriteConsensusOnConfigNodes = new AtomicBoolean(false); needPushPipeMetaToDataNodes = new AtomicBoolean(false); @@ -73,17 +73,8 @@ synchronized void parseHeartbeat(final int nodeId, final PipeHeartbeat pipeHeart if (heartbeatCount % registeredNodeNumber == 0) { canSubmitHandleMetaChangeProcedure.set(true); - // registeredNodeNumber may be changed, update it here when we can submit procedure - registeredNodeNumber = configManager.getNodeManager().getRegisteredNodeCount(); - if (registeredNodeNumber <= 0) { - LOGGER.warn( - "registeredNodeNumber is {} when parseHeartbeat from node (id={}).", - registeredNodeNumber, - nodeId); - // registeredNodeNumber can not be set to 0 in this class, otherwise may cause - // DivideByZeroException - registeredNodeNumber = 1; - } + // The expected reporter set may be changed, update it at the end of the current round. + registeredNodeNumber = getExpectedHeartbeatNodeCount(); } if (pipeHeartbeat.isEmpty() @@ -114,14 +105,14 @@ synchronized void parseHeartbeat(final int nodeId, final PipeHeartbeat pipeHeart if (canSubmitHandleMetaChangeProcedure.get() && (needWriteConsensusOnConfigNodes.get() || needPushPipeMetaToDataNodes.get())) { - configManager + if (configManager .getProcedureManager() .pipeHandleMetaChange( - needWriteConsensusOnConfigNodes.get(), needPushPipeMetaToDataNodes.get()); - - // Reset flags after procedure is submitted - needWriteConsensusOnConfigNodes.set(false); - needPushPipeMetaToDataNodes.set(false); + needWriteConsensusOnConfigNodes.get(), + needPushPipeMetaToDataNodes.get())) { + needWriteConsensusOnConfigNodes.set(false); + needPushPipeMetaToDataNodes.set(false); + } } } finally { configManager.getPipeManager().getPipeTaskCoordinator().unlock(); @@ -129,6 +120,17 @@ synchronized void parseHeartbeat(final int nodeId, final PipeHeartbeat pipeHeart }); } + private int getExpectedHeartbeatNodeCount() { + final int expectedNodeCount = + configManager.getNodeManager().getRegisteredDataNodeCount() + + (PipeConfig.getInstance().isSeperatedPipeHeartbeatEnabled() ? 1 : 0); + if (expectedNodeCount <= 0) { + LOGGER.warn("Expected pipe heartbeat node count is {}, fallback to 1.", expectedNodeCount); + return 1; + } + return expectedNodeCount; + } + private void parseHeartbeatAndSaveMetaChangeLocally( final AtomicReference pipeTaskInfo, final int nodeId, @@ -152,11 +154,24 @@ private void parseHeartbeatAndSaveMetaChangeLocally( if (Boolean.TRUE.equals(isPipeCompletedFromAgent)) { temporaryMeta.markDataNodeCompleted(nodeId); + LOGGER.info( + "Detected historical pipe completion report from DataNode {} for pipe {}. remainingEventCount: {}, remainingTime: {}, completedDataNodes: {}", + nodeId, + staticMeta.getPipeName(), + pipeHeartbeat.getRemainingEventCount(staticMeta), + pipeHeartbeat.getRemainingTime(staticMeta), + temporaryMeta.getCompletedDataNodeIds()); final Set uncompletedDataNodeIds = configManager.getNodeManager().getRegisteredDataNodeLocations().keySet(); uncompletedDataNodeIds.removeAll(temporaryMeta.getCompletedDataNodeIds()); if (uncompletedDataNodeIds.isEmpty()) { + LOGGER.info( + "All DataNodes reported historical pipe {} completed. globalRemainingEventCount: {}, globalRemainingTime: {}, staticMeta: {}", + staticMeta.getPipeName(), + temporaryMeta.getGlobalRemainingEvents(), + temporaryMeta.getGlobalRemainingTime(), + staticMeta); pipeTaskInfo.get().removePipeMeta(staticMeta.getPipeName()); LOGGER.info( "Detected completion of pipe {}, static meta: {}, remove it.", diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java index ee1ccd2a8fba4..f3430db3932fe 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java @@ -120,19 +120,8 @@ public TSStatus startPipe(String pipeName) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus stopPipe(String pipeName) { - final boolean isStoppedByRuntimeException = pipeTaskInfo.isStoppedByRuntimeException(pipeName); final TSStatus status = configManager.getProcedureManager().stopPipe(pipeName); - if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - if (isStoppedByRuntimeException) { - // Even if the return status is success, it doesn't imply the success of the - // `executeFromOperateOnDataNodes` phase of stopping pipe. However, we still need to set - // `isStoppedByRuntimeException` to false to avoid auto-restart. Meanwhile, - // `isStoppedByRuntimeException` does not need to be synchronized with DNs. - LOGGER.info("Pipe {} has stopped manually, stop its auto restart process.", pipeName); - pipeTaskInfo.setIsStoppedByRuntimeExceptionToFalse(pipeName); - configManager.getProcedureManager().pipeHandleMetaChange(true, false); - } - } else { + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn("Failed to stop pipe {}. Result status: {}.", pipeName, status); } return status; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinatorLock.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinatorLock.java index b86c556f20df8..58347f35af5a6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinatorLock.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinatorLock.java @@ -43,16 +43,9 @@ public class PipeTaskCoordinatorLock { public void lock() { LOGGER.debug( "PipeTaskCoordinator lock waiting for thread {}", Thread.currentThread().getName()); - try { - semaphore.acquire(); - LOGGER.debug( - "PipeTaskCoordinator lock acquired by thread {}", Thread.currentThread().getName()); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - LOGGER.error( - "Interrupted while waiting for PipeTaskCoordinator lock, current thread: {}", - Thread.currentThread().getName()); - } + semaphore.acquireUninterruptibly(); + LOGGER.debug( + "PipeTaskCoordinator lock acquired by thread {}", Thread.currentThread().getName()); } public boolean tryLock() { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/overview/PipeConfigNodeRemainingTimeOperator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/overview/PipeConfigNodeRemainingTimeOperator.java index c27e4cd898b06..ff9a7ea2648d5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/overview/PipeConfigNodeRemainingTimeOperator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/overview/PipeConfigNodeRemainingTimeOperator.java @@ -66,14 +66,11 @@ class PipeConfigNodeRemainingTimeOperator extends PipeRemainingOperator { .reduce(Long::sum) .orElse(0L); - configRegionCommitMeter.updateAndGet( - meter -> { - if (Objects.nonNull(meter)) { - lastConfigRegionCommitSmoothingValue = - pipeRemainingTimeCommitRateAverageTime.getMeterRate(meter); - } - return meter; - }); + final Meter configRegionMeter = configRegionCommitMeter.get(); + if (Objects.nonNull(configRegionMeter)) { + lastConfigRegionCommitSmoothingValue = + pipeRemainingTimeCommitRateAverageTime.getMeterRate(configRegionMeter); + } final double configRegionRemainingTime; if (totalConfigRegionWriteEventCount <= 0) { @@ -101,13 +98,10 @@ void register(final IoTDBConfigRegionSource extractor) { //////////////////////////// Rate //////////////////////////// void markConfigRegionCommit() { - configRegionCommitMeter.updateAndGet( - meter -> { - if (Objects.nonNull(meter)) { - meter.mark(); - } - return meter; - }); + final Meter meter = configRegionCommitMeter.get(); + if (Objects.nonNull(meter)) { + meter.mark(); + } } //////////////////////////// Switch //////////////////////////// diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionSink.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionSink.java index 5ee983d7945bd..b5f62ef4ccf48 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionSink.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionSink.java @@ -88,7 +88,7 @@ protected IoTDBSyncClientManager constructClient( protected PipeTransferFilePieceReq getTransferSingleFilePieceReq( final String fileName, final long position, final byte[] payLoad) { throw new UnsupportedOperationException( - "The config region connector does not support transferring single file piece req."); + "The config region sink does not support transferring single file piece req."); } @Override @@ -105,13 +105,13 @@ protected void mayLimitRateAndRecordIO(final long requiredBytes) { @Override public void transfer(final TabletInsertionEvent tabletInsertionEvent) throws Exception { throw new UnsupportedOperationException( - "IoTDBConfigRegionConnector can't transfer TabletInsertionEvent."); + "IoTDBConfigRegionSink can't transfer TabletInsertionEvent."); } @Override public void transfer(final TsFileInsertionEvent tsFileInsertionEvent) throws Exception { throw new UnsupportedOperationException( - "IoTDBConfigRegionConnector can't transfer TsFileInsertionEvent."); + "IoTDBConfigRegionSink can't transfer TsFileInsertionEvent."); } @Override @@ -121,8 +121,7 @@ public void transfer(final Event event) throws Exception { } else if (event instanceof PipeConfigRegionSnapshotEvent) { doTransferWrapper((PipeConfigRegionSnapshotEvent) event); } else if (!(event instanceof PipeHeartbeatEvent)) { - LOGGER.warn( - "IoTDBConfigRegionConnector does not support transferring generic event: {}.", event); + LOGGER.warn("IoTDBConfigRegionSink does not support transferring generic event: {}.", event); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java index e9e1058d79ba6..3bd17b5b5b76f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java @@ -113,6 +113,58 @@ public TPermissionInfoResp login(String username, String password) { return result; } + public TPermissionInfoResp login( + final String username, final String password, final boolean useEncryptedPassword) { + if (!useEncryptedPassword) { + return login(username, password); + } + + boolean status = false; + String loginMessage = null; + TSStatus tsStatus = new TSStatus(); + TPermissionInfoResp result = new TPermissionInfoResp(); + try { + final User user = authorizer.getUser(username); + status = user != null && password != null && password.equals(user.getPassword()); + if (status) { + result = getUserPermissionInfo(username); + result.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Login successfully")); + } else { + result = AuthUtils.generateEmptyPermissionInfoResp(); + } + } catch (AuthException e) { + LOGGER.error("meet error while logging in.", e); + loginMessage = e.getMessage(); + } + if (!status) { + tsStatus.setMessage(loginMessage != null ? loginMessage : "Authentication failed."); + tsStatus.setCode(TSStatusCode.WRONG_LOGIN_PASSWORD.getStatusCode()); + result.setStatus(tsStatus); + } + return result; + } + + public String login4Pipe(final String username, final String password) { + try { + final User user = authorizer.getUser(username); + if (user == null) { + return null; + } + if (password == null) { + return user.getPassword(); + } + final TPermissionInfoResp loginResp = login(username, password); + if (loginResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + && loginResp.isSetUserInfo()) { + return loginResp.getUserInfo().getPassword(); + } + return null; + } catch (final AuthException e) { + LOGGER.error("meet error while logging in for pipe.", e); + return null; + } + } + // if All paths fail, return No permission // if some paths fail, return SUCCESS and failed index list // if all path success, return success and empty index list diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java index d0c6b6da0134d..566c2b0715a90 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/TTLInfo.java @@ -68,13 +68,15 @@ public TSStatus setTTL(SetTTLPlan plan) { try { // check ttl rule capacity final int tTlRuleCapacity = CommonDescriptor.getInstance().getConfig().getTTlRuleCapacity(); - if (getTTLCount() >= tTlRuleCapacity) { + final int newTTLRuleCount = calculateNewTTLRuleCount(plan); + final int requestedTTLRuleCount = ttlCache.getTtlCount() + newTTLRuleCount; + if (newTTLRuleCount > 0 && requestedTTLRuleCount > tTlRuleCapacity) { TSStatus errorStatus = new TSStatus(TSStatusCode.OVERSIZE_TTL.getStatusCode()); errorStatus.setMessage( String.format( - "The number of TTL rules has reached the limit (%d). Please delete " - + "some existing rules first.", - tTlRuleCapacity)); + "The number of TTL rules has reached the limit " + + "(capacity: %d, requested total: %d). Please delete some existing rules first.", + tTlRuleCapacity, requestedTTLRuleCount)); return errorStatus; } ttlCache.setTTL(plan.getPathPattern(), plan.getTTL()); @@ -90,6 +92,20 @@ public TSStatus setTTL(SetTTLPlan plan) { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + private int calculateNewTTLRuleCount(SetTTLPlan plan) { + int newTTLRuleCount = isNewTTLRule(plan.getPathPattern()) ? 1 : 0; + if (plan.isDataBase()) { + String[] pathNodes = Arrays.copyOf(plan.getPathPattern(), plan.getPathPattern().length + 1); + pathNodes[pathNodes.length - 1] = IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; + newTTLRuleCount += isNewTTLRule(pathNodes) ? 1 : 0; + } + return newTTLRuleCount; + } + + private boolean isNewTTLRule(String[] pathNodes) { + return ttlCache.getLastNodeTTL(pathNodes) == TTLCache.NULL_TTL; + } + /** Only used for upgrading from database level ttl to device level ttl. */ public void setTTL(Map databaseTTLMap) throws IllegalPathException { lock.writeLock().lock(); @@ -157,6 +173,15 @@ public int getTTLCount() { } } + public long getTTL(final String[] pathPattern) { + lock.readLock().lock(); + try { + return ttlCache.getLastNodeTTL(pathPattern); + } finally { + lock.readLock().unlock(); + } + } + /** * Get the maximum ttl of the corresponding database level. * diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/UDFInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/UDFInfo.java index 1c7682dd35b7c..4b80a2625f0ef 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/UDFInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/UDFInfo.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.executable.ExecutableManager; +import org.apache.iotdb.commons.executable.ReferenceCountedJarMetaKeeper; import org.apache.iotdb.commons.snapshot.SnapshotProcessor; import org.apache.iotdb.commons.udf.UDFInformation; import org.apache.iotdb.commons.udf.UDFTable; @@ -37,7 +38,6 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.udf.api.exception.UDFManagementException; -import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +50,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReentrantLock; @@ -63,7 +62,7 @@ public class UDFInfo implements SnapshotProcessor { ConfigNodeDescriptor.getInstance().getConf(); private final UDFTable udfTable; - private final Map existedJarToMD5; + private final ReferenceCountedJarMetaKeeper jarMetaKeeper; private final UDFExecutableManager udfExecutableManager; @@ -73,7 +72,7 @@ public class UDFInfo implements SnapshotProcessor { public UDFInfo() throws IOException { udfTable = new UDFTable(); - existedJarToMD5 = new HashMap<>(); + jarMetaKeeper = new ReferenceCountedJarMetaKeeper(); udfExecutableManager = UDFExecutableManager.setupAndGetInstance( CONFIG_NODE_CONF.getUdfTemporaryLibDir(), CONFIG_NODE_CONF.getUdfDir()); @@ -97,7 +96,8 @@ public void validate(String udfName, String jarName, String jarMD5) String.format("Failed to create UDF [%s], the same name UDF has been created", udfName)); } - if (existedJarToMD5.containsKey(jarName) && !existedJarToMD5.get(jarName).equals(jarMD5)) { + if (jarMetaKeeper.containsJar(jarName) + && !jarMetaKeeper.jarNameExistsAndMatchesMd5(jarName, jarMD5)) { throw new UDFManagementException( String.format( "Failed to create UDF [%s], the same name Jar [%s] but different MD5 [%s] has existed", @@ -115,7 +115,7 @@ public void validate(String udfName) throws UDFManagementException { } public boolean needToSaveJar(String jarName) { - return !existedJarToMD5.containsKey(jarName); + return jarMetaKeeper.needToSaveJar(jarName); } public TSStatus addUDFInTable(CreateFunctionPlan physicalPlan) { @@ -123,7 +123,7 @@ public TSStatus addUDFInTable(CreateFunctionPlan physicalPlan) { final UDFInformation udfInformation = physicalPlan.getUdfInformation(); udfTable.addUDFInformation(udfInformation.getFunctionName(), udfInformation); if (udfInformation.isUsingURI()) { - existedJarToMD5.put(udfInformation.getJarName(), udfInformation.getJarMD5()); + addJarReference(udfInformation.getJarName(), udfInformation.getJarMD5()); if (physicalPlan.getJarFile() != null) { udfExecutableManager.saveToInstallDir( ByteBuffer.wrap(physicalPlan.getJarFile().getValues()), udfInformation.getJarName()); @@ -168,7 +168,10 @@ public JarResp getUDFJar(GetUDFJarPlan physicalPlan) { public TSStatus dropFunction(DropFunctionPlan req) { String udfName = req.getFunctionName(); if (udfTable.containsUDF(udfName)) { - existedJarToMD5.remove(udfTable.getUDFInformation(udfName).getJarName()); + final UDFInformation udfInformation = udfTable.getUDFInformation(udfName); + if (udfInformation.isUsingURI()) { + removeJarReference(udfInformation.getJarName()); + } udfTable.removeUDFInformation(udfName); } return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); @@ -181,7 +184,7 @@ public Map getRawUDFTable() { @TestOnly public Map getRawExistedJarToMD5() { - return existedJarToMD5; + return jarMetaKeeper.getJarNameToMd5Map(); } @Override @@ -225,30 +228,39 @@ public void processLoadSnapshot(File snapshotDir) throws IOException { deserializeExistedJarToMD5(fileInputStream); udfTable.deserializeUDFTable(fileInputStream); + rebuildJarMetadataFromUDFTable(); } finally { releaseUDFTableLock(); } } public void serializeExistedJarToMD5(OutputStream outputStream) throws IOException { - ReadWriteIOUtils.write(existedJarToMD5.size(), outputStream); - for (Map.Entry entry : existedJarToMD5.entrySet()) { - ReadWriteIOUtils.write(entry.getKey(), outputStream); - ReadWriteIOUtils.write(entry.getValue(), outputStream); - } + jarMetaKeeper.serializeJarNameToMd5(outputStream); } public void deserializeExistedJarToMD5(InputStream inputStream) throws IOException { - int size = ReadWriteIOUtils.readInt(inputStream); - while (size > 0) { - existedJarToMD5.put( - ReadWriteIOUtils.readString(inputStream), ReadWriteIOUtils.readString(inputStream)); - size--; - } + jarMetaKeeper.deserializeJarNameToMd5(inputStream); } public void clear() { - existedJarToMD5.clear(); + jarMetaKeeper.clear(); udfTable.clear(); } + + private void addJarReference(String jarName, String jarMD5) { + jarMetaKeeper.addReference(jarName, jarMD5); + } + + private void removeJarReference(String jarName) { + jarMetaKeeper.removeReference(jarName); + } + + private void rebuildJarMetadataFromUDFTable() { + jarMetaKeeper.clear(); + for (UDFInformation udfInformation : udfTable.getAllNonBuiltInUDFInformation()) { + if (udfInformation.isUsingURI()) { + addJarReference(udfInformation.getJarName(), udfInformation.getJarMD5()); + } + } + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java index 4c1225ef48a05..b24f1ca9aa528 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/cq/CQInfo.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.cq.CQState; import org.apache.iotdb.commons.cq.TimeoutPolicy; import org.apache.iotdb.commons.snapshot.SnapshotProcessor; +import org.apache.iotdb.confignode.consensus.request.read.cq.ShowCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.ActiveCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.DropCQPlan; @@ -44,7 +45,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -61,7 +64,7 @@ public class CQInfo implements SnapshotProcessor { private static final String CQ_NOT_EXIST_FORMAT = "CQ %s doesn't exist."; - private static final String MD5_NOT_MATCH_FORMAT = "MD5 of CQ %s doesn't match"; + private static final String CQ_TOKEN_NOT_MATCH_FORMAT = "Token of CQ %s doesn't match"; private final Map cqMap; @@ -91,7 +94,7 @@ public TSStatus addCQ(AddCQPlan plan) { CQEntry cqEntry = new CQEntry( plan.getReq(), - plan.getMd5(), + plan.getCqToken(), plan.getFirstExecutionTime() - plan.getReq().everyInterval); cqMap.put(cqId, cqEntry); res.code = TSStatusCode.SUCCESS_STATUS.getStatusCode(); @@ -105,13 +108,13 @@ public TSStatus addCQ(AddCQPlan plan) { /** * Drop the CQ whose ID is same as cqId in plan. * - * @return SUCCESS_STATUS if there is CQ whose ID and md5 is same as cqId in plan, + * @return SUCCESS_STATUS if there is CQ whose ID and token is same as cqId in plan, * otherwise NO_SUCH_CQ. */ public TSStatus dropCQ(DropCQPlan plan) { TSStatus res = new TSStatus(); String cqId = plan.getCqId(); - Optional md5 = plan.getMd5(); + Optional cqToken = plan.getCqToken(); lock.writeLock().lock(); try { CQEntry cqEntry = cqMap.get(cqId); @@ -119,10 +122,10 @@ public TSStatus dropCQ(DropCQPlan plan) { res.code = TSStatusCode.NO_SUCH_CQ.getStatusCode(); res.message = String.format(CQ_NOT_EXIST_FORMAT, cqId); LOGGER.warn("Drop CQ {} failed, because it doesn't exist.", cqId); - } else if ((md5.isPresent() && !md5.get().equals(cqEntry.md5))) { + } else if ((cqToken.isPresent() && !cqToken.get().equals(cqEntry.cqToken))) { res.code = TSStatusCode.NO_SUCH_CQ.getStatusCode(); - res.message = String.format(MD5_NOT_MATCH_FORMAT, cqId); - LOGGER.warn("Drop CQ {} failed, because its MD5 doesn't match.", cqId); + res.message = String.format(CQ_TOKEN_NOT_MATCH_FORMAT, cqId); + LOGGER.warn("Drop CQ {} failed, because its token doesn't match.", cqId); } else { cqMap.remove(cqId); res.code = TSStatusCode.SUCCESS_STATUS.getStatusCode(); @@ -135,11 +138,24 @@ public TSStatus dropCQ(DropCQPlan plan) { } public ShowCQResp showCQ() { + return showCQ(new ShowCQPlan()); + } + + public ShowCQResp showCQ(ShowCQPlan plan) { lock.readLock().lock(); try { - return new ShowCQResp( - new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()), - cqMap.values().stream().map(CQEntry::new).collect(Collectors.toList())); + Optional cqId = plan.getCqId(); + List cqList; + if (cqId.isPresent()) { + CQEntry cqEntry = cqMap.get(cqId.get()); + cqList = + cqEntry == null + ? Collections.emptyList() + : Collections.singletonList(new CQEntry(cqEntry)); + } else { + cqList = cqMap.values().stream().map(CQEntry::new).collect(Collectors.toList()); + } + return new ShowCQResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()), cqList); } finally { lock.readLock().unlock(); } @@ -153,16 +169,16 @@ public ShowCQResp showCQ() { public TSStatus activeCQ(ActiveCQPlan plan) { TSStatus res = new TSStatus(); String cqId = plan.getCqId(); - String md5 = plan.getMd5(); + String cqToken = plan.getCqToken(); lock.writeLock().lock(); try { CQEntry cqEntry = cqMap.get(cqId); if (cqEntry == null) { res.code = TSStatusCode.NO_SUCH_CQ.getStatusCode(); res.message = String.format(CQ_NOT_EXIST_FORMAT, cqId); - } else if (!md5.equals(cqEntry.md5)) { + } else if (!cqToken.equals(cqEntry.cqToken)) { res.code = TSStatusCode.NO_SUCH_CQ.getStatusCode(); - res.message = String.format(MD5_NOT_MATCH_FORMAT, cqId); + res.message = String.format(CQ_TOKEN_NOT_MATCH_FORMAT, cqId); } else if (cqEntry.state == CQState.ACTIVE) { res.code = TSStatusCode.CQ_ALREADY_ACTIVE.getStatusCode(); res.message = String.format("CQ %s has already been active", cqId); @@ -180,22 +196,22 @@ public TSStatus activeCQ(ActiveCQPlan plan) { * Update the last execution time of the corresponding CQ. * * @return SUCCESS_STATUS if successfully updated, or NO_SUCH_CQ if 1. the CQ doesn't exist; or 2. - * md5 is different. or CQ_UPDATE_LAST_EXEC_TIME_FAILED 3. original lastExecutionTime >= + * token is different. or CQ_UPDATE_LAST_EXEC_TIME_FAILED 3. original lastExecutionTime >= * current lastExecutionTime; */ public TSStatus updateCQLastExecutionTime(UpdateCQLastExecTimePlan plan) { TSStatus res = new TSStatus(); String cqId = plan.getCqId(); - String md5 = plan.getMd5(); + String cqToken = plan.getCqToken(); lock.writeLock().lock(); try { CQEntry cqEntry = cqMap.get(cqId); if (cqEntry == null) { res.code = TSStatusCode.NO_SUCH_CQ.getStatusCode(); res.message = String.format(CQ_NOT_EXIST_FORMAT, cqId); - } else if (!md5.equals(cqEntry.md5)) { + } else if (!cqToken.equals(cqEntry.cqToken)) { res.code = TSStatusCode.NO_SUCH_CQ.getStatusCode(); - res.message = String.format(MD5_NOT_MATCH_FORMAT, cqId); + res.message = String.format(CQ_TOKEN_NOT_MATCH_FORMAT, cqId); } else if (cqEntry.lastExecutionTime >= plan.getExecutionTime()) { res.code = TSStatusCode.CQ_UPDATE_LAST_EXEC_TIME_ERROR.getStatusCode(); res.message = @@ -299,7 +315,7 @@ public static class CQEntry { private final TimeoutPolicy timeoutPolicy; private final String queryBody; private final String sql; - private final String md5; + private final String cqToken; private final String zoneId; @@ -308,7 +324,7 @@ public static class CQEntry { private CQState state; private long lastExecutionTime; - private CQEntry(TCreateCQReq req, String md5, long lastExecutionTime) { + private CQEntry(TCreateCQReq req, String cqToken, long lastExecutionTime) { this( req.cqId, req.everyInterval, @@ -318,7 +334,7 @@ private CQEntry(TCreateCQReq req, String md5, long lastExecutionTime) { TimeoutPolicy.deserialize(req.timeoutPolicy), req.queryBody, req.sql, - md5, + cqToken, req.zoneId, req.username, CQState.INACTIVE, @@ -335,7 +351,7 @@ private CQEntry(CQEntry other) { other.timeoutPolicy, other.queryBody, other.sql, - other.md5, + other.cqToken, other.zoneId, other.username, other.state, @@ -352,7 +368,7 @@ private CQEntry( TimeoutPolicy timeoutPolicy, String queryBody, String sql, - String md5, + String cqToken, String zoneId, String username, CQState state, @@ -365,7 +381,7 @@ private CQEntry( this.timeoutPolicy = timeoutPolicy; this.queryBody = queryBody; this.sql = sql; - this.md5 = md5; + this.cqToken = cqToken; this.zoneId = zoneId; this.username = username; this.state = state; @@ -381,7 +397,7 @@ private void serialize(OutputStream stream) throws IOException { ReadWriteIOUtils.write(timeoutPolicy.getType(), stream); ReadWriteIOUtils.write(queryBody, stream); ReadWriteIOUtils.write(sql, stream); - ReadWriteIOUtils.write(md5, stream); + ReadWriteIOUtils.write(cqToken, stream); ReadWriteIOUtils.write(zoneId, stream); ReadWriteIOUtils.write(username, stream); ReadWriteIOUtils.write(state.getType(), stream); @@ -397,7 +413,7 @@ private static CQEntry deserialize(InputStream stream) throws IOException { TimeoutPolicy timeoutPolicy = TimeoutPolicy.deserialize(ReadWriteIOUtils.readByte(stream)); String queryBody = ReadWriteIOUtils.readString(stream); String sql = ReadWriteIOUtils.readString(stream); - String md5 = ReadWriteIOUtils.readString(stream); + String cqToken = ReadWriteIOUtils.readString(stream); String zoneId = ReadWriteIOUtils.readString(stream); String username = ReadWriteIOUtils.readString(stream); CQState state = CQState.deserialize(ReadWriteIOUtils.readByte(stream)); @@ -411,7 +427,7 @@ private static CQEntry deserialize(InputStream stream) throws IOException { timeoutPolicy, queryBody, sql, - md5, + cqToken, zoneId, username, state, @@ -450,8 +466,8 @@ public String getSql() { return sql; } - public String getMd5() { - return md5; + public String getCqToken() { + return cqToken; } public CQState getState() { @@ -488,7 +504,7 @@ public boolean equals(Object o) { && timeoutPolicy == cqEntry.timeoutPolicy && Objects.equals(queryBody, cqEntry.queryBody) && Objects.equals(sql, cqEntry.sql) - && Objects.equals(md5, cqEntry.md5) + && Objects.equals(cqToken, cqEntry.cqToken) && Objects.equals(zoneId, cqEntry.zoneId) && Objects.equals(username, cqEntry.username) && state == cqEntry.state; @@ -505,7 +521,7 @@ public int hashCode() { timeoutPolicy, queryBody, sql, - md5, + cqToken, zoneId, username, state, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index 4facb308d8a2c..60ec748fb5478 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -30,6 +30,7 @@ import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; import org.apache.iotdb.confignode.consensus.request.read.ainode.GetAINodeConfigurationPlan; import org.apache.iotdb.confignode.consensus.request.read.auth.AuthorReadPlan; +import org.apache.iotdb.confignode.consensus.request.read.cq.ShowCQPlan; import org.apache.iotdb.confignode.consensus.request.read.database.CountDatabasePlan; import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePlan; import org.apache.iotdb.confignode.consensus.request.read.datanode.GetDataNodeConfigurationPlan; @@ -98,6 +99,7 @@ import org.apache.iotdb.confignode.consensus.request.write.pipe.task.DropPipePlanV2; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.OperateMultiplePipesPlanV2; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusPlanV2; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusWithStoppedByRuntimeExceptionPlanV2; import org.apache.iotdb.confignode.consensus.request.write.procedure.DeleteProcedurePlan; import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan; import org.apache.iotdb.confignode.consensus.request.write.quota.SetSpaceQuotaPlan; @@ -321,7 +323,7 @@ public DataSet executeQueryPlan(final ConfigPhysicalReadPlan req) case GetSeriesSlotList: return partitionInfo.getSeriesSlotList((GetSeriesSlotListPlan) req); case SHOW_CQ: - return cqInfo.showCQ(); + return cqInfo.showCQ((ShowCQPlan) req); case GetFunctionTable: return udfInfo.getUDFTable(); case GetFunctionJar: @@ -497,6 +499,9 @@ public TSStatus executeNonQueryPlan(ConfigPhysicalPlan physicalPlan) return pipeInfo.createPipe((CreatePipePlanV2) physicalPlan); case SetPipeStatusV2: return pipeInfo.setPipeStatus((SetPipeStatusPlanV2) physicalPlan); + case SetPipeStatusWithStoppedByRuntimeExceptionV2: + return pipeInfo.setPipeStatusWithStoppedByRuntimeException( + (SetPipeStatusWithStoppedByRuntimeExceptionPlanV2) physicalPlan); case DropPipeV2: return pipeInfo.dropPipe((DropPipePlanV2) physicalPlan); case AlterPipeV2: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeInfo.java index d09e5c82845a0..032534f9161e6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeInfo.java @@ -29,6 +29,7 @@ import org.apache.iotdb.confignode.consensus.request.write.pipe.task.DropPipePlanV2; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.OperateMultiplePipesPlanV2; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusPlanV2; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusWithStoppedByRuntimeExceptionPlanV2; import org.apache.iotdb.confignode.manager.pipe.agent.PipeConfigNodeAgent; import org.apache.iotdb.confignode.manager.pipe.agent.runtime.PipeConfigRegionListener; import org.apache.iotdb.confignode.manager.pipe.agent.task.PipeConfigNodeSubtask; @@ -124,6 +125,25 @@ public TSStatus setPipeStatus(final SetPipeStatusPlanV2 plan) { } } + public TSStatus setPipeStatusWithStoppedByRuntimeException( + final SetPipeStatusWithStoppedByRuntimeExceptionPlanV2 plan) { + try { + pipeTaskInfo.setPipeStatusWithStoppedByRuntimeException(plan); + + PipeConfigNodeAgent.task() + .handleSinglePipeMetaChanges(pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeName())); + PipeTemporaryMetaInCoordinatorMetrics.getInstance() + .handleTemporaryMetaChanges(pipeTaskInfo.getPipeMetaList()); + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } catch (final Exception e) { + LOGGER.error("Failed to set pipe status with stopped-by-runtime-exception flag", e); + return new TSStatus(TSStatusCode.PIPE_ERROR.getStatusCode()) + .setMessage( + "Failed to set pipe status with stopped-by-runtime-exception flag, because " + + e.getMessage()); + } + } + public TSStatus dropPipe(final DropPipePlanV2 plan) { try { final Optional pipeMetaBeforeDrop = diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java index 2bfdbf9e49a3b..022144f1a9159 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java @@ -48,6 +48,7 @@ import org.apache.iotdb.confignode.consensus.request.write.pipe.task.DropPipePlanV2; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.OperateMultiplePipesPlanV2; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusPlanV2; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusWithStoppedByRuntimeExceptionPlanV2; import org.apache.iotdb.confignode.consensus.response.pipe.task.PipeTableResp; import org.apache.iotdb.confignode.manager.pipe.resource.PipeConfigNodeResourceManager; import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeHandleMetaChangeProcedure; @@ -523,6 +524,25 @@ public TSStatus setPipeStatus(final SetPipeStatusPlanV2 plan) { } } + public TSStatus setPipeStatusWithStoppedByRuntimeException( + final SetPipeStatusWithStoppedByRuntimeExceptionPlanV2 plan) { + acquireWriteLock(); + try { + pipeMetaKeeper + .getPipeMeta(plan.getPipeName()) + .getRuntimeMeta() + .getStatus() + .set(plan.getPipeStatus()); + pipeMetaKeeper + .getPipeMeta(plan.getPipeName()) + .getRuntimeMeta() + .setIsStoppedByRuntimeException(plan.isStoppedByRuntimeException()); + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } finally { + releaseWriteLock(); + } + } + public TSStatus dropPipe(final DropPipePlanV2 plan) { acquireWriteLock(); try { @@ -798,6 +818,14 @@ private boolean recordDataNodePushPipeMetaExceptionsInternal( final PipeRuntimeMeta runtimeMeta = pipeMetaKeeper.getPipeMeta(message.getPipeName()).getRuntimeMeta(); + // Keep user-stopped pipes out of the auto-restart flow. Otherwise, a failed + // STOPPED meta sync can turn a manually stopped pipe into a runtime-stopped one + // and the next PipeMetaSyncer round will restart it automatically. + if (PipeStatus.STOPPED.equals(runtimeMeta.getStatus().get()) + && !runtimeMeta.getIsStoppedByRuntimeException()) { + return; + } + // Mark the status of the pipe with exception as stopped runtimeMeta.getStatus().set(PipeStatus.STOPPED); runtimeMeta.setIsStoppedByRuntimeException(true); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java index 91af03d3971be..862bc449c7319 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; /** @@ -60,6 +61,7 @@ public abstract class Procedure implements Comparable> { private volatile long lastUpdate; private final AtomicReference result = new AtomicReference<>(); + private final AtomicBoolean executing = new AtomicBoolean(false); private volatile boolean locked = false; private boolean lockedWhenLoading = false; @@ -233,6 +235,16 @@ protected void releaseLock(Env env) { // no op } + /** + * Called after an execution attempt returns {@link ProcedureLockState#LOCK_EVENT_WAIT}. Override + * it to put the procedure into the corresponding lock wait queue. + * + * @param env env + */ + protected void waitForLock(Env env) { + // no op + } + /** * Used to keep procedure lock even when the procedure is yielded or suspended. * @@ -254,6 +266,14 @@ protected boolean isYieldAfterExecution(Env env) { } // -------------------------Internal methods - called by the procedureExecutor------------------ + final boolean tryAcquireExecution() { + return executing.compareAndSet(false, true); + } + + final void releaseExecution() { + executing.set(false); + } + /** * Internal method called by the ProcedureExecutor that starts the user-level code execute(). * diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java index 0d8368583b4eb..1633f78eec903 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler; import org.apache.iotdb.confignode.procedure.scheduler.SimpleProcedureScheduler; @@ -40,7 +39,6 @@ import java.util.Deque; import java.util.HashSet; import java.util.List; -import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -82,6 +80,16 @@ public class ProcedureExecutor { private final Env environment; private final IProcedureStore store; + private static final class LockStateResult { + private final ProcedureLockState lockState; + private final Procedure procedure; + + private LockStateResult(ProcedureLockState lockState, Procedure procedure) { + this.lockState = lockState; + this.procedure = procedure; + } + } + public ProcedureExecutor( final Env environment, final IProcedureStore store, final ProcedureScheduler scheduler) { this.environment = environment; @@ -320,32 +328,38 @@ private void executeProcedure(Procedure proc) { return; } ProcedureLockState lockState = null; + Procedure lockEventWaitProcedure = null; try { do { if (!rootProcStack.acquire()) { if (rootProcStack.setRollback()) { - lockState = executeRootStackRollback(rootProcId, rootProcStack); + LockStateResult lockStateResult = + executeRootStackRollback(rootProcId, rootProcStack); + lockState = lockStateResult.lockState; switch (lockState) { case LOCK_ACQUIRED: break; case LOCK_EVENT_WAIT: - LOG.info("LOCK_EVENT_WAIT rollback {}", proc); + LOG.info("LOCK_EVENT_WAIT rollback {}", lockStateResult.procedure); rootProcStack.unsetRollback(); + lockEventWaitProcedure = lockStateResult.procedure; break; case LOCK_YIELD_WAIT: rootProcStack.unsetRollback(); - scheduler.yield(proc); + scheduler.yield(lockStateResult.procedure); break; default: throw new UnsupportedOperationException(); } } else { if (!proc.wasExecuted()) { - switch (executeRollback(proc)) { + lockState = executeRollback(proc); + switch (lockState) { case LOCK_ACQUIRED: break; case LOCK_EVENT_WAIT: LOG.info("LOCK_EVENT_WAIT can't rollback child running for {}", proc); + lockEventWaitProcedure = proc; break; case LOCK_YIELD_WAIT: scheduler.yield(proc); @@ -357,19 +371,25 @@ private void executeProcedure(Procedure proc) { } break; } - lockState = acquireLock(proc); - switch (lockState) { - case LOCK_ACQUIRED: - executeProcedure(rootProcStack, proc); - break; - case LOCK_YIELD_WAIT: - case LOCK_EVENT_WAIT: - LOG.info("{} lockstate is {}", proc, lockState); - break; - default: - throw new UnsupportedOperationException(); + try { + lockState = acquireLock(proc); + switch (lockState) { + case LOCK_ACQUIRED: + executeProcedure(rootProcStack, proc); + break; + case LOCK_YIELD_WAIT: + case LOCK_EVENT_WAIT: + LOG.info("{} lockstate is {}", proc, lockState); + if (lockState == ProcedureLockState.LOCK_EVENT_WAIT) { + lockEventWaitProcedure = proc; + } + break; + default: + throw new UnsupportedOperationException(); + } + } finally { + rootProcStack.release(); } - rootProcStack.release(); if (proc.isSuccess()) { // update metrics on finishing the procedure @@ -387,9 +407,9 @@ private void executeProcedure(Procedure proc) { } finally { // Only after procedure has completed execution can it be allowed to be rescheduled to prevent // data races - if (Objects.equals(lockState, ProcedureLockState.LOCK_EVENT_WAIT)) { - LOG.info("procedureId {} wait for lock.", proc.getProcId()); - ((ConfigNodeProcedureEnv) this.environment).getNodeLock().waitProcedure(proc); + if (lockEventWaitProcedure != null) { + LOG.info("procedureId {} wait for lock.", lockEventWaitProcedure.getProcId()); + lockEventWaitProcedure.waitForLock(this.environment); } } } @@ -404,6 +424,7 @@ private void executeProcedure(RootProcedureStack rootProcStack, Procedure p if (proc.getState() != ProcedureState.RUNNABLE) { LOG.error( "The executing procedure should in RUNNABLE state, but it's not. Procedure is {}", proc); + releaseLock(proc, false); return; } boolean reExecute; @@ -570,8 +591,8 @@ private void yieldProcedure(Procedure proc) { * @param procedureStack root procedure stack * @return lock state */ - private ProcedureLockState executeRootStackRollback( - Long rootProcId, RootProcedureStack procedureStack) { + private LockStateResult executeRootStackRollback( + Long rootProcId, RootProcedureStack procedureStack) { Procedure rootProcedure = procedures.get(rootProcId); ProcedureException exception = rootProcedure.getException(); if (exception == null) { @@ -590,7 +611,7 @@ private ProcedureLockState executeRootStackRollback( } ProcedureLockState lockState = acquireLock(procedure); if (lockState != ProcedureLockState.LOCK_ACQUIRED) { - return lockState; + return new LockStateResult<>(lockState, procedure); } lockState = executeRollback(procedure); releaseLock(procedure, false); @@ -598,11 +619,11 @@ private ProcedureLockState executeRootStackRollback( boolean abortRollback = lockState != ProcedureLockState.LOCK_ACQUIRED; abortRollback |= !isRunning() || !store.isRunning(); if (abortRollback) { - return lockState; + return new LockStateResult<>(lockState, procedure); } if (!procedure.isFinished() && procedure.isYieldAfterExecution(this.environment)) { - return ProcedureLockState.LOCK_YIELD_WAIT; + return new LockStateResult<>(ProcedureLockState.LOCK_YIELD_WAIT, procedure); } if (procedure != rootProcedure) { @@ -612,7 +633,7 @@ private ProcedureLockState executeRootStackRollback( LOG.info("Rolled back {}, time duration is {}", rootProcedure, rootProcedure.elapsedTime()); rootProcedureCleanup(rootProcedure); - return ProcedureLockState.LOCK_ACQUIRED; + return new LockStateResult<>(ProcedureLockState.LOCK_ACQUIRED, rootProcedure); } private ProcedureLockState acquireLock(Procedure proc) { @@ -728,16 +749,33 @@ public void run() { Thread.sleep(1000); continue; } - this.activeProcedure.set(procedure); - activeExecutorCount.incrementAndGet(); - startTime.set(System.currentTimeMillis()); - executeProcedure(procedure); - activeExecutorCount.decrementAndGet(); - LOG.trace( - "Halt pid={}, activeCount={}", procedure.getProcId(), activeExecutorCount.get()); - this.activeProcedure.set(null); - lastUpdated = System.currentTimeMillis(); - startTime.set(lastUpdated); + boolean executionAcquired = false; + while (isRunning() && !(executionAcquired = procedure.tryAcquireExecution())) { + Thread.sleep(10); + } + if (!executionAcquired) { + continue; + } + try { + this.activeProcedure.set(procedure); + activeExecutorCount.incrementAndGet(); + startTime.set(System.currentTimeMillis()); + try { + executeProcedure(procedure); + } finally { + procedure.releaseExecution(); + activeExecutorCount.decrementAndGet(); + LOG.trace( + "Halt pid={}, activeCount={}", procedure.getProcId(), activeExecutorCount.get()); + this.activeProcedure.set(null); + lastUpdated = System.currentTimeMillis(); + startTime.set(lastUpdated); + } + } catch (Exception e) { + LOG.warn( + "Exception happened when worker {} execute procedure {}", getName(), procedure, e); + throw e; + } } } catch (Exception e) { @@ -748,6 +786,7 @@ public void run() { this.activeProcedure.get(), e); } + this.activeProcedure.set(null); } finally { LOG.info("Procedure worker {} terminated.", getName()); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java index 5aaf9a623f523..c998ad903c208 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java @@ -37,11 +37,13 @@ public TimeoutExecutorThread( } public void add(Procedure procedure) { - queue.add(new ProcedureDelayContainer<>(procedure)); + ProcedureDelayContainer delayTask = new ProcedureDelayContainer<>(procedure); + queue.remove(delayTask); + queue.add(delayTask); } public boolean remove(Procedure procedure) { - return queue.remove(new ProcedureDelayContainer<>(procedure)); + return queue.remove(new ProcedureDelayContainer<>(procedure)) || procedure.isFinished(); } private ProcedureDelayContainer takeQuietly() { @@ -62,10 +64,15 @@ public void run() { } Procedure procedure = delayTask.getProcedure(); if (procedure instanceof InternalProcedure) { + if (procedure.isFinished()) { + continue; + } InternalProcedure internal = (InternalProcedure) procedure; internal.periodicExecute(executor.getEnvironment()); - procedure.updateTimestamp(); - queue.add(delayTask); + if (!procedure.isFinished()) { + procedure.updateTimestamp(); + queue.add(delayTask); + } } else { if (procedure.setTimeoutFailure(executor.getEnvironment())) { long rootProcId = executor.getRootProcedureId(procedure); @@ -92,6 +99,23 @@ public Procedure getProcedure() { return procedure; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ProcedureDelayContainer)) { + return false; + } + ProcedureDelayContainer that = (ProcedureDelayContainer) o; + return procedure == that.procedure; + } + + @Override + public int hashCode() { + return System.identityHashCode(procedure); + } + @Override public long getDelay(TimeUnit unit) { long delay = procedure.getTimeoutTimestamp() - System.currentTimeMillis(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/StateMachineProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/StateMachineProcedure.java index f698735ee95be..400da5653181c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/StateMachineProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/StateMachineProcedure.java @@ -195,7 +195,8 @@ private void addNextStateAndCalculateCycles() { nextState); } } - if (getStateId(getCurrentState()) == stateToBeAdded) { + final TState currentState = getCurrentState(); + if (currentState != null && getStateId(currentState) == stateToBeAdded) { cycles++; } else { cycles = 0; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java index 3da60c5ad847b..76df7765c9a57 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedure.java @@ -22,10 +22,14 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; +import org.apache.iotdb.confignode.consensus.request.read.cq.ShowCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.ActiveCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.DropCQPlan; +import org.apache.iotdb.confignode.consensus.response.cq.ShowCQResp; +import org.apache.iotdb.confignode.manager.cq.CQManager; import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; +import org.apache.iotdb.confignode.persistence.cq.CQInfo; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.impl.node.AbstractNodeProcedure; @@ -35,7 +39,6 @@ import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.rpc.TSStatusCode; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +47,8 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Objects; +import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ScheduledExecutorService; import static org.apache.iotdb.confignode.procedure.state.cq.CreateCQState.INACTIVE; @@ -59,7 +64,7 @@ public class CreateCQProcedure extends AbstractNodeProcedure { private TCreateCQReq req; - private String md5; + private String cqToken; private long firstExecutionTime; @@ -74,7 +79,7 @@ public CreateCQProcedure(ScheduledExecutorService executor) { public CreateCQProcedure(TCreateCQReq req, ScheduledExecutorService executor) { super(); this.req = req; - this.md5 = DigestUtils.md2Hex(req.cqId); + this.cqToken = generateCQToken(); this.executor = executor; this.firstExecutionTime = CQScheduleTask.getFirstExecutionTime(req.boundaryTime, req.everyInterval); @@ -90,12 +95,16 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, CreateCQState state) addCQ(env); return Flow.HAS_MORE_STATE; case INACTIVE: - CQScheduleTask cqScheduleTask = - new CQScheduleTask(req, firstExecutionTime, md5, executor, env.getConfigManager()); - cqScheduleTask.submitSelf(); + submitScheduleTask( + env, + new CQScheduleTask( + req, firstExecutionTime, cqToken, executor, env.getConfigManager())); setNextState(SCHEDULED); break; case SCHEDULED: + if (isStateDeserialized()) { + recoverScheduledTask(env); + } activeCQ(env); return Flow.NO_MORE_STATE; default: @@ -125,7 +134,7 @@ private void addCQ(ConfigNodeProcedureEnv env) { res = env.getConfigManager() .getConsensusManager() - .write(new AddCQPlan(req, md5, firstExecutionTime)); + .write(new AddCQPlan(req, cqToken, firstExecutionTime)); } catch (ConsensusException e) { LOGGER.warn(CONSENSUS_WRITE_ERROR, e); res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); @@ -146,7 +155,7 @@ private void addCQ(ConfigNodeProcedureEnv env) { private void activeCQ(ConfigNodeProcedureEnv env) { TSStatus res; try { - res = env.getConfigManager().getConsensusManager().write(new ActiveCQPlan(req.cqId, md5)); + res = env.getConfigManager().getConsensusManager().write(new ActiveCQPlan(req.cqId, cqToken)); } catch (ConsensusException e) { LOGGER.warn(CONSENSUS_WRITE_ERROR, e); res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); @@ -164,6 +173,42 @@ private void activeCQ(ConfigNodeProcedureEnv env) { } } + void recoverScheduledTask(ConfigNodeProcedureEnv env) throws ConsensusException { + Optional cqEntry = getCurrentCQEntry(env); + if (!cqEntry.isPresent()) { + LOGGER.info( + "Skip recovering the schedule task of CQ {} because its metadata is unavailable.", + req.cqId); + return; + } + submitScheduleTask(env, new CQScheduleTask(cqEntry.get(), executor, env.getConfigManager())); + } + + Optional getCurrentCQEntry(ConfigNodeProcedureEnv env) throws ConsensusException { + ShowCQResp response = + (ShowCQResp) env.getConfigManager().getConsensusManager().read(new ShowCQPlan(req.cqId)); + return response.getCqList().stream() + .filter(entry -> cqToken.equals(entry.getCqToken())) + .findFirst(); + } + + private static String generateCQToken() { + return UUID.randomUUID().toString(); + } + + private void submitScheduleTask(ConfigNodeProcedureEnv env, CQScheduleTask cqScheduleTask) { + CQManager cqManager = env.getConfigManager().getCQManager(); + if (!cqManager.markCQLocallyScheduled(req.cqId, cqToken, cqScheduleTask)) { + return; + } + try { + cqScheduleTask.submitSelf(); + } catch (RuntimeException e) { + cqManager.unmarkCQLocallyScheduled(req.cqId, cqToken); + throw e; + } + } + @Override protected void rollbackState(ConfigNodeProcedureEnv env, CreateCQState state) throws IOException, InterruptedException, ProcedureException { @@ -176,7 +221,8 @@ protected void rollbackState(ConfigNodeProcedureEnv env, CreateCQState state) LOGGER.info("Start [INACTIVE] rollback of CQ {}", req.cqId); TSStatus res; try { - res = env.getConfigManager().getConsensusManager().write(new DropCQPlan(req.cqId, md5)); + res = + env.getConfigManager().getConsensusManager().write(new DropCQPlan(req.cqId, cqToken)); } catch (ConsensusException e) { LOGGER.warn(CONSENSUS_WRITE_ERROR, e); res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); @@ -227,7 +273,7 @@ public void serialize(DataOutputStream stream) throws IOException { stream.writeShort(ProcedureType.CREATE_CQ_PROCEDURE.getTypeCode()); super.serialize(stream); ThriftCommonsSerDeUtils.serializeTCreateCQReq(req, stream); - ReadWriteIOUtils.write(md5, stream); + ReadWriteIOUtils.write(cqToken, stream); ReadWriteIOUtils.write(firstExecutionTime, stream); } @@ -235,7 +281,7 @@ public void serialize(DataOutputStream stream) throws IOException { public void deserialize(ByteBuffer byteBuffer) { super.deserialize(byteBuffer); this.req = ThriftCommonsSerDeUtils.deserializeTCreateCQReq(byteBuffer); - this.md5 = ReadWriteIOUtils.readString(byteBuffer); + this.cqToken = ReadWriteIOUtils.readString(byteBuffer); this.firstExecutionTime = ReadWriteIOUtils.readLong(byteBuffer); } @@ -249,12 +295,12 @@ public boolean equals(Object o) { } CreateCQProcedure that = (CreateCQProcedure) o; return getProcId() == that.getProcId() - && getCurrentState().equals(that.getCurrentState()) + && Objects.equals(getCurrentState(), that.getCurrentState()) && getCycles() == that.getCycles() && isGeneratedByPipe == that.isGeneratedByPipe && firstExecutionTime == that.firstExecutionTime && Objects.equals(req, that.req) - && Objects.equals(md5, that.md5); + && Objects.equals(cqToken, that.cqToken); } @Override @@ -265,7 +311,15 @@ public int hashCode() { getCycles(), isGeneratedByPipe, req, - md5, + cqToken, firstExecutionTime); } + + public String getCqId() { + return req == null ? null : req.getCqId(); + } + + public String getCqToken() { + return cqToken; + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AbstractNodeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AbstractNodeProcedure.java index b141027917366..6cade537f1fbb 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AbstractNodeProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/AbstractNodeProcedure.java @@ -56,6 +56,18 @@ protected ProcedureLockState acquireLock(ConfigNodeProcedureEnv configNodeProced } } + @Override + protected void waitForLock(ConfigNodeProcedureEnv configNodeProcedureEnv) { + configNodeProcedureEnv.getSchedulerLock().lock(); + try { + configNodeProcedureEnv + .getNodeLock() + .waitProcedure(this, configNodeProcedureEnv.getScheduler()); + } finally { + configNodeProcedureEnv.getSchedulerLock().unlock(); + } + } + @Override protected void releaseLock(ConfigNodeProcedureEnv configNodeProcedureEnv) { configNodeProcedureEnv.getSchedulerLock().lock(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java index 857d5733f6a7a..43620ac915796 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java @@ -133,12 +133,7 @@ protected ProcedureLockState acquireLock(ConfigNodeProcedureEnv configNodeProced LOGGER.warn("ProcedureId {}: LOCK_EVENT_WAIT. Without acquiring pipe lock.", getProcId()); } else { LOGGER.debug("ProcedureId {}: LOCK_EVENT_WAIT. Pipe lock will be released.", getProcId()); - configNodeProcedureEnv - .getConfigManager() - .getPipeManager() - .getPipeTaskCoordinator() - .unlock(); - pipeTaskInfo = null; + releasePipeTaskCoordinatorLock(configNodeProcedureEnv); } break; default: @@ -152,12 +147,7 @@ protected ProcedureLockState acquireLock(ConfigNodeProcedureEnv configNodeProced "ProcedureId {}: {}. Invalid lock state. Pipe lock will be released.", getProcId(), procedureLockState); - configNodeProcedureEnv - .getConfigManager() - .getPipeManager() - .getPipeTaskCoordinator() - .unlock(); - pipeTaskInfo = null; + releasePipeTaskCoordinatorLock(configNodeProcedureEnv); } break; } @@ -181,11 +171,16 @@ protected void releaseLock(ConfigNodeProcedureEnv configNodeProcedureEnv) { } PipeProcedureMetrics.getInstance() .updateTimer(this.getOperation().getName(), this.elapsedTime()); - configNodeProcedureEnv.getConfigManager().getPipeManager().getPipeTaskCoordinator().unlock(); - pipeTaskInfo = null; + releasePipeTaskCoordinatorLock(configNodeProcedureEnv); } } + private void releasePipeTaskCoordinatorLock(ConfigNodeProcedureEnv configNodeProcedureEnv) { + // Clear before releasing the semaphore to avoid clobbering a re-scheduled execution's marker. + pipeTaskInfo = null; + configNodeProcedureEnv.getConfigManager().getPipeManager().getPipeTaskCoordinator().unlock(); + } + protected abstract PipeTaskOperation getOperation(); /** diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/plugin/CreatePipePluginProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/plugin/CreatePipePluginProcedure.java index f4fa738428d43..22fe0ef40f279 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/plugin/CreatePipePluginProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/plugin/CreatePipePluginProcedure.java @@ -314,7 +314,7 @@ public boolean equals(Object that) { if (that instanceof CreatePipePluginProcedure) { CreatePipePluginProcedure thatProcedure = (CreatePipePluginProcedure) that; return thatProcedure.getProcId() == getProcId() - && thatProcedure.getCurrentState().equals(getCurrentState()) + && Objects.equals(thatProcedure.getCurrentState(), getCurrentState()) && thatProcedure.getCycles() == getCycles() && thatProcedure.pipePluginMeta.equals(pipePluginMeta); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/plugin/DropPipePluginProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/plugin/DropPipePluginProcedure.java index efbe1ee6ccdda..5deb0ee4bf31f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/plugin/DropPipePluginProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/plugin/DropPipePluginProcedure.java @@ -275,7 +275,7 @@ public boolean equals(Object that) { if (that instanceof DropPipePluginProcedure) { final DropPipePluginProcedure thatProcedure = (DropPipePluginProcedure) that; return thatProcedure.getProcId() == getProcId() - && thatProcedure.getCurrentState().equals(this.getCurrentState()) + && Objects.equals(thatProcedure.getCurrentState(), this.getCurrentState()) && thatProcedure.getCycles() == this.getCycles() && (thatProcedure.pluginName).equals(pluginName); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleLeaderChangeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleLeaderChangeProcedure.java index f18737e8b7a7e..598d8192e06af 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleLeaderChangeProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleLeaderChangeProcedure.java @@ -164,7 +164,11 @@ public void deserialize(ByteBuffer byteBuffer) { final int oldDataRegionLeaderId = ReadWriteIOUtils.readInt(byteBuffer); final int newDataRegionLeaderId = ReadWriteIOUtils.readInt(byteBuffer); regionGroupToOldAndNewLeaderPairMap.put( - new TConsensusGroupId(TConsensusGroupType.DataRegion, dataRegionGroupId), + new TConsensusGroupId( + dataRegionGroupId == Integer.MIN_VALUE + ? TConsensusGroupType.ConfigRegion + : TConsensusGroupType.DataRegion, + dataRegionGroupId), new Pair<>(oldDataRegionLeaderId, newDataRegionLeaderId)); } } @@ -179,7 +183,7 @@ public boolean equals(Object o) { } PipeHandleLeaderChangeProcedure that = (PipeHandleLeaderChangeProcedure) o; return getProcId() == that.getProcId() - && getCurrentState().equals(that.getCurrentState()) + && Objects.equals(getCurrentState(), that.getCurrentState()) && getCycles() == that.getCycles() && this.regionGroupToOldAndNewLeaderPairMap.equals( that.regionGroupToOldAndNewLeaderPairMap); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleMetaChangeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleMetaChangeProcedure.java index 401859f0a7e0a..def062359e4c4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleMetaChangeProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleMetaChangeProcedure.java @@ -186,7 +186,7 @@ public boolean equals(Object o) { } PipeHandleMetaChangeProcedure that = (PipeHandleMetaChangeProcedure) o; return getProcId() == that.getProcId() - && getCurrentState().equals(that.getCurrentState()) + && Objects.equals(getCurrentState(), that.getCurrentState()) && getCycles() == that.getCycles() && needWriteConsensusOnConfigNodes == that.needWriteConsensusOnConfigNodes && needPushPipeMetaToDataNodes == that.needPushPipeMetaToDataNodes; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java index 5a7b9b8a5b262..dcebb0b3d33e9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; @@ -169,7 +170,12 @@ public void executeFromCalculateInfoForTask(ConfigNodeProcedureEnv env) { updatedConsensusGroupIdToTaskMetaMap.put( regionGroupId.getId(), new PipeTaskMeta( - currentPipeTaskMeta.getProgressIndex(), + PipeTaskAgent.isRealtimeOnlyPipe( + currentPipeStaticMeta.getExtractorParameters()) + && !PipeTaskAgent.isRealtimeOnlyPipe( + updatedPipeStaticMeta.getExtractorParameters()) + ? MinimumProgressIndex.INSTANCE + : currentPipeTaskMeta.getProgressIndex(), PipeTaskMeta.isNewlyAdded(currentPipeTaskMeta.getLeaderNodeId()) && !(!PipeTaskAgent.isHistoryOnlyPipe( currentPipeStaticMeta.getExtractorParameters()) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2.java index 26e9eb9bba145..ad18cc95bb050 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2.java @@ -25,11 +25,13 @@ import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.RecoverProgressIndex; import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex; +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeType; +import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2; @@ -41,8 +43,10 @@ import org.apache.iotdb.confignode.procedure.impl.pipe.PipeTaskOperation; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TPermissionInfoResp; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.pipe.api.PipePlugin; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.rpc.TSStatusCode; @@ -53,6 +57,8 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -136,9 +142,70 @@ public boolean executeFromValidateTask(ConfigNodeProcedureEnv env) throws PipeEx createPipeRequest.getProcessorAttributes(), createPipeRequest.getConnectorAttributes()); + checkAndEnrichSourceAuthentication(env, createPipeRequest.getExtractorAttributes()); + return pipeTaskInfo.get().checkBeforeCreatePipe(createPipeRequest); } + public static void checkAndEnrichSourceAuthentication( + final ConfigNodeProcedureEnv env, final Map sourceAttributes) { + if (Objects.isNull(sourceAttributes)) { + return; + } + final PipeParameters sourceParameters = new PipeParameters(sourceAttributes); + + final String pluginName = + sourceParameters + .getStringOrDefault( + Arrays.asList(PipeSourceConstant.EXTRACTOR_KEY, PipeSourceConstant.SOURCE_KEY), + BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginName()) + .toLowerCase(); + + if (!pluginName.equals(BuiltinPipePlugin.IOTDB_EXTRACTOR.getPipePluginName()) + && !pluginName.equals(BuiltinPipePlugin.IOTDB_SOURCE.getPipePluginName())) { + return; + } + + if (sourceParameters.hasAttribute(PipeSourceConstant.EXTRACTOR_IOTDB_USER_KEY) + || sourceParameters.hasAttribute(PipeSourceConstant.SOURCE_IOTDB_USER_KEY) + || sourceParameters.hasAttribute(PipeSourceConstant.EXTRACTOR_IOTDB_USERNAME_KEY) + || sourceParameters.hasAttribute(PipeSourceConstant.SOURCE_IOTDB_USERNAME_KEY) + || sourceParameters.hasAttribute(PipeSourceConstant.EXTRACTOR_IOTDB_PASSWORD_KEY) + || sourceParameters.hasAttribute(PipeSourceConstant.SOURCE_IOTDB_PASSWORD_KEY)) { + final String username = + sourceParameters.getStringByKeys( + PipeSourceConstant.EXTRACTOR_IOTDB_USER_KEY, + PipeSourceConstant.SOURCE_IOTDB_USER_KEY, + PipeSourceConstant.EXTRACTOR_IOTDB_USERNAME_KEY, + PipeSourceConstant.SOURCE_IOTDB_USERNAME_KEY); + final String password = + sourceParameters.getStringByKeys( + PipeSourceConstant.EXTRACTOR_IOTDB_PASSWORD_KEY, + PipeSourceConstant.SOURCE_IOTDB_PASSWORD_KEY); + String hashedPassword = null; + if (Objects.nonNull(password)) { + final TPermissionInfoResp loginResp = + env.getConfigManager().getPermissionManager().login(username, password, true); + if (Objects.nonNull(loginResp) + && Objects.nonNull(loginResp.getStatus()) + && loginResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + hashedPassword = password; + } + } + if (Objects.isNull(hashedPassword)) { + hashedPassword = + env.getConfigManager().getPermissionManager().login4Pipe(username, password); + } + if (Objects.isNull(hashedPassword)) { + throw new PipeException("Authentication failed."); + } + sourceParameters.addOrReplaceEquivalentAttributes( + new PipeParameters( + Collections.singletonMap( + PipeSourceConstant.SOURCE_IOTDB_PASSWORD_KEY, hashedPassword))); + } + } + @Override public void executeFromCalculateInfoForTask(ConfigNodeProcedureEnv env) { LOGGER.info( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/DropPipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/DropPipeProcedureV2.java index 0c7042caf3f07..0f9a8bc0d8c84 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/DropPipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/DropPipeProcedureV2.java @@ -165,7 +165,7 @@ public boolean equals(Object o) { } DropPipeProcedureV2 that = (DropPipeProcedureV2) o; return getProcId() == that.getProcId() - && getCurrentState().equals(that.getCurrentState()) + && Objects.equals(getCurrentState(), that.getCurrentState()) && getCycles() == that.getCycles() && pipeName.equals(that.pipeName); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2.java index fe36137b35f47..29cb2b51ab4e5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StartPipeProcedureV2.java @@ -184,7 +184,7 @@ public boolean equals(Object o) { } StartPipeProcedureV2 that = (StartPipeProcedureV2) o; return getProcId() == that.getProcId() - && getCurrentState().equals(that.getCurrentState()) + && Objects.equals(getCurrentState(), that.getCurrentState()) && getCycles() == that.getCycles() && pipeName.equals(that.pipeName); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2.java index 5349cc65640d0..44ca8a7bd5c1a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2.java @@ -21,7 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; -import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusPlanV2; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusWithStoppedByRuntimeExceptionPlanV2; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.impl.pipe.AbstractOperatePipeProcedureV2; import org.apache.iotdb.confignode.procedure.impl.pipe.PipeTaskOperation; @@ -44,6 +44,7 @@ public class StopPipeProcedureV2 extends AbstractOperatePipeProcedureV2 { private static final Logger LOGGER = LoggerFactory.getLogger(StopPipeProcedureV2.class); private String pipeName; + private boolean isStoppedByRuntimeExceptionBeforeStop; public StopPipeProcedureV2() { super(); @@ -71,7 +72,8 @@ public boolean executeFromValidateTask(ConfigNodeProcedureEnv env) throws PipeEx @Override public void executeFromCalculateInfoForTask(ConfigNodeProcedureEnv env) throws PipeException { LOGGER.info("StopPipeProcedureV2: executeFromCalculateInfoForTask({})", pipeName); - // Do nothing + isStoppedByRuntimeExceptionBeforeStop = + pipeTaskInfo.get().isStoppedByRuntimeException(pipeName); } @Override @@ -83,7 +85,9 @@ public void executeFromWriteConfigNodeConsensus(ConfigNodeProcedureEnv env) thro response = env.getConfigManager() .getConsensusManager() - .write(new SetPipeStatusPlanV2(pipeName, PipeStatus.STOPPED)); + .write( + new SetPipeStatusWithStoppedByRuntimeExceptionPlanV2( + pipeName, PipeStatus.STOPPED, false)); } catch (ConsensusException e) { LOGGER.warn("Failed in the write API executing the consensus layer due to: ", e); response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); @@ -128,7 +132,9 @@ public void rollbackFromWriteConfigNodeConsensus(ConfigNodeProcedureEnv env) { response = env.getConfigManager() .getConsensusManager() - .write(new SetPipeStatusPlanV2(pipeName, PipeStatus.RUNNING)); + .write( + new SetPipeStatusWithStoppedByRuntimeExceptionPlanV2( + pipeName, PipeStatus.RUNNING, isStoppedByRuntimeExceptionBeforeStop)); } catch (ConsensusException e) { LOGGER.warn("Failed in the write API executing the consensus layer due to: ", e); response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); @@ -159,12 +165,16 @@ public void serialize(DataOutputStream stream) throws IOException { stream.writeShort(ProcedureType.STOP_PIPE_PROCEDURE_V2.getTypeCode()); super.serialize(stream); ReadWriteIOUtils.write(pipeName, stream); + ReadWriteIOUtils.write(isStoppedByRuntimeExceptionBeforeStop, stream); } @Override public void deserialize(ByteBuffer byteBuffer) { super.deserialize(byteBuffer); pipeName = ReadWriteIOUtils.readString(byteBuffer); + // Legacy persisted procedures do not carry this field. + isStoppedByRuntimeExceptionBeforeStop = + byteBuffer.hasRemaining() && ReadWriteIOUtils.readBool(byteBuffer); } @Override @@ -177,13 +187,19 @@ public boolean equals(Object o) { } StopPipeProcedureV2 that = (StopPipeProcedureV2) o; return getProcId() == that.getProcId() - && getCurrentState().equals(that.getCurrentState()) + && Objects.equals(getCurrentState(), that.getCurrentState()) && getCycles() == that.getCycles() + && isStoppedByRuntimeExceptionBeforeStop == that.isStoppedByRuntimeExceptionBeforeStop && pipeName.equals(that.pipeName); } @Override public int hashCode() { - return Objects.hash(getProcId(), getCurrentState(), getCycles(), pipeName); + return Objects.hash( + getProcId(), + getCurrentState(), + getCycles(), + pipeName, + isStoppedByRuntimeExceptionBeforeStop); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java index 7288090e81559..b6ad21128af3b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java @@ -303,7 +303,7 @@ public boolean equals(Object that) { if (that instanceof DeleteDatabaseProcedure) { DeleteDatabaseProcedure thatProc = (DeleteDatabaseProcedure) that; return thatProc.getProcId() == this.getProcId() - && thatProc.getCurrentState().equals(this.getCurrentState()) + && Objects.equals(thatProc.getCurrentState(), this.getCurrentState()) && thatProc.getCycles() == this.getCycles() && thatProc.isGeneratedByPipe == this.isGeneratedByPipe && thatProc.deleteDatabaseSchema.equals(this.getDeleteDatabaseSchema()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java index aaf7bc1a7bff0..fd5d0d52f227c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java @@ -303,7 +303,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; DeleteLogicalViewProcedure that = (DeleteLogicalViewProcedure) o; return this.getProcId() == that.getProcId() - && this.getCurrentState().equals(that.getCurrentState()) + && Objects.equals(this.getCurrentState(), that.getCurrentState()) && this.getCycles() == that.getCycles() && isGeneratedByPipe == that.isGeneratedByPipe && patternTree.equals(that.patternTree); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java index 74f21abc72da2..0a223de1525f3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java @@ -367,7 +367,7 @@ public boolean equals(final Object o) { if (o == null || getClass() != o.getClass()) return false; final DeleteTimeSeriesProcedure that = (DeleteTimeSeriesProcedure) o; return this.getProcId() == that.getProcId() - && this.getCurrentState().equals(that.getCurrentState()) + && Objects.equals(this.getCurrentState(), that.getCurrentState()) && this.getCycles() == getCycles() && this.isGeneratedByPipe == that.isGeneratedByPipe && this.patternTree.equals(that.patternTree); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTTLProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTTLProcedure.java index 3e1af7dd1f55b..510690a8d6f91 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTTLProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTTLProcedure.java @@ -22,8 +22,10 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; +import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.schema.ttl.TTLCache; import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext; @@ -45,14 +47,21 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Collections; import java.util.Map; import java.util.Objects; public class SetTTLProcedure extends StateMachineProcedure { private static final Logger LOGGER = LoggerFactory.getLogger(SetTTLProcedure.class); + // Distinguishes no previous TTL from TTLCache.NULL_TTL, the explicit unset marker for rollback. + private static final long TTL_NOT_EXIST = Long.MIN_VALUE; + private static final int ROLLBACK_STATE_BYTES = Byte.BYTES + Long.BYTES * 2; private SetTTLPlan plan; + private long previousTTL = TTL_NOT_EXIST; + private long previousDatabaseWildcardTTL = TTL_NOT_EXIST; + private boolean previousTTLStateCaptured = false; public SetTTLProcedure(final boolean isGeneratedByPipe) { super(isGeneratedByPipe); @@ -69,6 +78,10 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, SetTTLState state) long startTime = System.currentTimeMillis(); try { switch (state) { + case CAPTURE_PREVIOUS_TTL: + capturePreviousTTLState(env); + setNextState(SetTTLState.SET_CONFIGNODE_TTL); + return Flow.HAS_MORE_STATE; case SET_CONFIGNODE_TTL: setConfigNodeTTL(env); return Flow.HAS_MORE_STATE; @@ -83,18 +96,13 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, SetTTLState state) } } - private void setConfigNodeTTL(ConfigNodeProcedureEnv env) { - TSStatus res; - try { - res = - env.getConfigManager() - .getConsensusManager() - .write(isGeneratedByPipe ? new PipeEnrichedPlan(this.plan) : this.plan); - } catch (ConsensusException e) { - LOGGER.warn("Failed in the write API executing the consensus layer due to: ", e); - res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); - res.setMessage(e.getMessage()); + void setConfigNodeTTL(final ConfigNodeProcedureEnv env) { + if (!previousTTLStateCaptured) { + capturePreviousTTLState(env); + setNextState(SetTTLState.SET_CONFIGNODE_TTL); + return; } + final TSStatus res = writeConfigNodePlan(env, plan); if (res.code != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.info("Failed to execute plan {} because {}", plan, res.message); setFailure(new ProcedureException(new IoTDBException(res))); @@ -103,34 +111,175 @@ private void setConfigNodeTTL(ConfigNodeProcedureEnv env) { } } - private void updateDataNodeTTL(ConfigNodeProcedureEnv env) { - Map dataNodeLocationMap = + void updateDataNodeTTL(final ConfigNodeProcedureEnv env) { + final Map dataNodeLocationMap = env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); - DataNodeAsyncRequestContext clientHandler = - new DataNodeAsyncRequestContext<>( - CnToDnAsyncRequestType.SET_TTL, - new TSetTTLReq( - Collections.singletonList(String.join(".", plan.getPathPattern())), - plan.getTTL(), - plan.isDataBase()), - dataNodeLocationMap); + final DataNodeAsyncRequestContext clientHandler = + sendTTLRequest( + dataNodeLocationMap, + buildSetTTLReq(plan.getPathPattern(), plan.getTTL(), plan.isDataBase())); + if (hasFailedDataNode(clientHandler)) { + LOGGER.error("Failed to update ttl cache of dataNode."); + setFailure(new ProcedureException(new MetadataException("Update dataNode ttl cache failed"))); + } + } + + private void capturePreviousTTLState(final ConfigNodeProcedureEnv env) { + if (previousTTLStateCaptured) { + return; + } + previousTTL = getTTLOrDefault(env, plan.getPathPattern()); + if (plan.isDataBase()) { + previousDatabaseWildcardTTL = + getTTLOrDefault(env, getDatabaseWildcardPathPattern(plan.getPathPattern())); + } + previousTTLStateCaptured = true; + } + + TSStatus writeConfigNodePlan(final ConfigNodeProcedureEnv env, final SetTTLPlan setTTLPlan) { + try { + return env.getConfigManager() + .getConsensusManager() + .write(isGeneratedByPipe ? new PipeEnrichedPlan(setTTLPlan) : setTTLPlan); + } catch (ConsensusException e) { + LOGGER.warn("Failed in the write API executing the consensus layer due to: ", e); + final TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + res.setMessage(e.getMessage()); + return res; + } + } + + DataNodeAsyncRequestContext sendTTLRequest( + final Map dataNodeLocationMap, final TSetTTLReq req) { + final DataNodeAsyncRequestContext clientHandler = + new DataNodeAsyncRequestContext<>(CnToDnAsyncRequestType.SET_TTL, req, dataNodeLocationMap); CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler); - Map statusMap = clientHandler.getResponseMap(); - for (TSStatus status : statusMap.values()) { - // all dataNodes must clear the related schemaengine cache + return clientHandler; + } + + private TSetTTLReq buildSetTTLReq( + final String[] pathPattern, final long ttl, final boolean isDataBase) { + return new TSetTTLReq( + Collections.singletonList(String.join(".", pathPattern)), ttl, isDataBase); + } + + private boolean hasFailedDataNode( + final DataNodeAsyncRequestContext clientHandler) { + if (!clientHandler.getRequestIndices().isEmpty()) { + return true; + } + for (TSStatus status : clientHandler.getResponseMap().values()) { if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - LOGGER.error("Failed to update ttl cache of dataNode."); - setFailure( - new ProcedureException(new MetadataException("Update dataNode ttl cache failed"))); - return; + return true; } } + return false; } + private long getTTLOrDefault(final ConfigNodeProcedureEnv env, final String[] pathPattern) { + final long ttl = env.getConfigManager().getTTLManager().getTTL(pathPattern); + return ttl == TTLCache.NULL_TTL ? TTL_NOT_EXIST : ttl; + } + + private String[] getDatabaseWildcardPathPattern(final String[] pathPattern) { + final String[] pathNodes = Arrays.copyOf(pathPattern, pathPattern.length + 1); + pathNodes[pathNodes.length - 1] = IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; + return pathNodes; + } + + private void rollbackConfigNodeTTL(final ConfigNodeProcedureEnv env) throws ProcedureException { + restoreTTLOnConfigNode(env, plan.getPathPattern(), previousTTL); + if (plan.isDataBase()) { + restoreTTLOnConfigNode( + env, getDatabaseWildcardPathPattern(plan.getPathPattern()), previousDatabaseWildcardTTL); + } + } + + private void restoreTTLOnConfigNode( + final ConfigNodeProcedureEnv env, final String[] pathPattern, final long ttl) + throws ProcedureException { + // TTL_NOT_EXIST means the original ttl was absent; NULL_TTL asks the executor to unset it. + final SetTTLPlan rollbackPlan = + new SetTTLPlan(pathPattern, ttl == TTL_NOT_EXIST ? TTLCache.NULL_TTL : ttl); + // Database rollback restores the database path and db.** separately, so avoid auto-expansion. + rollbackPlan.setDataBase(false); + final TSStatus status = writeConfigNodePlan(env, rollbackPlan); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new ProcedureException( + new MetadataException( + "Rollback ConfigNode ttl failed for " + + String.join(".", pathPattern) + + ": " + + status.getMessage())); + } + } + + private void rollbackDataNodeTTL(final ConfigNodeProcedureEnv env) throws ProcedureException { + final Map dataNodeLocationMap = + env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); + restoreTTLOnDataNodes(dataNodeLocationMap, plan.getPathPattern(), previousTTL); + if (plan.isDataBase()) { + restoreTTLOnDataNodes( + dataNodeLocationMap, + getDatabaseWildcardPathPattern(plan.getPathPattern()), + previousDatabaseWildcardTTL); + } + } + + private void restoreTTLOnDataNodes( + final Map dataNodeLocationMap, + final String[] pathPattern, + final long ttl) + throws ProcedureException { + if (dataNodeLocationMap.isEmpty()) { + return; + } + final DataNodeAsyncRequestContext clientHandler = + sendTTLRequest( + dataNodeLocationMap, + buildSetTTLReq(pathPattern, ttl == TTL_NOT_EXIST ? TTLCache.NULL_TTL : ttl, false)); + if (hasFailedDataNode(clientHandler)) { + throw new ProcedureException( + new MetadataException( + "Rollback dataNode ttl cache failed for " + String.join(".", pathPattern))); + } + } + + /** + * Best-effort rollback: restore both sides, throw the earliest failure, and suppress later ones. + */ @Override - protected void rollbackState( - ConfigNodeProcedureEnv configNodeProcedureEnv, SetTTLState setTTLState) - throws IOException, InterruptedException, ProcedureException {} + protected void rollbackState(final ConfigNodeProcedureEnv env, final SetTTLState setTTLState) + throws IOException, InterruptedException, ProcedureException { + if (setTTLState != SetTTLState.UPDATE_DATANODE_CACHE || !previousTTLStateCaptured) { + return; + } + ProcedureException rollbackFailure = null; + try { + rollbackConfigNodeTTL(env); + } catch (ProcedureException e) { + LOGGER.error("Failed to rollback ConfigNode ttl state.", e); + rollbackFailure = e; + } + try { + rollbackDataNodeTTL(env); + } catch (ProcedureException e) { + LOGGER.error("Failed to rollback DataNode ttl cache.", e); + if (rollbackFailure == null) { + rollbackFailure = e; + } else { + rollbackFailure.addSuppressed(e); + } + } + if (rollbackFailure != null) { + throw rollbackFailure; + } + } + + @Override + protected boolean isRollbackSupported(final SetTTLState state) { + return state == SetTTLState.UPDATE_DATANODE_CACHE; + } @Override protected SetTTLState getState(int stateId) { @@ -144,7 +293,7 @@ protected int getStateId(SetTTLState setTTLState) { @Override protected SetTTLState getInitialState() { - return SetTTLState.SET_CONFIGNODE_TTL; + return SetTTLState.CAPTURE_PREVIOUS_TTL; } @Override @@ -155,14 +304,25 @@ public void serialize(DataOutputStream stream) throws IOException { : ProcedureType.SET_TTL_PROCEDURE.getTypeCode()); super.serialize(stream); ReadWriteIOUtils.write(plan.serializeToByteBuffer(), stream); + stream.writeBoolean(previousTTLStateCaptured); + stream.writeLong(previousTTL); + stream.writeLong(previousDatabaseWildcardTTL); } @Override public void deserialize(ByteBuffer byteBuffer) { super.deserialize(byteBuffer); try { - ReadWriteIOUtils.readInt(byteBuffer); + final int length = ReadWriteIOUtils.readInt(byteBuffer); + final int position = byteBuffer.position(); this.plan = (SetTTLPlan) ConfigPhysicalPlan.Factory.create(byteBuffer); + // The serialized plan buffer may include padding; skip to the actual payload end. + byteBuffer.position(position + length); + if (byteBuffer.remaining() >= ROLLBACK_STATE_BYTES) { + this.previousTTLStateCaptured = byteBuffer.get() != 0; + this.previousTTL = byteBuffer.getLong(); + this.previousDatabaseWildcardTTL = byteBuffer.getLong(); + } } catch (IOException e) { LOGGER.error("IO error when deserialize setTTL plan.", e); } @@ -176,12 +336,21 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - return this.plan.equals(((SetTTLProcedure) o).plan) - && this.isGeneratedByPipe == (((SetTTLProcedure) o).isGeneratedByPipe); + final SetTTLProcedure that = (SetTTLProcedure) o; + return this.isGeneratedByPipe == that.isGeneratedByPipe + && this.previousTTL == that.previousTTL + && this.previousDatabaseWildcardTTL == that.previousDatabaseWildcardTTL + && this.previousTTLStateCaptured == that.previousTTLStateCaptured + && this.plan.equals(that.plan); } @Override public int hashCode() { - return Objects.hash(plan, isGeneratedByPipe); + return Objects.hash( + plan, + isGeneratedByPipe, + previousTTL, + previousDatabaseWildcardTTL, + previousTTLStateCaptured); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java index 4a48ebdd35d6a..6d468908ed71a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedure.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; +import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta; import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; @@ -86,10 +87,17 @@ protected boolean executeFromValidate(final ConfigNodeProcedureEnv env) throws SubscriptionException { LOGGER.info("CreateSubscriptionProcedure: executeFromValidate"); + alterConsumerGroupProcedure = null; + createPipeProcedures = new ArrayList<>(); + subscriptionInfo.get().validateBeforeSubscribe(subscribeReq); // Construct AlterConsumerGroupProcedure + final String consumerId = subscribeReq.getConsumerId(); final String consumerGroupId = subscribeReq.getConsumerGroupId(); + final ConsumerGroupMeta consumerGroupMeta = + subscriptionInfo.get().getConsumerGroupMeta(consumerGroupId); + final ConsumerMeta consumerMeta = consumerGroupMeta.getConsumerMeta(consumerId); final ConsumerGroupMeta updatedConsumerGroupMeta = subscriptionInfo.get().deepCopyConsumerGroupMeta(consumerGroupId); updatedConsumerGroupMeta.addSubscription( @@ -110,7 +118,9 @@ protected boolean executeFromValidate(final ConfigNodeProcedureEnv env) new CreatePipeProcedureV2( new TCreatePipeReq() .setPipeName(pipeName) - .setExtractorAttributes(topicMeta.generateExtractorAttributes()) + .setExtractorAttributes( + topicMeta.generateExtractorAttributes( + consumerMeta.getUsername(), consumerMeta.getSubscriptionAuthPassword())) .setProcessorAttributes(topicMeta.generateProcessorAttributes()) .setConnectorAttributes(topicMeta.generateConnectorAttributes(consumerGroupId)), pipeTaskInfo)); @@ -153,8 +163,7 @@ protected void executeFromOperateOnConfigNodes(final ConfigNodeProcedureEnv env) response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); response.setMessage(e.getMessage()); } - if (response.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() - && response.getSubStatusSize() > 0) { + if (response.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new SubscriptionException( String.format( "Failed to create subscription with request %s on config nodes, because %s", diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java index 6741a6c1e2a84..6f668f29c5dac 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedure.java @@ -85,6 +85,9 @@ protected boolean executeFromValidate(final ConfigNodeProcedureEnv env) throws SubscriptionException { LOGGER.info("DropSubscriptionProcedure: executeFromValidate"); + alterConsumerGroupProcedure = null; + dropPipeProcedures = new ArrayList<>(); + subscriptionInfo.get().validateBeforeUnsubscribe(unsubscribeReq); // Construct AlterConsumerGroupProcedure @@ -141,8 +144,7 @@ protected void executeFromOperateOnConfigNodes(final ConfigNodeProcedureEnv env) response = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); response.setMessage(e.getMessage()); } - if (response.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() - && response.getSubStatusSize() > 0) { + if (response.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new SubscriptionException( String.format( "Failed to drop subscription with request %s on config nodes, because %s", diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/trigger/CreateTriggerProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/trigger/CreateTriggerProcedure.java index 74f4112075a7e..548ba1ee22a35 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/trigger/CreateTriggerProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/trigger/CreateTriggerProcedure.java @@ -304,7 +304,7 @@ public boolean equals(Object that) { if (that instanceof CreateTriggerProcedure) { CreateTriggerProcedure thatProc = (CreateTriggerProcedure) that; return thatProc.getProcId() == this.getProcId() - && thatProc.getCurrentState().equals(this.getCurrentState()) + && Objects.equals(thatProc.getCurrentState(), this.getCurrentState()) && thatProc.getCycles() == this.getCycles() && thatProc.isGeneratedByPipe == this.isGeneratedByPipe && thatProc.triggerInformation.equals(this.triggerInformation); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/trigger/DropTriggerProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/trigger/DropTriggerProcedure.java index 19bfcdc30d2ba..958000d2b3fd6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/trigger/DropTriggerProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/trigger/DropTriggerProcedure.java @@ -178,7 +178,7 @@ public boolean equals(Object that) { if (that instanceof DropTriggerProcedure) { DropTriggerProcedure thatProc = (DropTriggerProcedure) that; return thatProc.getProcId() == this.getProcId() - && thatProc.getCurrentState().equals(this.getCurrentState()) + && Objects.equals(thatProc.getCurrentState(), this.getCurrentState()) && thatProc.getCycles() == this.getCycles() && thatProc.isGeneratedByPipe == this.isGeneratedByPipe && (thatProc.triggerName).equals(this.triggerName); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/LockQueue.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/LockQueue.java index 832e339c0aede..e2f5935a909ab 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/LockQueue.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/LockQueue.java @@ -45,7 +45,15 @@ public boolean releaseLock(Procedure procedure) { return true; } - public void waitProcedure(Procedure procedure) { + public void waitProcedure(Procedure procedure, ProcedureScheduler procedureScheduler) { + if (lockOwnerProcedure == null) { + procedureScheduler.addFront(procedure); + return; + } + if (deque.stream() + .anyMatch(waitingProcedure -> waitingProcedure.getProcId() == procedure.getProcId())) { + return; + } deque.addLast(procedure); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/SimpleProcedureScheduler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/SimpleProcedureScheduler.java index 3cd5ceacf4b6d..94b6f3119308a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/SimpleProcedureScheduler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/SimpleProcedureScheduler.java @@ -22,6 +22,7 @@ import org.apache.iotdb.confignode.procedure.Procedure; import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.locks.ReentrantLock; /** Simple scheduler for procedures */ public class SimpleProcedureScheduler extends AbstractProcedureScheduler { @@ -48,6 +49,7 @@ public void clear() { schedLock(); try { runnables.clear(); + waitings.clear(); } finally { schedUnlock(); } @@ -68,12 +70,37 @@ public int queueSize() { return runnables.size(); } - public void addWaiting(Procedure proc) { - waitings.add(proc); + public void waitProcedure(Procedure proc, ReentrantLock lock) { + boolean signal = false; + schedLock(); + try { + if (lock.isLocked()) { + waitings.add(proc); + } else { + runnables.addFirst(proc); + signal = true; + } + } finally { + schedUnlock(); + } + if (signal) { + signalAll(); + } } - public void releaseWaiting() { - runnables.addAll(waitings); - waitings.clear(); + public void releaseWaiting(ReentrantLock lock) { + boolean signal; + schedLock(); + try { + lock.unlock(); + signal = !waitings.isEmpty(); + runnables.addAll(waitings); + waitings.clear(); + } finally { + schedUnlock(); + } + if (signal) { + signalAll(); + } } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/SetTTLState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/SetTTLState.java index fbdc026fc709b..4dd3063ea3f62 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/SetTTLState.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/SetTTLState.java @@ -19,6 +19,8 @@ package org.apache.iotdb.confignode.procedure.state.schema; public enum SetTTLState { + // Keep existing state ordinals stable for persisted procedures. SET_CONFIGNODE_TTL, - UPDATE_DATANODE_CACHE + UPDATE_DATANODE_CACHE, + CAPTURE_PREVIOUS_TTL } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index be2ba4b696017..1c73fe323cec3 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -95,6 +95,7 @@ import org.apache.iotdb.confignode.consensus.request.write.pipe.task.DropPipePlanV2; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.OperateMultiplePipesPlanV2; import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusPlanV2; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusWithStoppedByRuntimeExceptionPlanV2; import org.apache.iotdb.confignode.consensus.request.write.procedure.DeleteProcedurePlan; import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan; import org.apache.iotdb.confignode.consensus.request.write.quota.SetSpaceQuotaPlan; @@ -902,6 +903,28 @@ public void SetPipeStatusPlanV2Test() throws IOException { Assert.assertEquals(setPipeStatusPlanV2.getPipeStatus(), setPipeStatusPlanV21.getPipeStatus()); } + @Test + public void SetPipeStatusWithStoppedByRuntimeExceptionPlanV2Test() throws IOException { + final SetPipeStatusWithStoppedByRuntimeExceptionPlanV2 + setPipeStatusWithStoppedByRuntimeExceptionPlanV2 = + new SetPipeStatusWithStoppedByRuntimeExceptionPlanV2( + "pipe", org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus.STOPPED, true); + final SetPipeStatusWithStoppedByRuntimeExceptionPlanV2 + setPipeStatusWithStoppedByRuntimeExceptionPlanV21 = + (SetPipeStatusWithStoppedByRuntimeExceptionPlanV2) + ConfigPhysicalPlan.Factory.create( + setPipeStatusWithStoppedByRuntimeExceptionPlanV2.serializeToByteBuffer()); + Assert.assertEquals( + setPipeStatusWithStoppedByRuntimeExceptionPlanV2.getPipeName(), + setPipeStatusWithStoppedByRuntimeExceptionPlanV21.getPipeName()); + Assert.assertEquals( + setPipeStatusWithStoppedByRuntimeExceptionPlanV2.getPipeStatus(), + setPipeStatusWithStoppedByRuntimeExceptionPlanV21.getPipeStatus()); + Assert.assertEquals( + setPipeStatusWithStoppedByRuntimeExceptionPlanV2.isStoppedByRuntimeException(), + setPipeStatusWithStoppedByRuntimeExceptionPlanV21.isStoppedByRuntimeException()); + } + @Test public void DropPipePlanV2Test() throws IOException { DropPipePlanV2 dropPipePlanV2 = new DropPipePlanV2("demo"); @@ -944,7 +967,7 @@ public void OperateMultiplePipesPlanV2Test() throws IOException { new SetPipeStatusPlanV2( "testSet", org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus.RUNNING); - List subPlans = new ArrayList<>(); + final List subPlans = new ArrayList<>(); subPlans.add(createPipePlanV2); subPlans.add(alterPipePlanV2); subPlans.add(dropPipePlanV2); @@ -1190,7 +1213,7 @@ public void UpdateTriggerStateInTablePlanTest() throws IOException { @Test public void ActiveCQPlanTest() throws IOException { - ActiveCQPlan activeCQPlan0 = new ActiveCQPlan("testCq", "testCq_md5"); + ActiveCQPlan activeCQPlan0 = new ActiveCQPlan("testCq", "testCqToken"); ActiveCQPlan activeCQPlan1 = (ActiveCQPlan) ConfigPhysicalPlan.Factory.create(activeCQPlan0.serializeToByteBuffer()); @@ -1213,7 +1236,7 @@ public void AddCQPlanTest() throws IOException { "create cq testCq1 BEGIN select s1 into root.backup.d1.s1 from root.sg.d1 END", "Asia", "root"), - "testCq1_md5", + "testCq1Token", executionTime); AddCQPlan addCQPlan1 = (AddCQPlan) ConfigPhysicalPlan.Factory.create(addCQPlan0.serializeToByteBuffer()); @@ -1228,7 +1251,7 @@ public void DropCQPlanTest() throws IOException { (DropCQPlan) ConfigPhysicalPlan.Factory.create(dropCQPlan0.serializeToByteBuffer()); Assert.assertEquals(dropCQPlan0, dropCQPlan1); - dropCQPlan0 = new DropCQPlan("testCq1", "testCq1_md5"); + dropCQPlan0 = new DropCQPlan("testCq1", "testCq1Token"); dropCQPlan1 = (DropCQPlan) ConfigPhysicalPlan.Factory.create(dropCQPlan0.serializeToByteBuffer()); Assert.assertEquals(dropCQPlan0, dropCQPlan1); @@ -1237,7 +1260,7 @@ public void DropCQPlanTest() throws IOException { @Test public void UpdateCQLastExecTimePlanTest() throws IOException { UpdateCQLastExecTimePlan updateCQLastExecTimePlan0 = - new UpdateCQLastExecTimePlan("testCq", System.currentTimeMillis(), "testCq_md5"); + new UpdateCQLastExecTimePlan("testCq", System.currentTimeMillis(), "testCqToken"); UpdateCQLastExecTimePlan updateCQLastExecTimePlan1 = (UpdateCQLastExecTimePlan) ConfigPhysicalPlan.Factory.create(updateCQLastExecTimePlan0.serializeToByteBuffer()); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java new file mode 100644 index 0000000000000..a0bc5a523ba70 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/cq/CQManagerTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.confignode.cq; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.cq.TimeoutPolicy; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.cq.CQManager; +import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; +import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertTrue; + +public class CQManagerTest { + + @SuppressWarnings("unchecked") + @Test + public void dropCQShouldCancelLocallyScheduledTask() throws Exception { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + ConsensusManager consensusManager = Mockito.mock(ConsensusManager.class); + Mockito.when(configManager.getConsensusManager()).thenReturn(consensusManager); + Mockito.when(consensusManager.write(Mockito.any())) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + CQManager cqManager = new CQManager(configManager); + ScheduledFuture future = Mockito.mock(ScheduledFuture.class); + CQScheduleTask task = newScheduledTask(configManager, future, "token"); + + try { + assertTrue(cqManager.markCQLocallyScheduled("testCq", "token", task)); + task.submitSelf(); + cqManager.dropCQ(new TDropCQReq("testCq")); + + Mockito.verify(future).cancel(false); + } finally { + cqManager.stopCQScheduler(); + } + } + + @SuppressWarnings("unchecked") + @Test + public void newTokenShouldCancelPreviousLocallyScheduledTask() { + ConfigManager configManager = Mockito.mock(ConfigManager.class); + CQManager cqManager = new CQManager(configManager); + ScheduledFuture previousFuture = Mockito.mock(ScheduledFuture.class); + CQScheduleTask previousTask = newScheduledTask(configManager, previousFuture, "previousToken"); + ScheduledFuture currentFuture = Mockito.mock(ScheduledFuture.class); + CQScheduleTask currentTask = newScheduledTask(configManager, currentFuture, "currentToken"); + + try { + assertTrue(cqManager.markCQLocallyScheduled("testCq", "previousToken", previousTask)); + previousTask.submitSelf(); + assertTrue(cqManager.markCQLocallyScheduled("testCq", "currentToken", currentTask)); + + Mockito.verify(previousFuture).cancel(false); + } finally { + cqManager.stopCQScheduler(); + } + } + + @SuppressWarnings("unchecked") + private CQScheduleTask newScheduledTask( + ConfigManager configManager, ScheduledFuture scheduledFuture, String cqToken) { + ScheduledExecutorService executor = Mockito.mock(ScheduledExecutorService.class); + Mockito.when( + executor.schedule( + Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.any(TimeUnit.class))) + .thenReturn((ScheduledFuture) scheduledFuture); + return new CQScheduleTask( + "testCq", + 1000, + 0, + 1000, + TimeoutPolicy.BLOCKED, + "select s1 into root.backup.d1.s1 from root.sg.d1", + cqToken, + "Asia", + "root", + executor, + configManager, + System.currentTimeMillis() + 10_000); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java new file mode 100644 index 0000000000000..d5a46d42c84a4 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeatParserTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.pipe.coordinator.runtime.heartbeat; + +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.ProcedureManager; +import org.apache.iotdb.confignode.manager.node.NodeManager; +import org.apache.iotdb.confignode.manager.pipe.coordinator.PipeManager; +import org.apache.iotdb.confignode.manager.pipe.coordinator.runtime.PipeRuntimeCoordinator; +import org.apache.iotdb.confignode.manager.pipe.coordinator.task.PipeTaskCoordinator; +import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class PipeHeartbeatParserTest { + + private boolean originalSeparatedPipeHeartbeatEnabled; + + @Before + public void setUp() { + originalSeparatedPipeHeartbeatEnabled = + CommonDescriptor.getInstance().getConfig().isSeperatedPipeHeartbeatEnabled(); + } + + @After + public void tearDown() { + CommonDescriptor.getInstance() + .getConfig() + .setSeperatedPipeHeartbeatEnabled(originalSeparatedPipeHeartbeatEnabled); + } + + @Test + public void testParseHeartbeatCountsOnlyDataNodesWhenSeparatedHeartbeatDisabled() + throws Exception { + CommonDescriptor.getInstance().getConfig().setSeperatedPipeHeartbeatEnabled(false); + + final ParserTestContext context = createParserTestContext(2); + setMetaChangeFlags(context.parser, true, false); + + context.parser.parseHeartbeat(1, emptyHeartbeat()); + verify(context.procedureManager, never()).pipeHandleMetaChange(anyBoolean(), anyBoolean()); + + context.parser.parseHeartbeat(2, emptyHeartbeat()); + verify(context.procedureManager, times(1)).pipeHandleMetaChange(true, false); + } + + @Test + public void testParseHeartbeatCountsLocalConfigNodeWhenSeparatedHeartbeatEnabled() + throws Exception { + CommonDescriptor.getInstance().getConfig().setSeperatedPipeHeartbeatEnabled(true); + + final ParserTestContext context = createParserTestContext(2); + setMetaChangeFlags(context.parser, true, false); + + context.parser.parseHeartbeat(1, emptyHeartbeat()); + context.parser.parseHeartbeat(2, emptyHeartbeat()); + verify(context.procedureManager, never()).pipeHandleMetaChange(anyBoolean(), anyBoolean()); + + context.parser.parseHeartbeat(3, emptyHeartbeat()); + verify(context.procedureManager, times(1)).pipeHandleMetaChange(true, false); + } + + @Test + public void testParseHeartbeatKeepsPendingFlagsWhenProcedureSubmissionFails() throws Exception { + CommonDescriptor.getInstance().getConfig().setSeperatedPipeHeartbeatEnabled(false); + + final ParserTestContext context = createParserTestContext(2); + when(context.procedureManager.pipeHandleMetaChange(anyBoolean(), anyBoolean())) + .thenReturn(false, true); + setMetaChangeFlags(context.parser, true, false); + + context.parser.parseHeartbeat(1, emptyHeartbeat()); + verify(context.procedureManager, never()).pipeHandleMetaChange(anyBoolean(), anyBoolean()); + + context.parser.parseHeartbeat(2, emptyHeartbeat()); + verify(context.procedureManager, times(1)).pipeHandleMetaChange(true, false); + + context.parser.parseHeartbeat(3, emptyHeartbeat()); + verify(context.procedureManager, times(1)).pipeHandleMetaChange(true, false); + + context.parser.parseHeartbeat(4, emptyHeartbeat()); + verify(context.procedureManager, times(2)).pipeHandleMetaChange(true, false); + } + + private ParserTestContext createParserTestContext(final int registeredDataNodeCount) { + final ConfigManager configManager = Mockito.mock(ConfigManager.class); + final NodeManager nodeManager = Mockito.mock(NodeManager.class); + final ProcedureManager procedureManager = Mockito.mock(ProcedureManager.class); + final PipeManager pipeManager = Mockito.mock(PipeManager.class); + final PipeRuntimeCoordinator pipeRuntimeCoordinator = + Mockito.mock(PipeRuntimeCoordinator.class); + final PipeTaskCoordinator pipeTaskCoordinator = Mockito.mock(PipeTaskCoordinator.class); + final ExecutorService procedureSubmitter = Mockito.mock(ExecutorService.class); + + when(configManager.getNodeManager()).thenReturn(nodeManager); + when(configManager.getProcedureManager()).thenReturn(procedureManager); + when(configManager.getPipeManager()).thenReturn(pipeManager); + when(nodeManager.getRegisteredDataNodeCount()).thenReturn(registeredDataNodeCount); + when(pipeManager.getPipeRuntimeCoordinator()).thenReturn(pipeRuntimeCoordinator); + when(pipeManager.getPipeTaskCoordinator()).thenReturn(pipeTaskCoordinator); + when(pipeRuntimeCoordinator.getProcedureSubmitter()).thenReturn(procedureSubmitter); + when(pipeTaskCoordinator.tryLock()).thenReturn(new AtomicReference<>(new PipeTaskInfo())); + when(procedureManager.pipeHandleMetaChange(anyBoolean(), anyBoolean())).thenReturn(true); + Mockito.doAnswer( + invocation -> { + ((Runnable) invocation.getArgument(0)).run(); + return CompletableFuture.completedFuture(null); + }) + .when(procedureSubmitter) + .submit(any(Runnable.class)); + + return new ParserTestContext(new PipeHeartbeatParser(configManager), procedureManager); + } + + private void setMetaChangeFlags( + final PipeHeartbeatParser parser, + final boolean needWriteConsensusOnConfigNodes, + final boolean needPushPipeMetaToDataNodes) + throws Exception { + setAtomicBooleanField( + parser, "needWriteConsensusOnConfigNodes", needWriteConsensusOnConfigNodes); + setAtomicBooleanField(parser, "needPushPipeMetaToDataNodes", needPushPipeMetaToDataNodes); + } + + private void setAtomicBooleanField( + final PipeHeartbeatParser parser, final String fieldName, final boolean value) + throws Exception { + final Field field = PipeHeartbeatParser.class.getDeclaredField(fieldName); + field.setAccessible(true); + ((AtomicBoolean) field.get(parser)).set(value); + } + + private PipeHeartbeat emptyHeartbeat() { + return new PipeHeartbeat(Collections.emptyList(), null, null, null); + } + + private static class ParserTestContext { + private final PipeHeartbeatParser parser; + private final ProcedureManager procedureManager; + + private ParserTestContext( + final PipeHeartbeatParser parser, final ProcedureManager procedureManager) { + this.parser = parser; + this.procedureManager = procedureManager; + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinatorLockTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinatorLockTest.java new file mode 100644 index 0000000000000..74d5d821d76c3 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinatorLockTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.pipe.coordinator.task; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public class PipeTaskCoordinatorLockTest { + + @Test + public void testInterruptedThreadDoesNotAcquireWithoutPermit() throws Exception { + PipeTaskCoordinatorLock lock = new PipeTaskCoordinatorLock(); + lock.lock(); + + CountDownLatch waiting = new CountDownLatch(1); + AtomicBoolean acquired = new AtomicBoolean(false); + Thread thread = + new Thread( + () -> { + Thread.currentThread().interrupt(); + waiting.countDown(); + lock.lock(); + acquired.set(true); + lock.unlock(); + }); + thread.start(); + + Assert.assertTrue(waiting.await(3, TimeUnit.SECONDS)); + TimeUnit.MILLISECONDS.sleep(200); + Assert.assertFalse(acquired.get()); + + lock.unlock(); + thread.join(TimeUnit.SECONDS.toMillis(3)); + + Assert.assertFalse(thread.isAlive()); + Assert.assertTrue(acquired.get()); + Assert.assertFalse(lock.isLocked()); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java index bfadd3d05e8df..8bf2d12cd5941 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/CQInfoTest.java @@ -18,9 +18,14 @@ */ package org.apache.iotdb.confignode.persistence; +import org.apache.iotdb.confignode.consensus.request.read.cq.ShowCQPlan; import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan; +import org.apache.iotdb.confignode.consensus.request.write.cq.DropCQPlan; +import org.apache.iotdb.confignode.consensus.request.write.cq.UpdateCQLastExecTimePlan; +import org.apache.iotdb.confignode.consensus.response.cq.ShowCQResp; import org.apache.iotdb.confignode.persistence.cq.CQInfo; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.commons.io.FileUtils; import org.apache.thrift.TException; @@ -70,7 +75,7 @@ public void testSnapshot() throws TException, IOException { "create cq testCq1 BEGIN select s1 into root.backup.d1.s1 from root.sg.d1 END", "Asia", "root"), - "testCq1_md5", + "testCq1Token", executionTime); cqInfo.addCQ(addCQPlan); @@ -89,7 +94,7 @@ public void testSnapshot() throws TException, IOException { "create cq testCq2 BEGIN select s1 into root.backup.d2.s1 from root.sg.d2 END", "Asia", "root"), - "testCq2_md5", + "testCq2Token", executionTime); cqInfo.addCQ(addCQPlan); @@ -99,4 +104,59 @@ public void testSnapshot() throws TException, IOException { Assert.assertEquals(cqInfo, actualCQInfo); } + + @Test + public void testOldCallbackCannotTouchRecreatedCQ() throws Exception { + long executionTime = System.currentTimeMillis(); + TCreateCQReq req = + new TCreateCQReq( + "testCq3", + 1000, + 0, + 1000, + 0, + (byte) 0, + "select s1 into root.backup.d3.s1 from root.sg.d3", + "create cq testCq3 BEGIN select s1 into root.backup.d3.s1 from root.sg.d3 END", + "Asia", + "root"); + + cqInfo.addCQ(new AddCQPlan(req, "oldToken", executionTime)); + cqInfo.dropCQ(new DropCQPlan("testCq3")); + cqInfo.addCQ(new AddCQPlan(req, "newToken", executionTime)); + + Assert.assertEquals( + TSStatusCode.NO_SUCH_CQ.getStatusCode(), + cqInfo.updateCQLastExecutionTime( + new UpdateCQLastExecTimePlan("testCq3", executionTime + 1000, "oldToken")) + .code); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + cqInfo.updateCQLastExecutionTime( + new UpdateCQLastExecTimePlan("testCq3", executionTime + 1000, "newToken")) + .code); + } + + @Test + public void testShowCQCanFilterByCQId() throws Exception { + long executionTime = System.currentTimeMillis(); + TCreateCQReq req = + new TCreateCQReq( + "testCq4", + 1000, + 0, + 1000, + 0, + (byte) 0, + "select s1 into root.backup.d4.s1 from root.sg.d4", + "create cq testCq4 BEGIN select s1 into root.backup.d4.s1 from root.sg.d4 END", + "Asia", + "root"); + cqInfo.addCQ(new AddCQPlan(req, "testCq4Token", executionTime)); + + ShowCQResp showCQResp = cqInfo.showCQ(new ShowCQPlan("testCq4")); + + Assert.assertEquals(1, showCQResp.getCqList().size()); + Assert.assertEquals("testCq4", showCQResp.getCqList().get(0).getCqId()); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/TTLInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/TTLInfoTest.java index d5deb1d2b2c1e..977f1dde53ca1 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/TTLInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/TTLInfoTest.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.ttl.TTLCache; import org.apache.iotdb.confignode.consensus.request.read.ttl.ShowTTLPlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; import org.apache.iotdb.confignode.consensus.response.ttl.ShowTTLResp; @@ -50,6 +51,7 @@ public class TTLInfoTest { private final File snapshotDir = new File(BASE_OUTPUT_PATH, "ttlInfo-snapshot"); private final long ttl = 123435565323L; private long[] originTTLArr; + private int originTTlRuleCapacity; @Before public void setup() throws IOException { @@ -57,6 +59,7 @@ public void setup() throws IOException { snapshotDir.mkdirs(); } originTTLArr = CommonDescriptor.getInstance().getConfig().getTierTTLInMs(); + originTTlRuleCapacity = CommonDescriptor.getInstance().getConfig().getTTlRuleCapacity(); long[] ttlArr = new long[2]; ttlArr[0] = 10000000L; ttlArr[1] = ttl; @@ -70,6 +73,7 @@ public void tearDown() throws IOException { FileUtils.deleteDirectory(snapshotDir); } CommonDescriptor.getInstance().getConfig().setTierTTLInMs(originTTLArr); + CommonDescriptor.getInstance().getConfig().setTTlRuleCapacity(originTTlRuleCapacity); } @Test @@ -208,6 +212,17 @@ public void testSetAndUnsetTTL() throws IllegalPathException { Assert.assertEquals(4, ttlInfo.getTTLCount()); } + @Test + public void testGetTTLReturnsExactPathTTL() throws IllegalPathException { + PartialPath path = new PartialPath("root.test.db1.**"); + ttlInfo.setTTL(new SetTTLPlan(Arrays.asList(path.getNodes()), 121322323L)); + + Assert.assertEquals(121322323L, ttlInfo.getTTL(path.getNodes())); + Assert.assertEquals( + TTLCache.NULL_TTL, ttlInfo.getTTL(new PartialPath("root.test.db1").getNodes())); + Assert.assertEquals(Long.MAX_VALUE, ttlInfo.getTTL(new PartialPath("root.**").getNodes())); + } + @Test public void testUnsetNonExistTTL() throws IllegalPathException { assertEquals( @@ -241,10 +256,61 @@ public void testTooManyTTL() { final TSStatus status = ttlInfo.setTTL(setTTLPlan); assertEquals(TSStatusCode.OVERSIZE_TTL.getStatusCode(), status.code); assertEquals( - "The number of TTL rules has reached the limit (1000). Please delete some existing rules first.", + "The number of TTL rules has reached the limit " + + "(capacity: 1000, requested total: 1001). Please delete some existing rules first.", status.message); } + @Test + public void testUpdateExistingTTLWhenCapacityIsReached() { + CommonDescriptor.getInstance().getConfig().setTTlRuleCapacity(2); + + SetTTLPlan setTTLPlan = + new SetTTLPlan(PathNodesGenerator.splitPathToNodes("root.sg1.d1.**"), 1000); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), ttlInfo.setTTL(setTTLPlan).code); + assertEquals(2, ttlInfo.getTTLCount()); + + setTTLPlan = new SetTTLPlan(PathNodesGenerator.splitPathToNodes("root.sg1.d1.**"), 2000); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), ttlInfo.setTTL(setTTLPlan).code); + assertEquals(2, ttlInfo.getTTLCount()); + assertEquals( + Long.valueOf(2000), + ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap().get("root.sg1.d1.**")); + } + + @Test + public void testUpdateExistingTTLWhenCurrentStateIsAlreadyOversize() { + SetTTLPlan setTTLPlan = + new SetTTLPlan(PathNodesGenerator.splitPathToNodes("root.sg1.d1.**"), 1000); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), ttlInfo.setTTL(setTTLPlan).code); + assertEquals(2, ttlInfo.getTTLCount()); + + CommonDescriptor.getInstance().getConfig().setTTlRuleCapacity(1); + + setTTLPlan = new SetTTLPlan(PathNodesGenerator.splitPathToNodes("root.sg1.d1.**"), 2000); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), ttlInfo.setTTL(setTTLPlan).code); + assertEquals(2, ttlInfo.getTTLCount()); + assertEquals( + Long.valueOf(2000), + ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap().get("root.sg1.d1.**")); + } + + @Test + public void testDatabaseTTLShouldReserveTwoSlots() { + CommonDescriptor.getInstance().getConfig().setTTlRuleCapacity(2); + + SetTTLPlan setTTLPlan = new SetTTLPlan(PathNodesGenerator.splitPathToNodes("root.sg1"), 1000); + setTTLPlan.setDataBase(true); + + final TSStatus status = ttlInfo.setTTL(setTTLPlan); + assertEquals(TSStatusCode.OVERSIZE_TTL.getStatusCode(), status.code); + assertEquals(1, ttlInfo.getTTLCount()); + assertEquals(1, ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap().size()); + assertEquals( + Long.valueOf(Long.MAX_VALUE), + ttlInfo.showTTL(new ShowTTLPlan()).getPathTTLMap().get("root.**")); + } + @Test public void testSnapshot() throws TException, IOException, IllegalPathException { // set ttl diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/UDFInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/UDFInfoTest.java index 708ea779fad72..c15cd0642288e 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/UDFInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/UDFInfoTest.java @@ -18,12 +18,12 @@ */ package org.apache.iotdb.confignode.persistence; -import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.udf.UDFInformation; import org.apache.iotdb.confignode.consensus.request.write.function.CreateFunctionPlan; +import org.apache.iotdb.confignode.consensus.request.write.function.DropFunctionPlan; +import org.apache.iotdb.udf.api.exception.UDFManagementException; import org.apache.commons.io.FileUtils; -import org.apache.thrift.TException; import org.apache.tsfile.utils.Binary; import org.junit.AfterClass; import org.junit.Assert; @@ -37,6 +37,10 @@ public class UDFInfoTest { + private static final String SHARED_JAR_NAME = "shared.jar"; + private static final String SHARED_JAR_MD5 = "12345"; + private static final String DIFFERENT_JAR_MD5 = "54321"; + private static UDFInfo udfInfo; private static UDFInfo udfInfoSaveBefore; private static final File snapshotDir = new File(BASE_OUTPUT_PATH, "snapshot"); @@ -59,18 +63,42 @@ public static void cleanup() throws IOException { } @Test - public void testSnapshot() throws TException, IOException, IllegalPathException { - UDFInformation udfInformation = - new UDFInformation("test1", "test1", false, true, "test1.jar", "12345"); - CreateFunctionPlan createFunctionPlan = - new CreateFunctionPlan(udfInformation, new Binary(new byte[] {1, 2, 3})); - udfInfo.addUDFInTable(createFunctionPlan); - udfInfoSaveBefore.addUDFInTable(createFunctionPlan); + public void testDropOneSharedJarReferenceKeepsJarMetadata() + throws IOException, UDFManagementException { + clearUdfInfos(); + + udfInfo.addUDFInTable(createFunctionPlan("test1", SHARED_JAR_NAME, SHARED_JAR_MD5, true)); + udfInfo.addUDFInTable(createFunctionPlan("test2", SHARED_JAR_NAME, SHARED_JAR_MD5, false)); + + udfInfo.dropFunction(new DropFunctionPlan("TEST1")); + + Assert.assertFalse(udfInfo.needToSaveJar(SHARED_JAR_NAME)); + Assert.assertEquals(1, udfInfo.getRawExistedJarToMD5().size()); + Assert.assertEquals(SHARED_JAR_MD5, udfInfo.getRawExistedJarToMD5().get(SHARED_JAR_NAME)); + + udfInfo.validate("TEST3", SHARED_JAR_NAME, SHARED_JAR_MD5); + try { + udfInfo.validate("TEST3", SHARED_JAR_NAME, DIFFERENT_JAR_MD5); + Assert.fail("Expected shared jar conflict after dropping only one referenced UDF."); + } catch (UDFManagementException e) { + Assert.assertTrue(e.getMessage().contains("different MD5")); + } + } + + @Test + public void testSnapshotRebuildsSharedJarReferences() throws IOException { + clearUdfInfos(); + FileUtils.cleanDirectory(snapshotDir); + + CreateFunctionPlan createFunctionPlan1 = + createFunctionPlan("test1", SHARED_JAR_NAME, SHARED_JAR_MD5, true); + CreateFunctionPlan createFunctionPlan2 = + createFunctionPlan("test2", SHARED_JAR_NAME, SHARED_JAR_MD5, false); - udfInformation = new UDFInformation("test2", "test2", false, true, "test2.jar", "123456"); - createFunctionPlan = new CreateFunctionPlan(udfInformation, new Binary(new byte[] {1, 2, 3})); - udfInfo.addUDFInTable(createFunctionPlan); - udfInfoSaveBefore.addUDFInTable(createFunctionPlan); + udfInfo.addUDFInTable(createFunctionPlan1); + udfInfo.addUDFInTable(createFunctionPlan2); + udfInfoSaveBefore.addUDFInTable(createFunctionPlan1); + udfInfoSaveBefore.addUDFInTable(createFunctionPlan2); udfInfo.processTakeSnapshot(snapshotDir); udfInfo.clear(); @@ -78,5 +106,22 @@ public void testSnapshot() throws TException, IOException, IllegalPathException Assert.assertEquals(udfInfoSaveBefore.getRawExistedJarToMD5(), udfInfo.getRawExistedJarToMD5()); Assert.assertEquals(udfInfoSaveBefore.getRawUDFTable(), udfInfo.getRawUDFTable()); + + udfInfo.dropFunction(new DropFunctionPlan("TEST1")); + Assert.assertFalse(udfInfo.needToSaveJar(SHARED_JAR_NAME)); + Assert.assertEquals(SHARED_JAR_MD5, udfInfo.getRawExistedJarToMD5().get(SHARED_JAR_NAME)); + } + + private static void clearUdfInfos() { + udfInfo.clear(); + udfInfoSaveBefore.clear(); + } + + private static CreateFunctionPlan createFunctionPlan( + String functionName, String jarName, String jarMD5, boolean includeJarFile) { + UDFInformation udfInformation = + new UDFInformation(functionName, functionName, false, true, jarName, jarMD5); + return new CreateFunctionPlan( + udfInformation, includeJarFile ? new Binary(new byte[] {1, 2, 3}) : null); } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoAutoRestartTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoAutoRestartTest.java new file mode 100644 index 0000000000000..762c3bf045c5b --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfoAutoRestartTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.persistence.pipe; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusPlanV2; +import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaResp; +import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaRespExceptionMessage; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class PipeTaskInfoAutoRestartTest { + + private static final int DATA_NODE_ID = 1; + + private PipeTaskInfo pipeTaskInfo; + + @Before + public void setUp() { + pipeTaskInfo = new PipeTaskInfo(); + } + + @Test + public void testRecordDataNodePushPipeMetaExceptionsMarksRunningPipeForAutoRestart() { + final String pipeName = "runningPipe"; + createPipe(pipeName, PipeStatus.RUNNING); + + Assert.assertTrue( + pipeTaskInfo.recordDataNodePushPipeMetaExceptions(createErrorRespMap(pipeName))); + + final PipeRuntimeMeta runtimeMeta = + pipeTaskInfo.getPipeMetaByPipeName(pipeName).getRuntimeMeta(); + Assert.assertEquals(PipeStatus.STOPPED, runtimeMeta.getStatus().get()); + Assert.assertTrue(runtimeMeta.getIsStoppedByRuntimeException()); + + Assert.assertTrue(pipeTaskInfo.autoRestart()); + Assert.assertEquals(PipeStatus.RUNNING, runtimeMeta.getStatus().get()); + } + + @Test + public void testRecordDataNodePushPipeMetaExceptionsKeepsUserStoppedPipeOutOfAutoRestart() { + final String pipeName = "stoppedPipe"; + createPipe(pipeName, PipeStatus.STOPPED); + + Assert.assertTrue(pipeTaskInfo.isPipeStoppedByUser(pipeName)); + Assert.assertTrue( + pipeTaskInfo.recordDataNodePushPipeMetaExceptions(createErrorRespMap(pipeName))); + + final PipeRuntimeMeta runtimeMeta = + pipeTaskInfo.getPipeMetaByPipeName(pipeName).getRuntimeMeta(); + Assert.assertEquals(PipeStatus.STOPPED, runtimeMeta.getStatus().get()); + Assert.assertFalse(runtimeMeta.getIsStoppedByRuntimeException()); + Assert.assertTrue(pipeTaskInfo.isPipeStoppedByUser(pipeName)); + + Assert.assertFalse(pipeTaskInfo.autoRestart()); + Assert.assertEquals(PipeStatus.STOPPED, runtimeMeta.getStatus().get()); + } + + private Map createErrorRespMap(final String pipeName) { + final TPushPipeMetaRespExceptionMessage exceptionMessage = + new TPushPipeMetaRespExceptionMessage( + pipeName, "failed to push pipe meta", System.currentTimeMillis()); + final TPushPipeMetaResp resp = + new TPushPipeMetaResp() + .setStatus(new TSStatus(TSStatusCode.PIPE_PUSH_META_ERROR.getStatusCode())) + .setExceptionMessages(Collections.singletonList(exceptionMessage)); + return Collections.singletonMap(DATA_NODE_ID, resp); + } + + private void createPipe(final String pipeName, final PipeStatus initialStatus) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + extractorAttributes.put("extractor", "iotdb-source"); + processorAttributes.put("processor", "do-nothing-processor"); + connectorAttributes.put("connector", "iotdb-thrift-sink"); + + final PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, DATA_NODE_ID); + final ConcurrentMap pipeTasks = new ConcurrentHashMap<>(); + pipeTasks.put(DATA_NODE_ID, pipeTaskMeta); + + final PipeStaticMeta pipeStaticMeta = + new PipeStaticMeta( + pipeName, + System.currentTimeMillis(), + extractorAttributes, + processorAttributes, + connectorAttributes); + final PipeRuntimeMeta pipeRuntimeMeta = new PipeRuntimeMeta(pipeTasks); + pipeTaskInfo.createPipe(new CreatePipePlanV2(pipeStaticMeta, pipeRuntimeMeta)); + + if (PipeStatus.RUNNING.equals(initialStatus)) { + pipeTaskInfo.setPipeStatus(new SetPipeStatusPlanV2(pipeName, PipeStatus.RUNNING)); + } + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/TestLockRegime.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/TestLockRegime.java index 500a51e9e3d07..967611ca8eef1 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/TestLockRegime.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/TestLockRegime.java @@ -19,7 +19,10 @@ package org.apache.iotdb.confignode.procedure; +import org.apache.iotdb.confignode.procedure.entity.NoopProcedure; import org.apache.iotdb.confignode.procedure.entity.SimpleLockProcedure; +import org.apache.iotdb.confignode.procedure.scheduler.LockQueue; +import org.apache.iotdb.confignode.procedure.scheduler.SimpleProcedureScheduler; import org.apache.iotdb.confignode.procedure.util.ProcedureTestUtil; import org.junit.Assert; @@ -43,4 +46,24 @@ public void testAcquireLock() { this.procExecutor, procIdList.stream().mapToLong(Long::longValue).toArray()); Assert.assertEquals(env.lockAcquireSeq.toString(), env.executeSeq.toString()); } + + @Test + public void testLockQueueDoesNotWakeDuplicateProcedure() { + LockQueue lockQueue = new LockQueue(); + SimpleProcedureScheduler scheduler = new SimpleProcedureScheduler(); + scheduler.start(); + + NoopProcedure lockOwner = new NoopProcedure(); + lockOwner.setProcId(0); + Assert.assertTrue(lockQueue.tryLock(lockOwner)); + + NoopProcedure procedure = new NoopProcedure(); + procedure.setProcId(1); + lockQueue.waitProcedure(procedure, scheduler); + lockQueue.waitProcedure(procedure, scheduler); + + Assert.assertEquals(1, lockQueue.wakeWaitingProcedures(scheduler)); + Assert.assertEquals(1, scheduler.size()); + scheduler.stop(); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/TestProcedureExecutor.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/TestProcedureExecutor.java index dce7f2ba5dc48..ba5f635507a4a 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/TestProcedureExecutor.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/TestProcedureExecutor.java @@ -23,11 +23,14 @@ import org.apache.iotdb.confignode.procedure.entity.NoopProcedure; import org.apache.iotdb.confignode.procedure.entity.StuckProcedure; import org.apache.iotdb.confignode.procedure.env.TestProcEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.state.ProcedureState; import org.apache.iotdb.confignode.procedure.util.ProcedureTestUtil; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -96,6 +99,37 @@ public void testWorkerThreadStuck() throws InterruptedException { ProcedureTestUtil.waitForProcedure(procExecutor, busyProcId2); } + @Test + public void testDuplicatedSchedulingDoesNotExecuteConcurrently() throws InterruptedException { + BlockingProcedure blockingProcedure = new BlockingProcedure(); + long procId = procExecutor.submitProcedure(blockingProcedure); + + Assert.assertTrue(blockingProcedure.awaitExecution(30, TimeUnit.SECONDS)); + + procExecutor.getScheduler().addFront(blockingProcedure); + boolean duplicated = blockingProcedure.awaitExecution(3, TimeUnit.SECONDS); + + blockingProcedure.releaseExecutions(duplicated ? 2 : 1); + ProcedureTestUtil.waitForProcedure(procExecutor, procId); + + Assert.assertFalse(duplicated); + Assert.assertEquals(1, blockingProcedure.getExecutionCount()); + } + + @Test + public void testInternalProcedureCanBeDeduplicatedAndRemoved() throws InterruptedException { + CompletingInternalProcedure internalProcedure = new CompletingInternalProcedure(); + + procExecutor.addInternalProcedure(internalProcedure); + procExecutor.addInternalProcedure(internalProcedure); + + Assert.assertTrue(internalProcedure.awaitExecution(30, TimeUnit.SECONDS)); + Assert.assertFalse(internalProcedure.awaitExecution(300, TimeUnit.MILLISECONDS)); + Assert.assertEquals(1, internalProcedure.getExecutionCount()); + + Assert.assertTrue(procExecutor.removeInternalProcedure(internalProcedure)); + } + private int waitThreadCount(final int expectedThreads) { long startTime = System.currentTimeMillis(); while (procExecutor.isRunning() @@ -107,4 +141,62 @@ private int waitThreadCount(final int expectedThreads) { } return procExecutor.getWorkerThreadCount(); } + + private static class BlockingProcedure extends Procedure { + + private final Semaphore entered = new Semaphore(0); + private final Semaphore finish = new Semaphore(0); + private final AtomicInteger executionCount = new AtomicInteger(); + + @Override + protected Procedure[] execute(TestProcEnv env) throws InterruptedException { + executionCount.incrementAndGet(); + entered.release(); + finish.acquire(); + return null; + } + + @Override + protected void rollback(TestProcEnv env) + throws IOException, InterruptedException, ProcedureException { + // No state to roll back. + } + + private boolean awaitExecution(long timeout, TimeUnit unit) throws InterruptedException { + return entered.tryAcquire(timeout, unit); + } + + private void releaseExecutions(int permits) { + finish.release(permits); + } + + private int getExecutionCount() { + return executionCount.get(); + } + } + + private static class CompletingInternalProcedure extends InternalProcedure { + + private final Semaphore entered = new Semaphore(0); + private final AtomicInteger executionCount = new AtomicInteger(); + + private CompletingInternalProcedure() { + super(0); + } + + @Override + protected void periodicExecute(TestProcEnv env) { + executionCount.incrementAndGet(); + entered.release(); + setState(ProcedureState.SUCCESS); + } + + private boolean awaitExecution(long timeout, TimeUnit unit) throws InterruptedException { + return entered.tryAcquire(timeout, unit); + } + + private int getExecutionCount() { + return executionCount.get(); + } + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/entity/SimpleLockProcedure.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/entity/SimpleLockProcedure.java index ce9fea39d5589..42badd700799e 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/entity/SimpleLockProcedure.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/entity/SimpleLockProcedure.java @@ -53,18 +53,21 @@ protected ProcedureLockState acquireLock(TestProcEnv testProcEnv) { return ProcedureLockState.LOCK_ACQUIRED; } - SimpleProcedureScheduler scheduler = (SimpleProcedureScheduler) testProcEnv.getScheduler(); - scheduler.addWaiting(this); System.out.println(procName + " wait for lock."); return ProcedureLockState.LOCK_EVENT_WAIT; } + @Override + protected void waitForLock(TestProcEnv testProcEnv) { + SimpleProcedureScheduler scheduler = (SimpleProcedureScheduler) testProcEnv.getScheduler(); + scheduler.waitProcedure(this, testProcEnv.getEnvLock()); + } + @Override protected void releaseLock(TestProcEnv testProcEnv) { System.out.println(procName + " release lock."); - testProcEnv.getEnvLock().unlock(); SimpleProcedureScheduler scheduler = (SimpleProcedureScheduler) testProcEnv.getScheduler(); - scheduler.releaseWaiting(); + scheduler.releaseWaiting(testProcEnv.getEnvLock()); } @Override diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java index d0e92b3281666..3e7fd2052ad53 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateCQProcedureTest.java @@ -36,10 +36,36 @@ import java.util.concurrent.ScheduledExecutorService; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.fail; public class CreateCQProcedureTest { + @Test + public void tokenShouldBeUniqueForSameCQId() { + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + try { + TCreateCQReq req = + new TCreateCQReq( + "testCq1", + 1000, + 0, + 1000, + 0, + (byte) 0, + "select s1 into root.backup.d1(s1) from root.sg.d1", + "create cq testCq1 BEGIN select s1 into root.backup.d1(s1) from root.sg.d1 END", + "Asia", + "root"); + CreateCQProcedure createCQProcedure1 = new CreateCQProcedure(req, executor); + CreateCQProcedure createCQProcedure2 = new CreateCQProcedure(req, executor); + + assertNotEquals(createCQProcedure1.getCqToken(), createCQProcedure2.getCqToken()); + } finally { + executor.shutdown(); + } + } + @Test public void serializeDeserializeTest() { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedureRecoveryTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedureRecoveryTest.java new file mode 100644 index 0000000000000..a90e282494f0e --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/cq/CreateCQProcedureRecoveryTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.cq; + +import org.apache.iotdb.confignode.consensus.request.read.cq.ShowCQPlan; +import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.cq.CQManager; +import org.apache.iotdb.confignode.manager.cq.CQScheduleTask; +import org.apache.iotdb.confignode.persistence.cq.CQInfo; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; + +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +public class CreateCQProcedureRecoveryTest { + + private TCreateCQReq newCreateCQReq() { + return new TCreateCQReq( + "testCq1", + 1000, + 0, + 1000, + 0, + (byte) 0, + "select s1 into root.backup.d1.s1 from root.sg.d1", + "create cq testCq1 BEGIN select s1 into root.backup.d1.s1 from root.sg.d1 END", + "Asia", + "root"); + } + + @SuppressWarnings("unchecked") + @Test + public void recoverScheduledTaskShouldResubmitFromLatestMetadata() throws Exception { + ScheduledExecutorService executor = Mockito.mock(ScheduledExecutorService.class); + Mockito.when( + executor.schedule( + Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.any(TimeUnit.class))) + .thenReturn(Mockito.mock(ScheduledFuture.class)); + + ConfigManager configManager = Mockito.mock(ConfigManager.class); + ConsensusManager consensusManager = Mockito.mock(ConsensusManager.class); + CQManager cqManager = Mockito.mock(CQManager.class); + ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + Mockito.when(env.getConfigManager()).thenReturn(configManager); + Mockito.when(configManager.getConsensusManager()).thenReturn(consensusManager); + Mockito.when(configManager.getCQManager()).thenReturn(cqManager); + Mockito.when( + cqManager.markCQLocallyScheduled( + Mockito.anyString(), Mockito.anyString(), Mockito.any(CQScheduleTask.class))) + .thenReturn(true); + + TCreateCQReq req = newCreateCQReq(); + CreateCQProcedure procedure = new CreateCQProcedure(req, executor); + + CQInfo cqInfo = new CQInfo(); + cqInfo.addCQ(new AddCQPlan(req, procedure.getCqToken(), System.currentTimeMillis() + 10_000)); + Mockito.when(consensusManager.read(Mockito.any(ShowCQPlan.class))).thenReturn(cqInfo.showCQ()); + + procedure.recoverScheduledTask(env); + + Mockito.verify(executor) + .schedule(Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.any(TimeUnit.class)); + } + + @SuppressWarnings("unchecked") + @Test + public void recoverScheduledTaskShouldSkipDuplicatedLocalSchedule() throws Exception { + ScheduledExecutorService executor = Mockito.mock(ScheduledExecutorService.class); + ConfigManager configManager = Mockito.mock(ConfigManager.class); + ConsensusManager consensusManager = Mockito.mock(ConsensusManager.class); + CQManager cqManager = Mockito.mock(CQManager.class); + ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + Mockito.when(env.getConfigManager()).thenReturn(configManager); + Mockito.when(configManager.getConsensusManager()).thenReturn(consensusManager); + Mockito.when(configManager.getCQManager()).thenReturn(cqManager); + Mockito.when( + cqManager.markCQLocallyScheduled( + Mockito.anyString(), Mockito.anyString(), Mockito.any(CQScheduleTask.class))) + .thenReturn(false); + + TCreateCQReq req = newCreateCQReq(); + CreateCQProcedure procedure = new CreateCQProcedure(req, executor); + + CQInfo cqInfo = new CQInfo(); + cqInfo.addCQ(new AddCQPlan(req, procedure.getCqToken(), System.currentTimeMillis() + 10_000)); + Mockito.when(consensusManager.read(Mockito.any(ShowCQPlan.class))).thenReturn(cqInfo.showCQ()); + + procedure.recoverScheduledTask(env); + + Mockito.verify(executor, Mockito.never()) + .schedule(Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.any(TimeUnit.class)); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleLeaderChangeProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleLeaderChangeProcedureTest.java index b76e291b0c3b9..b2ec615fbcd5b 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleLeaderChangeProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/runtime/PipeHandleLeaderChangeProcedureTest.java @@ -21,7 +21,10 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.confignode.procedure.Procedure; +import org.apache.iotdb.confignode.procedure.state.ProcedureState; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.PublicBAOS; @@ -45,6 +48,9 @@ public void serializeDeserializeTest() { leaderMap.put(new TConsensusGroupId(TConsensusGroupType.DataRegion, 1), new Pair<>(1, 2)); leaderMap.put(new TConsensusGroupId(TConsensusGroupType.DataRegion, 2), new Pair<>(2, 3)); leaderMap.put(new TConsensusGroupId(TConsensusGroupType.DataRegion, 3), new Pair<>(4, 5)); + leaderMap.put( + new TConsensusGroupId(TConsensusGroupType.ConfigRegion, Integer.MIN_VALUE), + new Pair<>(6, 7)); PipeHandleLeaderChangeProcedure proc = new PipeHandleLeaderChangeProcedure(leaderMap); @@ -60,4 +66,91 @@ public void serializeDeserializeTest() { fail(); } } + + @Test + public void deserializeOldFormatConfigRegionTest() { + PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + + Map> leaderMap = new HashMap<>(); + leaderMap.put( + new TConsensusGroupId(TConsensusGroupType.ConfigRegion, Integer.MIN_VALUE), + new Pair<>(6, 7)); + + try { + outputStream.writeShort(ProcedureType.PIPE_HANDLE_LEADER_CHANGE_PROCEDURE.getTypeCode()); + outputStream.writeLong(Procedure.NO_PROC_ID); + outputStream.writeInt(ProcedureState.INITIALIZING.ordinal()); + outputStream.writeLong(0L); + outputStream.writeLong(0L); + outputStream.writeLong(Procedure.NO_PROC_ID); + outputStream.writeLong(Procedure.NO_TIMEOUT); + outputStream.writeInt(-1); + outputStream.write((byte) 0); + outputStream.writeInt(-1); + outputStream.write((byte) 0); + outputStream.writeInt(0); + outputStream.write((byte) 0); + outputStream.writeInt(leaderMap.size()); + outputStream.writeInt(Integer.MIN_VALUE); + outputStream.writeInt(6); + outputStream.writeInt(7); + + ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + PipeHandleLeaderChangeProcedure proc = + (PipeHandleLeaderChangeProcedure) ProcedureFactory.getInstance().create(buffer); + + assertEquals(new PipeHandleLeaderChangeProcedure(leaderMap), proc); + } catch (Exception e) { + fail(); + } + } + + @Test + public void completedProcedureEqualsTest() { + Map> leaderMap = new HashMap<>(); + leaderMap.put(new TConsensusGroupId(TConsensusGroupType.DataRegion, 1), new Pair<>(1, 2)); + + try { + PipeHandleLeaderChangeProcedure proc = deserializeCompletedProcedure(leaderMap); + PipeHandleLeaderChangeProcedure proc2 = deserializeCompletedProcedure(leaderMap); + + assertEquals(proc, proc2); + assertEquals(proc.hashCode(), proc2.hashCode()); + } catch (Exception e) { + fail(); + } + } + + private PipeHandleLeaderChangeProcedure deserializeCompletedProcedure( + Map> leaderMap) throws Exception { + PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + + outputStream.writeShort(ProcedureType.PIPE_HANDLE_LEADER_CHANGE_PROCEDURE.getTypeCode()); + outputStream.writeLong(Procedure.NO_PROC_ID); + outputStream.writeInt(ProcedureState.SUCCESS.ordinal()); + outputStream.writeLong(0L); + outputStream.writeLong(0L); + outputStream.writeLong(Procedure.NO_PROC_ID); + outputStream.writeLong(Procedure.NO_TIMEOUT); + outputStream.writeInt(-1); + outputStream.write((byte) 0); + outputStream.writeInt(-1); + outputStream.write((byte) 0); + outputStream.writeInt(1); + outputStream.writeInt(Integer.MIN_VALUE); + outputStream.write((byte) 0); + outputStream.writeInt(leaderMap.size()); + for (Map.Entry> entry : leaderMap.entrySet()) { + outputStream.writeInt(entry.getKey().getId()); + outputStream.writeInt(entry.getValue().getLeft()); + outputStream.writeInt(entry.getValue().getRight()); + } + + ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + return (PipeHandleLeaderChangeProcedure) ProcedureFactory.getInstance().create(buffer); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2Test.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2Test.java index 6c095de98529a..ac6ca8c740c36 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2Test.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/CreatePipeProcedureV2Test.java @@ -19,11 +19,19 @@ package org.apache.iotdb.confignode.procedure.impl.pipe.task; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.PermissionManager; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TPermissionInfoResp; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.PublicBAOS; import org.junit.Test; +import org.mockito.Mockito; import java.io.DataOutputStream; import java.nio.ByteBuffer; @@ -68,4 +76,57 @@ public void serializeDeserializeTest() { fail(); } } + + @Test + public void testCheckAndEnrichSourceAuthenticationWithEncryptedPassword() { + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + final ConfigManager configManager = Mockito.mock(ConfigManager.class); + final PermissionManager permissionManager = Mockito.mock(PermissionManager.class); + Mockito.when(env.getConfigManager()).thenReturn(configManager); + Mockito.when(configManager.getPermissionManager()).thenReturn(permissionManager); + + final TPermissionInfoResp loginResp = new TPermissionInfoResp(); + loginResp.setStatus(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + Mockito.when(permissionManager.login("user", "encrypted-password", true)).thenReturn(loginResp); + + final Map sourceAttributes = new HashMap<>(); + sourceAttributes.put("extractor", "iotdb-source"); + sourceAttributes.put("username", "user"); + sourceAttributes.put("password", "encrypted-password"); + + CreatePipeProcedureV2.checkAndEnrichSourceAuthentication(env, sourceAttributes); + + assertEquals( + "encrypted-password", sourceAttributes.get(PipeSourceConstant.SOURCE_IOTDB_PASSWORD_KEY)); + Mockito.verify(permissionManager).login("user", "encrypted-password", true); + Mockito.verify(permissionManager, Mockito.never()) + .login4Pipe(Mockito.anyString(), Mockito.any()); + } + + @Test + public void testCheckAndEnrichSourceAuthenticationFallsBackToRawPassword() { + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + final ConfigManager configManager = Mockito.mock(ConfigManager.class); + final PermissionManager permissionManager = Mockito.mock(PermissionManager.class); + Mockito.when(env.getConfigManager()).thenReturn(configManager); + Mockito.when(configManager.getPermissionManager()).thenReturn(permissionManager); + + final TPermissionInfoResp loginResp = new TPermissionInfoResp(); + loginResp.setStatus(new TSStatus(TSStatusCode.WRONG_LOGIN_PASSWORD.getStatusCode())); + Mockito.when(permissionManager.login("user", "raw-password", true)).thenReturn(loginResp); + Mockito.when(permissionManager.login4Pipe("user", "raw-password")) + .thenReturn("hashed-password"); + + final Map sourceAttributes = new HashMap<>(); + sourceAttributes.put(PipeSourceConstant.SOURCE_KEY, "iotdb-source"); + sourceAttributes.put(PipeSourceConstant.SOURCE_IOTDB_USERNAME_KEY, "user"); + sourceAttributes.put(PipeSourceConstant.SOURCE_IOTDB_PASSWORD_KEY, "raw-password"); + + CreatePipeProcedureV2.checkAndEnrichSourceAuthentication(env, sourceAttributes); + + assertEquals( + "hashed-password", sourceAttributes.get(PipeSourceConstant.SOURCE_IOTDB_PASSWORD_KEY)); + Mockito.verify(permissionManager).login("user", "raw-password", true); + Mockito.verify(permissionManager).login4Pipe("user", "raw-password"); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2Test.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2Test.java index e6d676ea27b1c..e3da356059b5e 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2Test.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/StopPipeProcedureV2Test.java @@ -19,18 +19,73 @@ package org.apache.iotdb.confignode.procedure.impl.pipe.task; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStatus; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.SetPipeStatusWithStoppedByRuntimeExceptionPlanV2; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; +import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.PublicBAOS; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; import java.io.DataOutputStream; import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; public class StopPipeProcedureV2Test { + + private static class TestStopPipeProcedureV2 extends StopPipeProcedureV2 { + + private TestStopPipeProcedureV2(final String pipeName) throws PipeException { + super(pipeName); + } + + private void setPipeTaskInfo(final PipeTaskInfo pipeTaskInfo) { + this.pipeTaskInfo = new AtomicReference<>(pipeTaskInfo); + } + } + + private static PipeTaskInfo createExceptionStoppedPipeTaskInfo(final String pipeName) { + final PipeTaskInfo pipeTaskInfo = new PipeTaskInfo(); + + final PipeTaskMeta pipeTaskMeta = new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1); + final ConcurrentMap pipeTasks = new ConcurrentHashMap<>(); + pipeTasks.put(1, pipeTaskMeta); + + final PipeStaticMeta pipeStaticMeta = + new PipeStaticMeta( + pipeName, + System.currentTimeMillis(), + Collections.singletonMap("extractor", "iotdb-source"), + Collections.singletonMap("processor", "do-nothing-processor"), + Collections.singletonMap("connector", "iotdb-thrift-connector")); + final PipeRuntimeMeta pipeRuntimeMeta = new PipeRuntimeMeta(pipeTasks); + pipeRuntimeMeta.getStatus().set(PipeStatus.STOPPED); + pipeRuntimeMeta.setIsStoppedByRuntimeException(true); + + pipeTaskInfo.createPipe(new CreatePipePlanV2(pipeStaticMeta, pipeRuntimeMeta)); + return pipeTaskInfo; + } + @Test public void serializeDeserializeTest() { PublicBAOS byteArrayOutputStream = new PublicBAOS(); @@ -50,4 +105,54 @@ public void serializeDeserializeTest() { fail(); } } + + @Test + public void serializeDeserializeLegacyFormatTest() { + final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + + final StopPipeProcedureV2 proc = new StopPipeProcedureV2("testPipe"); + + try { + proc.serialize(outputStream); + final ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size() - 1); + final StopPipeProcedureV2 proc2 = + (StopPipeProcedureV2) ProcedureFactory.getInstance().create(buffer); + + assertEquals(proc, proc2); + } catch (Exception e) { + fail(); + } + } + + @Test + public void testStopPipeWritesStatusAndRuntimeExceptionFlagAtomically() throws Exception { + final String pipeName = "testPipe"; + final TestStopPipeProcedureV2 proc = new TestStopPipeProcedureV2(pipeName); + proc.setPipeTaskInfo(createExceptionStoppedPipeTaskInfo(pipeName)); + proc.executeFromCalculateInfoForTask(Mockito.mock(ConfigNodeProcedureEnv.class)); + + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + final ConfigManager configManager = Mockito.mock(ConfigManager.class); + final ConsensusManager consensusManager = Mockito.mock(ConsensusManager.class); + Mockito.when(env.getConfigManager()).thenReturn(configManager); + Mockito.when(configManager.getConsensusManager()).thenReturn(consensusManager); + Mockito.when(consensusManager.write(Mockito.any(ConfigPhysicalPlan.class))) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + + proc.executeFromWriteConfigNodeConsensus(env); + proc.rollbackFromWriteConfigNodeConsensus(env); + + final ArgumentCaptor planCaptor = + ArgumentCaptor.forClass(ConfigPhysicalPlan.class); + Mockito.verify(consensusManager, Mockito.times(2)).write(planCaptor.capture()); + + assertEquals( + new SetPipeStatusWithStoppedByRuntimeExceptionPlanV2(pipeName, PipeStatus.STOPPED, false), + planCaptor.getAllValues().get(0)); + assertEquals( + new SetPipeStatusWithStoppedByRuntimeExceptionPlanV2(pipeName, PipeStatus.RUNNING, true), + planCaptor.getAllValues().get(1)); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTTLProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTTLProcedureTest.java index 5042eb1dd0f13..cb09c23659c39 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTTLProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/SetTTLProcedureTest.java @@ -19,19 +19,42 @@ package org.apache.iotdb.confignode.procedure.impl.schema; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.ttl.TTLCache; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; +import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.TTLManager; +import org.apache.iotdb.confignode.manager.node.NodeManager; +import org.apache.iotdb.confignode.procedure.Procedure; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.state.ProcedureState; +import org.apache.iotdb.confignode.procedure.state.schema.SetTTLState; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.io.DataOutputStream; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class SetTTLProcedureTest { @@ -65,4 +88,308 @@ public void serializeDeserializeTest() throws IOException, IllegalPathException buffer.clear(); byteArrayOutputStream.reset(); } + + @Test + public void serializeDeserializeTestWithCapturedRollbackState() throws Exception { + final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + + final SetTTLPlan setTTLPlan = + new SetTTLPlan(Arrays.asList(new PartialPath("root.db").getNodes()), 2000L); + setTTLPlan.setDataBase(true); + final TestingSetTTLProcedure procedure = new TestingSetTTLProcedure(setTTLPlan); + + final Map ttlMap = new HashMap<>(); + ttlMap.put("root.**", Long.MAX_VALUE); + ttlMap.put("root.db", 500L); + ttlMap.put("root.db.**", 600L); + + procedure.executeFromState(mockProcedureEnv(ttlMap), SetTTLState.CAPTURE_PREVIOUS_TTL); + + procedure.serialize(outputStream); + final ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + final SetTTLProcedure deserializedProcedure = + (SetTTLProcedure) ProcedureFactory.getInstance().create(buffer); + assertSerializedProcedure( + deserializedProcedure, "root.db", 2000L, true, true, 500L, 600L, false); + } + + @Test + public void deserializeOldFormatWithoutRollbackStateTest() throws Exception { + final SetTTLPlan setTTLPlan = + new SetTTLPlan(Arrays.asList(new PartialPath("root.db").getNodes()), 2000L); + setTTLPlan.setDataBase(true); + + final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); + writeOldFormatProcedure(outputStream, setTTLPlan); + + final ByteBuffer buffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + final SetTTLProcedure deserializedProcedure = + (SetTTLProcedure) ProcedureFactory.getInstance().create(buffer); + + assertSerializedProcedure( + deserializedProcedure, + "root.db", + 2000L, + true, + false, + Long.MIN_VALUE, + Long.MIN_VALUE, + false); + } + + @Test + public void setConfigNodeTTLShouldNotWriteBeforePreviousStateIsCaptured() throws Exception { + final SetTTLPlan setTTLPlan = + new SetTTLPlan(Arrays.asList(new PartialPath("root.db").getNodes()), 2000L); + setTTLPlan.setDataBase(true); + final TestingSetTTLProcedure procedure = new TestingSetTTLProcedure(setTTLPlan); + + final Map ttlMap = new HashMap<>(); + ttlMap.put("root.**", Long.MAX_VALUE); + ttlMap.put("root.db", 500L); + ttlMap.put("root.db.**", 600L); + + procedure.executeFromState(mockProcedureEnv(ttlMap), SetTTLState.SET_CONFIGNODE_TTL); + + Assert.assertTrue(procedure.getWrittenPlans().isEmpty()); + assertSerializedProcedure(procedure, "root.db", 2000L, true, true, 500L, 600L, false); + + procedure.executeFromState(mockProcedureEnv(ttlMap), SetTTLState.SET_CONFIGNODE_TTL); + + Assert.assertEquals(1, procedure.getWrittenPlans().size()); + assertPlan(procedure.getWrittenPlans().get(0), "root.db", 2000L, true); + } + + @Test + public void rollbackStateShouldUnsetNewTTLWhenPreviousStateDidNotExist() throws Exception { + final SetTTLPlan setTTLPlan = + new SetTTLPlan(Arrays.asList(new PartialPath("root.test.sg1.**").getNodes()), 1000L); + final TestingSetTTLProcedure procedure = new TestingSetTTLProcedure(setTTLPlan); + procedure.failFirstDataNodeUpdateForTest(); + + final ConfigNodeProcedureEnv env = + mockProcedureEnv(Collections.singletonMap("root.**", Long.MAX_VALUE)); + + procedure.executeFromState(env, SetTTLState.CAPTURE_PREVIOUS_TTL); + procedure.executeFromState(env, SetTTLState.SET_CONFIGNODE_TTL); + procedure.executeFromState(env, SetTTLState.UPDATE_DATANODE_CACHE); + Assert.assertTrue(procedure.isFailed()); + + procedure.rollbackState(env, SetTTLState.UPDATE_DATANODE_CACHE); + + Assert.assertEquals(2, procedure.getWrittenPlans().size()); + assertPlan(procedure.getWrittenPlans().get(0), "root.test.sg1.**", 1000L, false); + assertPlan(procedure.getWrittenPlans().get(1), "root.test.sg1.**", -1L, false); + + Assert.assertEquals(2, procedure.getRequests().size()); + assertRequest(procedure.getRequests().get(0), "root.test.sg1.**", 1000L, false); + assertRequest(procedure.getRequests().get(1), "root.test.sg1.**", -1L, false); + } + + @Test + public void rollbackStateShouldRestoreDatabaseWildcardTTLSeparately() throws Exception { + final SetTTLPlan setTTLPlan = + new SetTTLPlan(Arrays.asList(new PartialPath("root.db").getNodes()), 2000L); + setTTLPlan.setDataBase(true); + final TestingSetTTLProcedure procedure = new TestingSetTTLProcedure(setTTLPlan); + procedure.failFirstDataNodeUpdateForTest(); + + final Map ttlMap = new HashMap<>(); + ttlMap.put("root.**", Long.MAX_VALUE); + ttlMap.put("root.db", 500L); + ttlMap.put("root.db.**", 600L); + final ConfigNodeProcedureEnv env = mockProcedureEnv(ttlMap); + + procedure.executeFromState(env, SetTTLState.CAPTURE_PREVIOUS_TTL); + procedure.executeFromState(env, SetTTLState.SET_CONFIGNODE_TTL); + procedure.executeFromState(env, SetTTLState.UPDATE_DATANODE_CACHE); + Assert.assertTrue(procedure.isFailed()); + + procedure.rollbackState(env, SetTTLState.UPDATE_DATANODE_CACHE); + + Assert.assertEquals(3, procedure.getWrittenPlans().size()); + assertPlan(procedure.getWrittenPlans().get(0), "root.db", 2000L, true); + assertPlan(procedure.getWrittenPlans().get(1), "root.db", 500L, false); + assertPlan(procedure.getWrittenPlans().get(2), "root.db.**", 600L, false); + + Assert.assertEquals(3, procedure.getRequests().size()); + assertRequest(procedure.getRequests().get(0), "root.db", 2000L, true); + assertRequest(procedure.getRequests().get(1), "root.db", 500L, false); + assertRequest(procedure.getRequests().get(2), "root.db.**", 600L, false); + } + + private ConfigNodeProcedureEnv mockProcedureEnv(final Map ttlMap) { + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + final ConfigManager configManager = Mockito.mock(ConfigManager.class); + final TTLManager ttlManager = Mockito.mock(TTLManager.class); + final NodeManager nodeManager = Mockito.mock(NodeManager.class); + + final TDataNodeLocation dataNodeLocation = new TDataNodeLocation(); + dataNodeLocation.setDataNodeId(1); + + Mockito.when(env.getConfigManager()).thenReturn(configManager); + Mockito.when(configManager.getTTLManager()).thenReturn(ttlManager); + Mockito.when(ttlManager.getTTL(Mockito.any(String[].class))) + .thenAnswer( + invocation -> { + final String[] pathPattern = invocation.getArgument(0); + return ttlMap.getOrDefault(String.join(".", pathPattern), TTLCache.NULL_TTL); + }); + Mockito.when(configManager.getNodeManager()).thenReturn(nodeManager); + Mockito.when(nodeManager.getRegisteredDataNodeLocations()) + .thenReturn(Collections.singletonMap(1, dataNodeLocation)); + return env; + } + + private void assertPlan( + final SetTTLPlan plan, final String path, final long ttl, final boolean isDataBase) { + Assert.assertEquals(path, String.join(".", plan.getPathPattern())); + Assert.assertEquals(ttl, plan.getTTL()); + Assert.assertEquals(isDataBase, plan.isDataBase()); + } + + private void assertRequest( + final TSetTTLReq req, final String path, final long ttl, final boolean isDataBase) { + Assert.assertEquals(Collections.singletonList(path), req.getPathPattern()); + Assert.assertEquals(ttl, req.getTTL()); + Assert.assertEquals(isDataBase, req.isDataBase); + } + + private void writeOldFormatProcedure(final DataOutputStream stream, final SetTTLPlan plan) + throws IOException { + stream.writeShort(ProcedureType.SET_TTL_PROCEDURE.getTypeCode()); + // Procedure fields. + stream.writeLong(Procedure.NO_PROC_ID); + stream.writeInt(ProcedureState.INITIALIZING.ordinal()); + stream.writeLong(0L); + stream.writeLong(0L); + stream.writeLong(Procedure.NO_PROC_ID); + stream.writeLong(Procedure.NO_TIMEOUT); + stream.writeInt(-1); // no stack indexes + stream.write((byte) 0); // no exception + stream.writeInt(-1); // no result + stream.write((byte) 0); // no lock + // StateMachineProcedure fields. + stream.writeInt(0); // no states + ReadWriteIOUtils.write(plan.serializeToByteBuffer(), stream); + } + + private void assertSerializedProcedure( + final SetTTLProcedure procedure, + final String path, + final long ttl, + final boolean isDataBase, + final boolean previousTTLStateCaptured, + final long previousTTL, + final long previousDatabaseWildcardTTL, + final boolean isGeneratedByPipe) + throws Exception { + final Field planField = findField(SetTTLProcedure.class, "plan"); + planField.setAccessible(true); + assertPlan((SetTTLPlan) planField.get(procedure), path, ttl, isDataBase); + + final Field previousTTLStateCapturedField = + findField(SetTTLProcedure.class, "previousTTLStateCaptured"); + previousTTLStateCapturedField.setAccessible(true); + Assert.assertEquals(previousTTLStateCaptured, previousTTLStateCapturedField.get(procedure)); + + final Field previousTTLField = findField(SetTTLProcedure.class, "previousTTL"); + previousTTLField.setAccessible(true); + Assert.assertEquals(previousTTL, previousTTLField.get(procedure)); + + final Field previousDatabaseWildcardTTLField = + findField(SetTTLProcedure.class, "previousDatabaseWildcardTTL"); + previousDatabaseWildcardTTLField.setAccessible(true); + Assert.assertEquals( + previousDatabaseWildcardTTL, previousDatabaseWildcardTTLField.get(procedure)); + + final Field isGeneratedByPipeField = findField(SetTTLProcedure.class, "isGeneratedByPipe"); + isGeneratedByPipeField.setAccessible(true); + Assert.assertEquals(isGeneratedByPipe, isGeneratedByPipeField.get(procedure)); + } + + private Field findField(final Class clazz, final String fieldName) + throws NoSuchFieldException { + Class current = clazz; + while (current != null) { + try { + return current.getDeclaredField(fieldName); + } catch (NoSuchFieldException e) { + current = current.getSuperclass(); + } + } + throw new NoSuchFieldException(fieldName); + } + + private static class TestingSetTTLProcedure extends SetTTLProcedure { + + private final List requests = new ArrayList<>(); + private final List writtenPlans = new ArrayList<>(); + private boolean failFirstDataNodeUpdate = false; + private int requestCount = 0; + + private TestingSetTTLProcedure(final SetTTLPlan plan) { + super(plan, false); + } + + private void failFirstDataNodeUpdateForTest() { + failFirstDataNodeUpdate = true; + } + + private List getRequests() { + return requests; + } + + private List getWrittenPlans() { + return writtenPlans; + } + + @Override + TSStatus writeConfigNodePlan(final ConfigNodeProcedureEnv env, final SetTTLPlan setTTLPlan) { + writtenPlans.add(copyPlan(setTTLPlan)); + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } + + @Override + DataNodeAsyncRequestContext sendTTLRequest( + final Map dataNodeLocationMap, final TSetTTLReq req) { + requests.add(copyRequest(req)); + + final DataNodeAsyncRequestContext clientHandler = + new DataNodeAsyncRequestContext<>( + CnToDnAsyncRequestType.SET_TTL, copyRequest(req), dataNodeLocationMap); + final List requestIds = new ArrayList<>(clientHandler.getNodeLocationMap().keySet()); + final boolean shouldFail = failFirstDataNodeUpdate && requestCount++ == 0; + + for (Integer requestId : requestIds) { + clientHandler + .getResponseMap() + .put( + requestId, + new TSStatus( + shouldFail + ? TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode() + : TSStatusCode.SUCCESS_STATUS.getStatusCode())); + if (!shouldFail) { + clientHandler.getNodeLocationMap().remove(requestId); + } + } + return clientHandler; + } + + private SetTTLPlan copyPlan(final SetTTLPlan plan) { + final SetTTLPlan copiedPlan = + new SetTTLPlan(Arrays.asList(plan.getPathPattern()), plan.getTTL()); + copiedPlan.setDataBase(plan.isDataBase()); + return copiedPlan; + } + + private TSetTTLReq copyRequest(final TSetTTLReq req) { + return new TSetTTLReq(new ArrayList<>(req.getPathPattern()), req.getTTL(), req.isDataBase); + } + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedureTest.java index 93d9941fbf333..e2a4d0615d886 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/CreateSubscriptionProcedureTest.java @@ -19,18 +19,36 @@ package org.apache.iotdb.confignode.procedure.impl.subscription.subscription; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta; +import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta; +import org.apache.iotdb.confignode.consensus.request.write.pipe.task.CreatePipePlanV2; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.PermissionManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.manager.load.LoadManager; +import org.apache.iotdb.confignode.manager.pipe.coordinator.PipeManager; +import org.apache.iotdb.confignode.manager.pipe.coordinator.plugin.PipePluginCoordinator; +import org.apache.iotdb.confignode.persistence.pipe.PipePluginInfo; +import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo; +import org.apache.iotdb.confignode.persistence.subscription.SubscriptionInfo; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.impl.pipe.task.CreatePipeProcedureV2; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.AlterConsumerGroupProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.tsfile.utils.PublicBAOS; +import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.io.DataOutputStream; +import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; @@ -39,6 +57,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -102,4 +121,119 @@ public void serializeDeserializeTest() { fail(); } } + + @Test + public void executeFromOperateOnConfigNodesShouldFailOnTopLevelConsensusError() throws Exception { + final CreateSubscriptionProcedure proc = + new CreateSubscriptionProcedure( + new TSubscribeReq( + "old_consumer", "test_consumer_group", Collections.singleton("test_topic"))); + proc.setAlterConsumerGroupProcedure(Mockito.mock(AlterConsumerGroupProcedure.class)); + + final CreatePipeProcedureV2 createPipeProcedure = Mockito.mock(CreatePipeProcedureV2.class); + Mockito.when(createPipeProcedure.constructPlan()) + .thenReturn(Mockito.mock(CreatePipePlanV2.class)); + proc.setCreatePipeProcedures(Collections.singletonList(createPipeProcedure)); + + try { + proc.executeFromOperateOnConfigNodes( + mockConsensusFailureEnv( + new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()) + .setMessage("consensus write failed"))); + fail(); + } catch (SubscriptionException e) { + Assert.assertTrue(e.getMessage().contains("Failed to create subscription")); + } + } + + @Test + public void executeFromValidateShouldResetCreatePipeProceduresOnRetry() throws Exception { + final Map consumerAttributes = new HashMap<>(); + consumerAttributes.put("username", "user"); + consumerAttributes.put("password", "password"); + + final ConsumerGroupMeta consumerGroupMeta = + new ConsumerGroupMeta( + "test_consumer_group", 1, new ConsumerMeta("old_consumer", 1, consumerAttributes)); + final TopicMeta topicMeta = new TopicMeta("test_topic", 1, Collections.emptyMap()); + + final SubscriptionInfo subscriptionInfo = Mockito.mock(SubscriptionInfo.class); + Mockito.when(subscriptionInfo.getConsumerGroupMeta("test_consumer_group")) + .thenReturn(consumerGroupMeta); + Mockito.when(subscriptionInfo.deepCopyConsumerGroupMeta("test_consumer_group")) + .thenAnswer(invocation -> consumerGroupMeta.deepCopy()); + Mockito.when( + subscriptionInfo.isTopicSubscribedByConsumerGroup("test_topic", "test_consumer_group")) + .thenReturn(false); + Mockito.when(subscriptionInfo.deepCopyTopicMeta("test_topic")).thenReturn(topicMeta); + + final PipeTaskInfo pipeTaskInfo = Mockito.mock(PipeTaskInfo.class); + Mockito.when(pipeTaskInfo.checkBeforeCreatePipe(Mockito.any(TCreatePipeReq.class))) + .thenReturn(true); + + final CreateSubscriptionProcedure proc = + new CreateSubscriptionProcedure( + new TSubscribeReq( + "old_consumer", "test_consumer_group", Collections.singleton("test_topic"))); + setField(proc, "subscriptionInfo", new AtomicReference<>(subscriptionInfo)); + setField(proc, "pipeTaskInfo", new AtomicReference<>(pipeTaskInfo)); + + final ConfigNodeProcedureEnv env = mockCreateSubscriptionValidationEnv(); + proc.executeFromValidate(env); + Assert.assertEquals(1, proc.getCreatePipeProcedures().size()); + + proc.executeFromValidate(env); + Assert.assertEquals(1, proc.getCreatePipeProcedures().size()); + } + + private static ConfigNodeProcedureEnv mockConsensusFailureEnv(final TSStatus response) + throws Exception { + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + final ConfigManager configManager = Mockito.mock(ConfigManager.class); + final ConsensusManager consensusManager = Mockito.mock(ConsensusManager.class); + + Mockito.when(env.getConfigManager()).thenReturn(configManager); + Mockito.when(configManager.getConsensusManager()).thenReturn(consensusManager); + Mockito.when(consensusManager.write(Mockito.any())).thenReturn(response); + + return env; + } + + private static ConfigNodeProcedureEnv mockCreateSubscriptionValidationEnv() { + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + final ConfigManager configManager = Mockito.mock(ConfigManager.class); + final PermissionManager permissionManager = Mockito.mock(PermissionManager.class); + final PipeManager pipeManager = Mockito.mock(PipeManager.class); + final PipePluginCoordinator pipePluginCoordinator = Mockito.mock(PipePluginCoordinator.class); + final PipePluginInfo pipePluginInfo = Mockito.mock(PipePluginInfo.class); + final LoadManager loadManager = Mockito.mock(LoadManager.class); + + Mockito.when(env.getConfigManager()).thenReturn(configManager); + Mockito.when(configManager.getPermissionManager()).thenReturn(permissionManager); + Mockito.when(configManager.getPipeManager()).thenReturn(pipeManager); + Mockito.when(pipeManager.getPipePluginCoordinator()).thenReturn(pipePluginCoordinator); + Mockito.when(pipePluginCoordinator.getPipePluginInfo()).thenReturn(pipePluginInfo); + Mockito.when(configManager.getLoadManager()).thenReturn(loadManager); + Mockito.when(loadManager.getRegionLeaderMap()).thenReturn(Collections.emptyMap()); + Mockito.when(permissionManager.login4Pipe(Mockito.anyString(), Mockito.any())) + .thenReturn("hashedPassword"); + + return env; + } + + private static void setField(final Object target, final String fieldName, final Object value) + throws Exception { + Class clazz = target.getClass(); + while (clazz != null) { + try { + final Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + return; + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new NoSuchFieldException(fieldName); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedureTest.java index 9ecce2a522ca4..910648bbe5132 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/subscription/subscription/DropSubscriptionProcedureTest.java @@ -19,24 +19,37 @@ package org.apache.iotdb.confignode.procedure.impl.subscription.subscription; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerGroupMeta; import org.apache.iotdb.commons.subscription.meta.consumer.ConsumerMeta; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; +import org.apache.iotdb.confignode.persistence.pipe.PipeTaskInfo; +import org.apache.iotdb.confignode.persistence.subscription.SubscriptionInfo; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.impl.pipe.task.DropPipeProcedureV2; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.AlterConsumerGroupProcedure; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.rpc.subscription.exception.SubscriptionException; import org.apache.tsfile.utils.PublicBAOS; +import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.io.DataOutputStream; +import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -91,4 +104,90 @@ public void serializeDeserializeTest() { fail(); } } + + @Test + public void executeFromOperateOnConfigNodesShouldFailOnTopLevelConsensusError() throws Exception { + final DropSubscriptionProcedure proc = + new DropSubscriptionProcedure( + new TUnsubscribeReq( + "old_consumer", "test_consumer_group", Collections.singleton("test_topic"))); + proc.setAlterConsumerGroupProcedure(Mockito.mock(AlterConsumerGroupProcedure.class)); + + final DropPipeProcedureV2 dropPipeProcedure = Mockito.mock(DropPipeProcedureV2.class); + Mockito.when(dropPipeProcedure.getPipeName()).thenReturn("pipe_topic"); + proc.setDropPipeProcedures(Collections.singletonList(dropPipeProcedure)); + + try { + proc.executeFromOperateOnConfigNodes( + mockConsensusFailureEnv( + new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()) + .setMessage("consensus write failed"))); + fail(); + } catch (SubscriptionException e) { + Assert.assertTrue(e.getMessage().contains("Failed to drop subscription")); + } + } + + @Test + public void executeFromValidateShouldResetDropPipeProceduresOnRetry() throws Exception { + final Map consumerAttributes = new HashMap<>(); + consumerAttributes.put("username", "user"); + consumerAttributes.put("password", "password"); + + final ConsumerGroupMeta consumerGroupMeta = + new ConsumerGroupMeta( + "test_consumer_group", 1, new ConsumerMeta("old_consumer", 1, consumerAttributes)); + consumerGroupMeta.addSubscription("old_consumer", Collections.singleton("test_topic")); + + final SubscriptionInfo subscriptionInfo = Mockito.mock(SubscriptionInfo.class); + Mockito.when(subscriptionInfo.getConsumerGroupMeta("test_consumer_group")) + .thenReturn(consumerGroupMeta); + Mockito.when(subscriptionInfo.deepCopyConsumerGroupMeta("test_consumer_group")) + .thenAnswer(invocation -> consumerGroupMeta.deepCopy()); + + final PipeTaskInfo pipeTaskInfo = Mockito.mock(PipeTaskInfo.class); + + final DropSubscriptionProcedure proc = + new DropSubscriptionProcedure( + new TUnsubscribeReq( + "old_consumer", "test_consumer_group", Collections.singleton("test_topic"))); + setField(proc, "subscriptionInfo", new AtomicReference<>(subscriptionInfo)); + setField(proc, "pipeTaskInfo", new AtomicReference<>(pipeTaskInfo)); + + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + proc.executeFromValidate(env); + Assert.assertEquals(1, proc.getDropPipeProcedures().size()); + + proc.executeFromValidate(env); + Assert.assertEquals(1, proc.getDropPipeProcedures().size()); + } + + private static ConfigNodeProcedureEnv mockConsensusFailureEnv(final TSStatus response) + throws Exception { + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + final ConfigManager configManager = Mockito.mock(ConfigManager.class); + final ConsensusManager consensusManager = Mockito.mock(ConsensusManager.class); + + Mockito.when(env.getConfigManager()).thenReturn(configManager); + Mockito.when(configManager.getConsensusManager()).thenReturn(consensusManager); + Mockito.when(consensusManager.write(Mockito.any())).thenReturn(response); + + return env; + } + + private static void setField(final Object target, final String fieldName, final Object value) + throws Exception { + Class clazz = target.getClass(); + while (clazz != null) { + try { + final Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + return; + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new NoSuchFieldException(fieldName); + } } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java index 6c89e19291b35..25b084c5329d5 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/IoTConsensusConfig.java @@ -70,8 +70,6 @@ public Builder setReplication(Replication replication) { public static class RPC { - private final int rpcSelectorThreadNum; - private final int rpcMinConcurrentClientNum; private final int rpcMaxConcurrentClientNum; private final int thriftServerAwaitTimeForStopService; private final boolean isRpcThriftCompressionEnabled; @@ -83,8 +81,6 @@ public static class RPC { private final int maxClientNumForEachNode; private RPC( - int rpcSelectorThreadNum, - int rpcMinConcurrentClientNum, int rpcMaxConcurrentClientNum, int thriftServerAwaitTimeForStopService, boolean isRpcThriftCompressionEnabled, @@ -93,8 +89,6 @@ private RPC( boolean printLogWhenThriftClientEncounterException, int thriftMaxFrameSize, int maxClientNumForEachNode) { - this.rpcSelectorThreadNum = rpcSelectorThreadNum; - this.rpcMinConcurrentClientNum = rpcMinConcurrentClientNum; this.rpcMaxConcurrentClientNum = rpcMaxConcurrentClientNum; this.thriftServerAwaitTimeForStopService = thriftServerAwaitTimeForStopService; this.isRpcThriftCompressionEnabled = isRpcThriftCompressionEnabled; @@ -105,14 +99,6 @@ private RPC( this.maxClientNumForEachNode = maxClientNumForEachNode; } - public int getRpcSelectorThreadNum() { - return rpcSelectorThreadNum; - } - - public int getRpcMinConcurrentClientNum() { - return rpcMinConcurrentClientNum; - } - public int getRpcMaxConcurrentClientNum() { return rpcMaxConcurrentClientNum; } @@ -151,7 +137,6 @@ public static RPC.Builder newBuilder() { public static class Builder { - private int rpcSelectorThreadNum = 1; private int rpcMinConcurrentClientNum = Runtime.getRuntime().availableProcessors(); private int rpcMaxConcurrentClientNum = 65535; private int thriftServerAwaitTimeForStopService = 60; @@ -163,11 +148,6 @@ public static class Builder { private int thriftMaxFrameSize = 536870912; private int maxClientNumForEachNode = DefaultProperty.MAX_CLIENT_NUM_FOR_EACH_NODE; - public RPC.Builder setRpcSelectorThreadNum(int rpcSelectorThreadNum) { - this.rpcSelectorThreadNum = rpcSelectorThreadNum; - return this; - } - public RPC.Builder setRpcMinConcurrentClientNum(int rpcMinConcurrentClientNum) { this.rpcMinConcurrentClientNum = rpcMinConcurrentClientNum; return this; @@ -218,8 +198,6 @@ public Builder setMaxClientNumForEachNode(int maxClientNumForEachNode) { public RPC build() { return new RPC( - rpcSelectorThreadNum, - rpcMinConcurrentClientNum, rpcMaxConcurrentClientNum, thriftServerAwaitTimeForStopService, isRpcThriftCompressionEnabled, diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/PipeConsensusConfig.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/PipeConsensusConfig.java index 2cb149b601b01..81c6e53e73a12 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/PipeConsensusConfig.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/config/PipeConsensusConfig.java @@ -79,8 +79,6 @@ public PipeConsensusConfig build() { } public static class RPC { - private final int rpcSelectorThreadNum; - private final int rpcMinConcurrentClientNum; private final int rpcMaxConcurrentClientNum; private final int thriftServerAwaitTimeForStopService; private final boolean isRpcThriftCompressionEnabled; @@ -88,15 +86,11 @@ public static class RPC { private final int thriftMaxFrameSize; public RPC( - int rpcSelectorThreadNum, - int rpcMinConcurrentClientNum, int rpcMaxConcurrentClientNum, int thriftServerAwaitTimeForStopService, boolean isRpcThriftCompressionEnabled, int connectionTimeoutInMs, int thriftMaxFrameSize) { - this.rpcSelectorThreadNum = rpcSelectorThreadNum; - this.rpcMinConcurrentClientNum = rpcMinConcurrentClientNum; this.rpcMaxConcurrentClientNum = rpcMaxConcurrentClientNum; this.thriftServerAwaitTimeForStopService = thriftServerAwaitTimeForStopService; this.isRpcThriftCompressionEnabled = isRpcThriftCompressionEnabled; @@ -104,14 +98,6 @@ public RPC( this.thriftMaxFrameSize = thriftMaxFrameSize; } - public int getRpcSelectorThreadNum() { - return rpcSelectorThreadNum; - } - - public int getRpcMinConcurrentClientNum() { - return rpcMinConcurrentClientNum; - } - public int getRpcMaxConcurrentClientNum() { return rpcMaxConcurrentClientNum; } @@ -137,24 +123,12 @@ public static RPC.Builder newBuilder() { } public static class Builder { - private int rpcSelectorThreadNum = 1; - private int rpcMinConcurrentClientNum = Runtime.getRuntime().availableProcessors(); private int rpcMaxConcurrentClientNum = 65535; private int thriftServerAwaitTimeForStopService = 60; private boolean isRpcThriftCompressionEnabled = false; private int connectionTimeoutInMs = (int) TimeUnit.SECONDS.toMillis(60); private int thriftMaxFrameSize = 536870912; - public RPC.Builder setRpcSelectorThreadNum(int rpcSelectorThreadNum) { - this.rpcSelectorThreadNum = rpcSelectorThreadNum; - return this; - } - - public RPC.Builder setRpcMinConcurrentClientNum(int rpcMinConcurrentClientNum) { - this.rpcMinConcurrentClientNum = rpcMinConcurrentClientNum; - return this; - } - public RPC.Builder setRpcMaxConcurrentClientNum(int rpcMaxConcurrentClientNum) { this.rpcMaxConcurrentClientNum = rpcMaxConcurrentClientNum; return this; @@ -183,8 +157,6 @@ public RPC.Builder setThriftMaxFrameSize(int thriftMaxFrameSize) { public RPC build() { return new RPC( - rpcSelectorThreadNum, - rpcMinConcurrentClientNum, rpcMaxConcurrentClientNum, thriftServerAwaitTimeForStopService, isRpcThriftCompressionEnabled, diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/RecoverReadTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/RecoverReadTest.java index ac8115ced8555..82c81a29a6707 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/RecoverReadTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/RecoverReadTest.java @@ -82,6 +82,7 @@ public boolean takeSnapshot(File snapshotDir) { @Before public void setUp() throws Exception { logger.info("[RECOVER TEST] start setting up the test env"); + TestUtils.prepareJvmForRatisTest(); final TestUtils.MiniClusterFactory factory = new TestUtils.MiniClusterFactory(); miniCluster = factory @@ -117,8 +118,12 @@ public void setUp() throws Exception { @After public void tearUp() throws Exception { logger.info("[RECOVER TEST] start tearing down the test env"); - miniCluster.cleanUp(); - logger.info("[RECOVER TEST] end tearing down the test env"); + try { + miniCluster.cleanUp(); + } finally { + TestUtils.assertNoUnexpectedRatisExit(); + logger.info("[RECOVER TEST] end tearing down the test env"); + } } /* mimics the situation before this patch */ diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/TestUtils.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/TestUtils.java index f5997c261fdff..a8731c9b37d99 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/TestUtils.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/ratis/TestUtils.java @@ -40,6 +40,7 @@ import org.apache.iotdb.consensus.ratis.utils.Utils; import org.apache.ratis.thirdparty.com.google.common.base.Preconditions; +import org.apache.ratis.util.ExitUtils; import org.apache.ratis.util.FileUtils; import org.apache.ratis.util.TimeDuration; import org.apache.ratis.util.Timestamp; @@ -55,9 +56,11 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Scanner; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -70,6 +73,9 @@ public class TestUtils { private static final Logger logger = LoggerFactory.getLogger(TestUtils.class); + private static final TimeDuration PORT_RELEASE_WAIT = TimeDuration.valueOf(10, TimeUnit.SECONDS); + private static final TimeDuration PORT_RELEASE_POLL = + TimeDuration.valueOf(100, TimeUnit.MILLISECONDS); public static class TestDataSet implements DataSet { @@ -230,6 +236,7 @@ static class MiniCluster { private final RatisConfig config; private final List servers; private final ConsensusGroup group; + private final List peerPorts; private Supplier smProvider; private final AtomicBoolean isStopped = new AtomicBoolean(false); @@ -250,9 +257,10 @@ private MiniCluster( this.peerStorage = new ArrayList<>(); this.stateMachines = new ArrayList<>(); this.servers = new ArrayList<>(); + this.peerPorts = randomDistinctPorts(replicas); for (int i = 0; i < replicas; i++) { - peers.add(new Peer(gid, i, new TEndPoint("127.0.0.1", randomFreePort()))); + peers.add(new Peer(gid, i, new TEndPoint("127.0.0.1", peerPorts.get(i)))); final File storage = storageProvider.apply(i); FileUtils.deleteFileQuietly(storage); @@ -300,6 +308,7 @@ void stop() throws IOException { for (RatisConsensus server : servers) { server.stop(); } + waitUntilPortsReleased(peerPorts); isStopped.set(true); } @@ -495,6 +504,65 @@ MiniCluster create() { } } + static void prepareJvmForRatisTest() { + ExitUtils.disableSystemExit(); + ExitUtils.clear(); + } + + static void assertNoUnexpectedRatisExit() { + ExitUtils.assertNotTerminated(); + ExitUtils.clear(); + } + + private static List randomDistinctPorts(int count) { + final List ports = new ArrayList<>(count); + final Set uniquePorts = new HashSet<>(); + while (ports.size() < count) { + final int port = randomFreePort(); + if (uniquePorts.add(port)) { + ports.add(port); + } + } + return ports; + } + + private static void waitUntilPortsReleased(List ports) throws IOException { + final Timestamp start = Timestamp.currentTime(); + while (true) { + boolean allReleased = true; + for (int port : ports) { + if (!isLocalPortAvailable(port)) { + allReleased = false; + break; + } + } + + if (allReleased) { + return; + } + + if (start.elapsedTime().compareTo(PORT_RELEASE_WAIT) > 0) { + throw new IOException("Timed out waiting for Ratis test ports to be released: " + ports); + } + + try { + PORT_RELEASE_POLL.sleep(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while waiting for Ratis test ports to be released", e); + } + } + } + + private static boolean isLocalPortAvailable(int port) { + try (ServerSocket socket = new ServerSocket(port)) { + socket.setReuseAddress(true); + return true; + } catch (IOException e) { + return false; + } + } + private static int randomFreePort() { try (ServerSocket socket = new ServerSocket(0)) { return socket.getLocalPort(); diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml index d6f8d12434581..299e50f1be932 100644 --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@ -308,10 +308,6 @@ com.google.guava guava - - com.lmax - disruptor - org.java-websocket Java-WebSocket diff --git a/iotdb-core/datanode/src/assembly/resources/tools/tsfile/print-tsfile.sh b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/print-tsfile.sh index 8864ca877e744..8e27793a7ff0d 100644 --- a/iotdb-core/datanode/src/assembly/resources/tools/tsfile/print-tsfile.sh +++ b/iotdb-core/datanode/src/assembly/resources/tools/tsfile/print-tsfile.sh @@ -24,9 +24,11 @@ echo --------------------- source "$(dirname "$0")/../../sbin/iotdb-common.sh" #get_iotdb_include and checkAllVariables is in iotdb-common.sh +if [ -z "${IOTDB_HOME}" ]; then + export IOTDB_HOME="$(cd "`dirname "$0"`"/../..; pwd)" +fi VARS=$(get_iotdb_include "$*") checkAllVariables -export IOTDB_HOME="${IOTDB_HOME}/.." eval set -- "$VARS" @@ -48,5 +50,5 @@ done MAIN_CLASS=org.apache.iotdb.db.tools.TsFileSketchTool -"$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@" -exit $? \ No newline at end of file +"$JAVA" $JVM_OPTS -cp "$CLASSPATH" "$MAIN_CLASS" "$@" +exit $? diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 989dc75746400..07c73b43d87f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -61,8 +61,10 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.lang.reflect.Field; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -139,12 +141,6 @@ public class IoTDBConfig { /** ssl key Store password. */ private String keyStorePwd = ""; - /** Rpc Selector thread num */ - private int rpcSelectorThreadCount = 1; - - /** Min concurrent client number */ - private int rpcMinConcurrentClientNum = Runtime.getRuntime().availableProcessors(); - /** Max concurrent client number */ private int rpcMaxConcurrentClientNum = 1000; @@ -330,6 +326,14 @@ public class IoTDBConfig { tierDataDirs[0][0] + File.separator + IoTDBConstant.LOAD_TSFILE_FOLDER_NAME }; + private String[] loadTsFileAllowedDirs = new String[0]; + + private CanonicalPaths loadTsFileDirCanonicalPaths = canonicalPaths(loadTsFileDirs); + + private CanonicalPaths loadTsFileAllowedDirCanonicalPaths = canonicalPaths(loadTsFileAllowedDirs); + + private boolean loadTsFileSourcePathCheckEnabled = false; + /** Strategy of multiple directories. */ private String multiDirStrategyClassName = null; @@ -868,7 +872,7 @@ public class IoTDBConfig { private float udfCollectorMemoryBudgetInMB = (float) (1.0 / 3 * udfMemoryBudgetInMB); /** Unit: byte */ - private int thriftMaxFrameSize = 536870912; + private int thriftMaxFrameSize = getDefaultThriftMaxFrameSize(); private int thriftDefaultBufferSize = RpcUtils.THRIFT_DEFAULT_BUF_CAPACITY; @@ -999,6 +1003,8 @@ public class IoTDBConfig { */ private int maxClientNumForEachNode = DefaultProperty.MAX_CLIENT_NUM_FOR_EACH_NODE; + private int maxIdleClientNumForEachNode = DefaultProperty.MAX_IDLE_CLIENT_NUM_FOR_EACH_NODE; + /** * Cache size of partition cache in {@link * org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher} @@ -1035,9 +1041,6 @@ public class IoTDBConfig { /** ThreadPool size for read operation in coordinator */ private int coordinatorReadExecutorSize = 20; - /** ThreadPool size for write operation in coordinator */ - private int coordinatorWriteExecutorSize = 50; - private int[] schemaMemoryProportion = new int[] {5, 4, 1}; /** Memory allocated for schemaRegion */ @@ -1401,6 +1404,10 @@ private void formulateFolders() { for (int i = 0; i < loadActiveListeningDirs.length; i++) { loadActiveListeningDirs[i] = addDataHomeDir(loadActiveListeningDirs[i]); } + for (int i = 0; i < loadTsFileAllowedDirs.length; i++) { + loadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]); + } + loadTsFileAllowedDirCanonicalPaths = canonicalPaths(loadTsFileAllowedDirs); loadActiveListeningPipeDir = addDataHomeDir(loadActiveListeningPipeDir); loadActiveListeningFailDir = addDataHomeDir(loadActiveListeningFailDir); udfDir = addDataHomeDir(udfDir); @@ -1596,6 +1603,36 @@ public String[] getLoadTsFileDirs() { return this.loadTsFileDirs; } + public String[] getLoadTsFileAllowedDirs() { + return this.loadTsFileAllowedDirs.length == 0 + ? getLoadTsFileDirs() + : this.loadTsFileAllowedDirs; + } + + public Path[] getLoadTsFileAllowedDirCanonicalPaths() throws FileNotFoundException { + return (this.loadTsFileAllowedDirs.length == 0 + ? this.loadTsFileDirCanonicalPaths + : this.loadTsFileAllowedDirCanonicalPaths) + .getPaths(); + } + + public boolean isLoadTsFileSourcePathCheckEnabled() { + return loadTsFileSourcePathCheckEnabled; + } + + public void setLoadTsFileSourcePathCheckEnabled(boolean loadTsFileSourcePathCheckEnabled) { + this.loadTsFileSourcePathCheckEnabled = loadTsFileSourcePathCheckEnabled; + } + + public void setLoadTsFileAllowedDirs(String[] loadTsFileAllowedDirs) { + final String[] newLoadTsFileAllowedDirs = new String[loadTsFileAllowedDirs.length]; + for (int i = 0; i < loadTsFileAllowedDirs.length; i++) { + newLoadTsFileAllowedDirs[i] = addDataHomeDir(loadTsFileAllowedDirs[i]); + } + this.loadTsFileAllowedDirs = newLoadTsFileAllowedDirs; + this.loadTsFileAllowedDirCanonicalPaths = canonicalPaths(newLoadTsFileAllowedDirs); + } + public void formulateLoadTsFileDirs(String[][] tierDataDirs) { if (tierDataDirs.length < 1) { logger.warn("No data directory is set. loadTsFileDirs is kept as the default value."); @@ -1613,6 +1650,45 @@ public void formulateLoadTsFileDirs(String[][] tierDataDirs) { // or the newLoadTsFileDirs will be used in the middle of the process // and cause the undefined behavior. this.loadTsFileDirs = newLoadTsFileDirs; + this.loadTsFileDirCanonicalPaths = canonicalPaths(newLoadTsFileDirs); + } + + private static CanonicalPaths canonicalPaths(final String[] dirs) { + final Path[] paths = new Path[dirs.length]; + for (int i = 0; i < dirs.length; i++) { + try { + paths[i] = new File(dirs[i]).getCanonicalFile().toPath(); + } catch (final IOException e) { + return new CanonicalPaths( + String.format( + "Failed to resolve canonical path for Load TsFile allowed directory %s: %s", + dirs[i], e.getMessage())); + } + } + return new CanonicalPaths(paths); + } + + private static class CanonicalPaths { + + private final Path[] paths; + private final String errorMessage; + + private CanonicalPaths(final Path[] paths) { + this.paths = paths; + this.errorMessage = null; + } + + private CanonicalPaths(final String errorMessage) { + this.paths = new Path[0]; + this.errorMessage = errorMessage; + } + + private Path[] getPaths() throws FileNotFoundException { + if (errorMessage != null) { + throw new FileNotFoundException(errorMessage); + } + return paths; + } } public String getSchemaDir() { @@ -1872,22 +1948,6 @@ public void setUnSeqTsFileSize(long unSeqTsFileSize) { this.unSeqTsFileSize = unSeqTsFileSize; } - public int getRpcSelectorThreadCount() { - return rpcSelectorThreadCount; - } - - public void setRpcSelectorThreadCount(int rpcSelectorThreadCount) { - this.rpcSelectorThreadCount = rpcSelectorThreadCount; - } - - public int getRpcMinConcurrentClientNum() { - return rpcMinConcurrentClientNum; - } - - public void setRpcMinConcurrentClientNum(int rpcMinConcurrentClientNum) { - this.rpcMinConcurrentClientNum = rpcMinConcurrentClientNum; - } - public int getRpcMaxConcurrentClientNum() { return rpcMaxConcurrentClientNum; } @@ -2784,10 +2844,16 @@ public int getThriftMaxFrameSize() { } public void setThriftMaxFrameSize(int thriftMaxFrameSize) { - this.thriftMaxFrameSize = thriftMaxFrameSize; + this.thriftMaxFrameSize = + thriftMaxFrameSize <= 0 ? getDefaultThriftMaxFrameSize() : thriftMaxFrameSize; BaseRpcTransportFactory.setThriftMaxFrameSize(this.thriftMaxFrameSize); } + private static int getDefaultThriftMaxFrameSize() { + return Math.min( + 64 * 1024 * 1024, (int) Math.min(Runtime.getRuntime().maxMemory() / 64, Integer.MAX_VALUE)); + } + public int getThriftDefaultBufferSize() { return thriftDefaultBufferSize; } @@ -3332,6 +3398,14 @@ public void setMaxClientNumForEachNode(int maxClientNumForEachNode) { this.maxClientNumForEachNode = maxClientNumForEachNode; } + public int getMaxIdleClientNumForEachNode() { + return maxIdleClientNumForEachNode; + } + + public void setMaxIdleClientNumForEachNode(int maxIdleClientNumForEachNode) { + this.maxIdleClientNumForEachNode = maxIdleClientNumForEachNode; + } + public int getSelectorNumOfClientManager() { return selectorNumOfClientManager; } @@ -3478,14 +3552,6 @@ public void setCoordinatorReadExecutorSize(int coordinatorReadExecutorSize) { this.coordinatorReadExecutorSize = coordinatorReadExecutorSize; } - public int getCoordinatorWriteExecutorSize() { - return coordinatorWriteExecutorSize; - } - - public void setCoordinatorWriteExecutorSize(int coordinatorWriteExecutorSize) { - this.coordinatorWriteExecutorSize = coordinatorWriteExecutorSize; - } - public TEndPoint getAddressAndPort() { return new TEndPoint(rpcAddress, rpcPort); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index c755381e88dd9..b380a31eb6486 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -287,13 +287,23 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException String.valueOf(conf.getMaxClientNumForEachNode())) .trim())); - conf.setSelectorNumOfClientManager( + int dnMaxIdleClientNumForEachNode = Integer.parseInt( - properties - .getProperty( - "dn_selector_thread_count_of_client_manager", - String.valueOf(conf.getSelectorNumOfClientManager())) - .trim())); + properties.getProperty( + "dn_max_idle_client_count_for_each_node_in_client_manager", + String.valueOf(conf.getMaxIdleClientNumForEachNode()))); + if (dnMaxIdleClientNumForEachNode >= 0) { + conf.setMaxIdleClientNumForEachNode(dnMaxIdleClientNumForEachNode); + } + + int dnSelectorNumOfClientManager = + Integer.parseInt( + properties.getProperty( + "dn_selector_thread_nums_of_client_manager", + String.valueOf(conf.getSelectorNumOfClientManager()))); + if (dnSelectorNumOfClientManager > 0) { + conf.setSelectorNumOfClientManager(dnSelectorNumOfClientManager); + } conf.setRpcPort( Integer.parseInt( @@ -775,28 +785,6 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException properties.getProperty( "0.13_data_insert_adapt", String.valueOf(conf.isEnable13DataInsertAdapt())))); - int rpcSelectorThreadNum = - Integer.parseInt( - properties.getProperty( - "dn_rpc_selector_thread_count", - Integer.toString(conf.getRpcSelectorThreadCount()).trim())); - if (rpcSelectorThreadNum <= 0) { - rpcSelectorThreadNum = 1; - } - - conf.setRpcSelectorThreadCount(rpcSelectorThreadNum); - - int minConcurrentClientNum = - Integer.parseInt( - properties.getProperty( - "dn_rpc_min_concurrent_client_num", - Integer.toString(conf.getRpcMinConcurrentClientNum()).trim())); - if (minConcurrentClientNum <= 0) { - minConcurrentClientNum = Runtime.getRuntime().availableProcessors(); - } - - conf.setRpcMinConcurrentClientNum(minConcurrentClientNum); - int maxConcurrentClientNum = Integer.parseInt( properties.getProperty( @@ -861,10 +849,6 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException properties.getProperty( "dn_thrift_max_frame_size", String.valueOf(conf.getThriftMaxFrameSize())))); - if (conf.getThriftMaxFrameSize() < IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2) { - conf.setThriftMaxFrameSize(IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2); - } - conf.setThriftDefaultBufferSize( Integer.parseInt( properties.getProperty( @@ -1028,11 +1012,6 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException properties.getProperty( "coordinator_read_executor_size", Integer.toString(conf.getCoordinatorReadExecutorSize())))); - conf.setCoordinatorWriteExecutorSize( - Integer.parseInt( - properties.getProperty( - "coordinator_write_executor_size", - Integer.toString(conf.getCoordinatorWriteExecutorSize())))); // Commons commonDescriptor.loadCommonProps(properties); @@ -2470,6 +2449,17 @@ private void loadLoadTsFileProps(TrimProperties properties) throws IOException { properties.getProperty( "load_write_throughput_bytes_per_second", String.valueOf(conf.getLoadWriteThroughputBytesPerSecond())))); + + conf.setLoadTsFileAllowedDirs( + Arrays.stream(properties.getProperty("load_tsfile_allowed_dirs", "").trim().split(",")) + .filter(dir -> !dir.isEmpty()) + .toArray(String[]::new)); + conf.setLoadTsFileSourcePathCheckEnabled( + Boolean.parseBoolean( + properties.getProperty( + "load_tsfile_source_path_check_enable", + Boolean.toString(conf.isLoadTsFileSourcePathCheckEnabled())))); + conf.setLoadTabletConversionThresholdBytes( Long.parseLong( properties.getProperty( @@ -2581,6 +2571,25 @@ private void loadLoadTsFileHotModifiedProp(TrimProperties properties) throws IOE ConfigurationFileUtils.getConfigurationDefaultValue( "load_write_throughput_bytes_per_second")))); + conf.setLoadTsFileAllowedDirs( + Arrays.stream( + properties + .getProperty( + "load_tsfile_allowed_dirs", + Optional.ofNullable( + ConfigurationFileUtils.getConfigurationDefaultValue( + "load_tsfile_allowed_dirs")) + .orElse("")) + .trim() + .split(",")) + .filter(dir -> !dir.isEmpty()) + .toArray(String[]::new)); + conf.setLoadTsFileSourcePathCheckEnabled( + Boolean.parseBoolean( + properties.getProperty( + "load_tsfile_source_path_check_enable", + Boolean.toString(conf.isLoadTsFileSourcePathCheckEnabled())))); + conf.setLoadActiveListeningEnable( Boolean.parseBoolean( properties.getProperty( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java index 7dbc720a4a852..fa8e1efc6d8ed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java @@ -123,8 +123,6 @@ private static ConsensusConfig buildConsensusConfig() { .setRpc( RPC.newBuilder() .setConnectionTimeoutInMs(CONF.getConnectionTimeoutInMS()) - .setRpcSelectorThreadNum(CONF.getRpcSelectorThreadCount()) - .setRpcMinConcurrentClientNum(CONF.getRpcMinConcurrentClientNum()) .setRpcMaxConcurrentClientNum(CONF.getRpcMaxConcurrentClientNum()) .setRpcThriftCompressionEnabled(CONF.isRpcThriftCompressionEnable()) .setSelectorNumOfClientManager(CONF.getSelectorNumOfClientManager()) @@ -151,8 +149,6 @@ private static ConsensusConfig buildConsensusConfig() { PipeConsensusConfig.RPC .newBuilder() .setConnectionTimeoutInMs(CONF.getConnectionTimeoutInMS()) - .setRpcSelectorThreadNum(CONF.getRpcSelectorThreadCount()) - .setRpcMinConcurrentClientNum(CONF.getRpcMinConcurrentClientNum()) .setRpcMaxConcurrentClientNum(CONF.getRpcMaxConcurrentClientNum()) .setIsRpcThriftCompressionEnabled(CONF.isRpcThriftCompressionEnable()) .setThriftServerAwaitTimeForStopService( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java index 437c15bdc1c6d..d47a8c4dbbd8a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeDataNodeRuntimeAgent.java @@ -75,7 +75,7 @@ public class PipeDataNodeRuntimeAgent implements IService { public synchronized void preparePipeResources( final ResourcesInformationHolder resourcesInformationHolder) throws StartupException { // Clean sender (connector) hardlink file dir and snapshot dir - PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.clean(); + PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.clean(this::registerPeriodicalJob); // Clean receiver file dir PipeDataNodeAgent.receiver().cleanPipeReceiverDirs(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTask.java index d33ec44a86e53..0d0b955c2109d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTask.java @@ -32,32 +32,32 @@ public class PipeDataNodeTask implements PipeTask { private final String pipeName; private final int regionId; - private final PipeTaskStage extractorStage; + private final PipeTaskStage sourceStage; private final PipeTaskStage processorStage; - private final PipeTaskStage connectorStage; + private final PipeTaskStage sinkStage; private volatile boolean isCompleted = false; public PipeDataNodeTask( final String pipeName, final int regionId, - final PipeTaskStage extractorStage, + final PipeTaskStage sourceStage, final PipeTaskStage processorStage, - final PipeTaskStage connectorStage) { + final PipeTaskStage sinkStage) { this.pipeName = pipeName; this.regionId = regionId; - this.extractorStage = extractorStage; + this.sourceStage = sourceStage; this.processorStage = processorStage; - this.connectorStage = connectorStage; + this.sinkStage = sinkStage; } @Override public void create() { final long startTime = System.currentTimeMillis(); - extractorStage.create(); + sourceStage.create(); processorStage.create(); - connectorStage.create(); + sinkStage.create(); LOGGER.info( "Create pipe DN task {} successfully within {} ms", this, @@ -67,9 +67,9 @@ public void create() { @Override public void drop() { final long startTime = System.currentTimeMillis(); - extractorStage.drop(); + sourceStage.drop(); processorStage.drop(); - connectorStage.drop(); + sinkStage.drop(); LOGGER.info( "Drop pipe DN task {} successfully within {} ms", this, @@ -79,9 +79,9 @@ public void drop() { @Override public void start() { final long startTime = System.currentTimeMillis(); - extractorStage.start(); + sourceStage.start(); processorStage.start(); - connectorStage.start(); + sinkStage.start(); LOGGER.info( "Start pipe DN task {} successfully within {} ms", this, @@ -91,9 +91,9 @@ public void start() { @Override public void stop() { final long startTime = System.currentTimeMillis(); - extractorStage.stop(); + sourceStage.stop(); processorStage.stop(); - connectorStage.stop(); + sinkStage.stop(); LOGGER.info( "Stop pipe DN task {} successfully within {} ms", this, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java index c437507f4b085..67b9460c15a12 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/PipeDataNodeTaskAgent.java @@ -374,7 +374,7 @@ public void stopAllPipesWithCriticalExceptionAndTrackException( ///////////////////////// Heartbeat ///////////////////////// public void collectPipeMetaList(final TDataNodeHeartbeatResp resp) throws TException { - if (!tryReadLockWithTimeOut( + if (!tryReadLockWithTimeOutInMs( CommonDescriptor.getInstance().getConfig().getDnConnectionTimeoutInMS() * 2L / 3)) { return; } @@ -586,7 +586,7 @@ public boolean hasPipeReleaseRegionRelatedResource(final int consensusGroupId) { } } - public boolean isFullSync(final PipeParameters parameters) { + public boolean isFullSync(final PipeParameters parameters) throws IllegalPathException { if (isSnapshotMode(parameters)) { return false; } @@ -600,7 +600,10 @@ public boolean isFullSync(final PipeParameters parameters) { Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY), EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE); - return isHistoryEnable && isRealtimeEnable; + return isHistoryEnable + && isRealtimeEnable + && DataRegionListeningFilter.parseInsertionDeletionListeningOptionPair(parameters) + .getLeft(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java index 4728f62c94fd5..f0cc462161287 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.source.schemaregion.IoTDBSchemaRegionSource; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -131,12 +132,20 @@ private void parseAndCollectEvent(final PipeTsFileInsertionEvent sourceEvent) th if (skipParsing || !forceTabletFormat && canSkipParsing4TsFileEvent(sourceEvent)) { collectEvent(sourceEvent); + if (sourceEvent.isGeneratedByHistoricalExtractor()) { + PipeTerminateEvent.markHistoricalTsFileUnsplit( + sourceEvent.getPipeName(), sourceEvent.getCreationTime(), regionId); + } return; } try { sourceEvent.consumeTabletInsertionEventsWithRetry( this::collectParsedRawTableEvent, "PipeEventCollector::parseAndCollectEvent"); + if (sourceEvent.isGeneratedByHistoricalExtractor()) { + PipeTerminateEvent.markHistoricalTsFileSplit( + sourceEvent.getPipeName(), sourceEvent.getCreationTime(), regionId); + } } finally { sourceEvent.close(); } @@ -178,7 +187,8 @@ private void parseAndCollectEvent(final PipeSchemaRegionWritePlanEvent deleteDat private void collectEvent(final Event event) { if (event instanceof EnrichedEvent) { - if (!((EnrichedEvent) event).increaseReferenceCount(PipeEventCollector.class.getName())) { + final EnrichedEvent enrichedEvent = (EnrichedEvent) event; + if (!enrichedEvent.increaseReferenceCount(PipeEventCollector.class.getName())) { LOGGER.warn("PipeEventCollector: The event {} is already released, skipping it.", event); isFailedToIncreaseReferenceCount = true; return; @@ -186,18 +196,28 @@ private void collectEvent(final Event event) { // Assign a commit id for this event in order to report progress in order. PipeEventCommitManager.getInstance() - .enrichWithCommitterKeyAndCommitId((EnrichedEvent) event, creationTime, regionId); + .enrichWithCommitterKeyAndCommitId(enrichedEvent, creationTime, regionId); + + // Assign a rebootTime for iotConsensusV2 + enrichedEvent.setRebootTimes(PipeDataNodeAgent.runtime().getRebootTimes()); - // Assign a rebootTime for pipeConsensus - ((EnrichedEvent) event).setRebootTimes(PipeDataNodeAgent.runtime().getRebootTimes()); + if (enrichedEvent.getPipeName() != null + && (pendingQueue.isEventFromDroppedPipe(enrichedEvent) + || (enrichedEvent.getCommitterKey() == null + && pendingQueue.isPipeDropped( + enrichedEvent.getPipeName(), creationTime, regionId)))) { + enrichedEvent.clearReferenceCount(PipeEventCollector.class.getName()); + return; + } } if (event instanceof PipeHeartbeatEvent) { ((PipeHeartbeatEvent) event).recordConnectorQueueSize(pendingQueue); } - pendingQueue.offer(event); - collectInvocationCount.incrementAndGet(); + if (pendingQueue.offer(event)) { + collectInvocationCount.incrementAndGet(); + } } public void resetFlags() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSinkSubtaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSinkSubtaskExecutor.java index 9a88ad74d7ccb..4abc8daed3002 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSinkSubtaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/execution/PipeSinkSubtaskExecutor.java @@ -31,7 +31,7 @@ public class PipeSinkSubtaskExecutor extends PipeSubtaskExecutor { public PipeSinkSubtaskExecutor() { super( PipeConfig.getInstance().getPipeSubtaskExecutorMaxThreadNum(), - ThreadName.PIPE_CONNECTOR_EXECUTOR_POOL.getName() + "-" + id.get(), + ThreadName.PIPE_SINK_EXECUTOR_POOL.getName() + "-" + id.get(), ThreadName.PIPE_SUBTASK_CALLBACK_EXECUTOR_POOL.getName() + "-" + id.getAndIncrement(), true); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskProcessorStage.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskProcessorStage.java index ddc194716d2d1..c5f58a248e265 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskProcessorStage.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskProcessorStage.java @@ -54,8 +54,8 @@ public class PipeTaskProcessorStage extends PipeTaskStage { * @param creationTime pipe creation time * @param pipeProcessorParameters used to create {@link PipeProcessor} * @param regionId {@link DataRegion} id - * @param pipeExtractorInputEventSupplier used to input {@link Event}s from {@link PipeExtractor} - * @param pipeConnectorOutputPendingQueue used to output {@link Event}s to {@link PipeConnector} + * @param pipeSourceInputEventSupplier used to input {@link Event}s from {@link PipeExtractor} + * @param pipeSinkOutputPendingQueue used to output {@link Event}s to {@link PipeConnector} * @throws PipeException if failed to {@link PipeProcessor#validate(PipeParameterValidator)} or * {@link PipeProcessor#customize(PipeParameters, PipeProcessorRuntimeConfiguration)}} */ @@ -64,8 +64,8 @@ public PipeTaskProcessorStage( final long creationTime, final PipeParameters pipeProcessorParameters, final int regionId, - final EventSupplier pipeExtractorInputEventSupplier, - final UnboundedBlockingPendingQueue pipeConnectorOutputPendingQueue, + final EventSupplier pipeSourceInputEventSupplier, + final UnboundedBlockingPendingQueue pipeSinkOutputPendingQueue, final PipeProcessorSubtaskExecutor executor, final PipeTaskMeta pipeTaskMeta, final boolean forceTabletFormat, @@ -99,22 +99,18 @@ public PipeTaskProcessorStage( // removed, the new subtask will have the same pipeName and regionId as the // old one, so we need creationTime to make their hash code different in the map. final String taskId = pipeName + "_" + regionId + "_" + creationTime; - final PipeEventCollector pipeConnectorOutputEventCollector = + final PipeEventCollector pipeSinkOutputEventCollector = new PipeEventCollector( - pipeConnectorOutputPendingQueue, - creationTime, - regionId, - forceTabletFormat, - skipParsing); + pipeSinkOutputPendingQueue, creationTime, regionId, forceTabletFormat, skipParsing); this.pipeProcessorSubtask = new PipeProcessorSubtask( taskId, pipeName, creationTime, regionId, - pipeExtractorInputEventSupplier, + pipeSourceInputEventSupplier, pipeProcessor, - pipeConnectorOutputEventCollector); + pipeSinkOutputEventCollector); this.executor = executor; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskSinkStage.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskSinkStage.java index a22fbb536d704..88eac560cde2b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskSinkStage.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskSinkStage.java @@ -38,7 +38,7 @@ public class PipeTaskSinkStage extends PipeTaskStage { protected final int regionId; protected final Supplier executor; - protected String connectorSubtaskId; + protected String sinkSubtaskId; public PipeTaskSinkStage( String pipeName, @@ -56,7 +56,7 @@ public PipeTaskSinkStage( } protected void registerSubtask() { - this.connectorSubtaskId = + this.sinkSubtaskId = PipeSinkSubtaskManager.instance() .register( executor, @@ -71,21 +71,20 @@ public void createSubtask() throws PipeException { @Override public void startSubtask() throws PipeException { - PipeSinkSubtaskManager.instance().start(connectorSubtaskId); + PipeSinkSubtaskManager.instance().start(sinkSubtaskId); } @Override public void stopSubtask() throws PipeException { - PipeSinkSubtaskManager.instance().stop(connectorSubtaskId); + PipeSinkSubtaskManager.instance().stop(sinkSubtaskId); } @Override public void dropSubtask() throws PipeException { - PipeSinkSubtaskManager.instance() - .deregister(pipeName, creationTime, regionId, connectorSubtaskId); + PipeSinkSubtaskManager.instance().deregister(pipeName, creationTime, regionId, sinkSubtaskId); } public UnboundedBlockingPendingQueue getPipeSinkPendingQueue() { - return PipeSinkSubtaskManager.instance().getPipeSinkPendingQueue(connectorSubtaskId); + return PipeSinkSubtaskManager.instance().getPipeSinkPendingQueue(sinkSubtaskId); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskSourceStage.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskSourceStage.java index 6acc0fc3d4af2..240b5499e92b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskSourceStage.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/stage/PipeTaskSourceStage.java @@ -44,32 +44,31 @@ public class PipeTaskSourceStage extends PipeTaskStage { public PipeTaskSourceStage( String pipeName, long creationTime, - PipeParameters extractorParameters, + PipeParameters sourceParameters, int regionId, PipeTaskMeta pipeTaskMeta) { pipeExtractor = StorageEngine.getInstance().getAllDataRegionIds().contains(new DataRegionId(regionId)) - ? PipeDataNodeAgent.plugin().dataRegion().reflectSource(extractorParameters) - : PipeDataNodeAgent.plugin().schemaRegion().reflectSource(extractorParameters); + ? PipeDataNodeAgent.plugin().dataRegion().reflectSource(sourceParameters) + : PipeDataNodeAgent.plugin().schemaRegion().reflectSource(sourceParameters); - // Validate and customize should be called before createSubtask. this allows extractor exposing + // Validate and customize should be called before createSubtask. this allows source exposing // exceptions in advance. try { - // 1. Validate extractor parameters - pipeExtractor.validate(new PipeParameterValidator(extractorParameters)); + // 1. Validate source parameters + pipeExtractor.validate(new PipeParameterValidator(sourceParameters)); - // 2. Customize extractor + // 2. Customize source final PipeTaskRuntimeConfiguration runtimeConfiguration = new PipeTaskRuntimeConfiguration( new PipeTaskSourceRuntimeEnvironment(pipeName, creationTime, regionId, pipeTaskMeta)); - pipeExtractor.customize(extractorParameters, runtimeConfiguration); + pipeExtractor.customize(sourceParameters, runtimeConfiguration); } catch (Exception e) { try { pipeExtractor.close(); } catch (Exception closeException) { LOGGER.warn( - "Failed to close extractor after failed to initialize extractor. " - + "Ignore this exception.", + "Failed to close source after failed to initialize source. " + "Ignore this exception.", closeException); } throw new PipeException(e.getMessage(), e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java index ca5a8d0f4dba7..193693c5a95bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java @@ -191,15 +191,15 @@ protected boolean executeOnce() throws Exception { } catch (final PipeRuntimeOutOfMemoryCriticalException e) { PipeLogger.log( LOGGER::info, - e, - "Temporarily out of memory in pipe event processing, will wait for the memory to release."); + "Temporarily out of memory in pipe event processing, will wait for the memory to release. Message: %s", + e.getMessage()); return false; } catch (final Exception e) { if (ExceptionUtils.getRootCause(e) instanceof PipeRuntimeOutOfMemoryCriticalException) { PipeLogger.log( LOGGER::info, - e, - "Temporarily out of memory in pipe event processing, will wait for the memory to release."); + "Temporarily out of memory in pipe event processing, will wait for the memory to release. Message: %s", + e.getMessage()); return false; } if (!isClosed.get()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java index 6d227ac31fdce..e35763c30123a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeRealtimePriorityBlockingQueue.java @@ -73,7 +73,9 @@ public PipeRealtimePriorityBlockingQueue() { @Override public boolean offer(final Event event) { - checkBeforeOffer(event); + if (!checkBeforeOffer(event)) { + return false; + } if (event instanceof TsFileInsertionEvent) { tsfileInsertEventDeque.add((TsFileInsertionEvent) event); @@ -356,13 +358,18 @@ public void discardAllEvents() { } @Override - public void discardEventsOfPipe(final String pipeNameToDrop, final int regionId) { - super.discardEventsOfPipe(pipeNameToDrop, regionId); + public void discardEventsOfPipe( + final String pipeNameToDrop, final long creationTimeToDrop, final int regionId) { + discardEventsOfPipe(new CommitterKey(pipeNameToDrop, creationTimeToDrop, regionId, -1)); + } + + @Override + public void discardEventsOfPipe(final CommitterKey committerKey) { + super.discardEventsOfPipe(committerKey); tsfileInsertEventDeque.removeIf( event -> { if (event instanceof EnrichedEvent - && pipeNameToDrop.equals(((EnrichedEvent) event).getPipeName()) - && regionId == ((EnrichedEvent) event).getRegionId()) { + && isEventFromPipe((EnrichedEvent) event, committerKey)) { if (((EnrichedEvent) event) .clearReferenceCount(PipeRealtimePriorityBlockingQueue.class.getName())) { eventCounter.decreaseEventCount(event); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtask.java index 4b4794891f6cc..c855eb5714013 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtask.java @@ -21,10 +21,12 @@ import org.apache.iotdb.commons.exception.pipe.PipeRuntimeException; import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.agent.task.subtask.PipeAbstractSinkSubtask; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.sink.protocol.IoTDBSink; +import org.apache.iotdb.commons.pipe.sink.protocol.PipeConnectorWithEventDiscard; import org.apache.iotdb.commons.utils.ErrorHandlingCommonUtils; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.pipe.event.UserDefinedEnrichedEvent; @@ -32,6 +34,7 @@ import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; import org.apache.iotdb.db.pipe.metric.schema.PipeSchemaRegionSinkMetrics; import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionSinkMetrics; +import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBDataRegionAirGapSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBDataRegionSyncSink; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -199,9 +202,9 @@ public void close() { * When a pipe is dropped, the connector maybe reused and will not be closed. So we just discard * its queued events in the output pipe connector. */ - public void discardEventsOfPipe(final String pipeNameToDrop, int regionId) { + public void discardEventsOfPipe(final CommitterKey committerKey) { // Try to remove the events as much as possible - inputPendingQueue.discardEventsOfPipe(pipeNameToDrop, regionId); + inputPendingQueue.discardEventsOfPipe(committerKey); try { increaseHighPriorityTaskCount(); @@ -214,8 +217,7 @@ public void discardEventsOfPipe(final String pipeNameToDrop, int regionId) { // use a new thread to stop all the pipes, we will not encounter deadlock here. Or else we // will. if (lastEvent instanceof EnrichedEvent - && pipeNameToDrop.equals(((EnrichedEvent) lastEvent).getPipeName()) - && regionId == ((EnrichedEvent) lastEvent).getRegionId()) { + && isEventFromPipe((EnrichedEvent) lastEvent, committerKey)) { // Do not clear the last event's reference counts because it may be on transferring lastEvent = null; // Submit self to avoid that the lastEvent has been retried "max times" times and has @@ -237,8 +239,7 @@ public void discardEventsOfPipe(final String pipeNameToDrop, int regionId) { // clear the lastExceptionEvent. It's safe to potentially clear it twice because we have the // "nonnull" detection. if (lastExceptionEvent instanceof EnrichedEvent - && pipeNameToDrop.equals(((EnrichedEvent) lastExceptionEvent).getPipeName()) - && regionId == ((EnrichedEvent) lastExceptionEvent).getRegionId()) { + && isEventFromPipe((EnrichedEvent) lastExceptionEvent, committerKey)) { clearReferenceCountAndReleaseLastExceptionEvent(); } } @@ -246,11 +247,19 @@ public void discardEventsOfPipe(final String pipeNameToDrop, int regionId) { decreaseHighPriorityTaskCount(); } - if (outputPipeConnector instanceof IoTDBSink) { - ((IoTDBSink) outputPipeConnector).discardEventsOfPipe(pipeNameToDrop, regionId); + if (outputPipeConnector instanceof PipeConnectorWithEventDiscard) { + ((PipeConnectorWithEventDiscard) outputPipeConnector).discardEventsOfPipe(committerKey); } } + private static boolean isEventFromPipe( + final EnrichedEvent event, final CommitterKey committerKey) { + return committerKey.getPipeName().equals(event.getPipeName()) + && committerKey.getCreationTime() == event.getCreationTime() + && committerKey.getRegionId() == event.getRegionId() + && (committerKey.getRestartTimes() < 0 || committerKey.equals(event.getCommitterKey())); + } + //////////////////////////// APIs provided for metric framework //////////////////////////// public String getAttributeSortedString() { @@ -295,6 +304,9 @@ public int getBatchSize() { if (outputPipeConnector instanceof IoTDBDataRegionSyncSink) { return ((IoTDBDataRegionSyncSink) outputPipeConnector).getBatchSize(); } + if (outputPipeConnector instanceof IoTDBDataRegionAirGapSink) { + return ((IoTDBDataRegionAirGapSink) outputPipeConnector).getBatchSize(); + } return 0; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskLifeCycle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskLifeCycle.java index 35f7983075dfe..61a064e1637a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskLifeCycle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskLifeCycle.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.agent.task.subtask.sink; import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.db.pipe.agent.task.execution.PipeSinkSubtaskExecutor; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.pipe.api.event.Event; @@ -86,18 +87,17 @@ public synchronized void register() { * Otherwise, the {@link PipeSinkSubtaskLifeCycle#runningTaskCount} might be inconsistent with the * {@link PipeSinkSubtaskLifeCycle#registeredTaskCount} because of parallel connector scheduling. * - * @param pipeNameToDeregister pipe name - * @param regionId region id + * @param committerKey committer key of the pipe task to deregister * @return {@code true} if the {@link PipeSinkSubtask} is out of life cycle, indicating that the * {@link PipeSinkSubtask} should never be used again * @throws IllegalStateException if {@link PipeSinkSubtaskLifeCycle#registeredTaskCount} <= 0 */ - public synchronized boolean deregister(final String pipeNameToDeregister, int regionId) { + public synchronized boolean deregister(final CommitterKey committerKey) { if (registeredTaskCount <= 0) { throw new IllegalStateException("registeredTaskCount <= 0"); } - subtask.discardEventsOfPipe(pipeNameToDeregister, regionId); + subtask.discardEventsOfPipe(committerKey); try { if (registeredTaskCount > 1) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java index caff425f790b3..817471c785ab6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskManager.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; @@ -209,7 +210,10 @@ public synchronized void deregister( // Shall not be empty final PipeSinkSubtaskExecutor executor = lifeCycles.get(0).executor; - lifeCycles.removeIf(o -> o.deregister(pipeName, regionId)); + final CommitterKey committerKey = + PipeEventCommitManager.getInstance().getCommitterKey(pipeName, creationTime, regionId); + + lifeCycles.removeIf(o -> o.deregister(committerKey)); if (lifeCycles.isEmpty()) { attributeSortedString2SubtaskLifeCycleMap.remove(attributeSortedString); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/heartbeat/PipeHeartbeatEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/heartbeat/PipeHeartbeatEvent.java index ad29b2854424a..567e14f4fe386 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/heartbeat/PipeHeartbeatEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/heartbeat/PipeHeartbeatEvent.java @@ -27,10 +27,10 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.db.pipe.metric.overview.PipeDataNodeSinglePipeMetrics; import org.apache.iotdb.db.pipe.metric.overview.PipeHeartbeatEventMetrics; +import org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor.RingBuffer; import org.apache.iotdb.db.utils.DateTimeUtils; import org.apache.iotdb.pipe.api.event.Event; -import com.lmax.disruptor.RingBuffer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -183,7 +183,7 @@ public void onTransferred() { /////////////////////////////// Queue size Reporting /////////////////////////////// - public void recordDisruptorSize(final RingBuffer ringBuffer) { + public void recordDisruptorSize(final RingBuffer ringBuffer) { if (shouldPrintMessage) { disruptorSize = ringBuffer.getBufferSize() - (int) ringBuffer.remainingCapacity(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java index 33bf0a5925c39..a8579b32bb771 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRow.java @@ -152,7 +152,10 @@ public Type getDataType(final int columnIndex) { @Override public boolean isNull(final int columnIndex) { - return bitMaps[columnIndex].isMarked(rowIndex); + return bitMaps != null + && columnIndex < bitMaps.length + && bitMaps[columnIndex] != null + && bitMaps[columnIndex].isMarked(rowIndex); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowCollector.java index 42560f23a5c59..c26b05f6756b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowCollector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/row/PipeRowCollector.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletEventConverter; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; import org.apache.iotdb.pipe.api.access.Row; import org.apache.iotdb.pipe.api.collector.RowCollector; @@ -66,27 +67,26 @@ public void collectRow(Row row) { Pair rowCountAndMemorySize = PipeMemoryWeightUtil.calculateTabletRowCountAndMemory(pipeRow); tablet = new Tablet(deviceId, measurementSchemaList, rowCountAndMemorySize.getLeft()); - tablet.initBitMaps(); isAligned = pipeRow.isAligned(); } final int rowIndex = tablet.rowSize; - tablet.addTimestamp(rowIndex, row.getTime()); + PipeTabletUtils.putTimestamp(tablet, rowIndex, row.getTime()); for (int i = 0; i < row.size(); i++) { final Object value = row.getObject(i); - if (value instanceof org.apache.iotdb.pipe.api.type.Binary) { - tablet.addValue( - measurementSchemaArray[i].getMeasurementId(), - rowIndex, - PipeBinaryTransformer.transformToBinary((org.apache.iotdb.pipe.api.type.Binary) value)); - } else { - tablet.addValue(measurementSchemaArray[i].getMeasurementId(), rowIndex, value); - } + PipeTabletUtils.putValue( + tablet, + rowIndex, + i, + measurementSchemaArray[i].getType(), + value instanceof org.apache.iotdb.pipe.api.type.Binary + ? PipeBinaryTransformer.transformToBinary( + (org.apache.iotdb.pipe.api.type.Binary) value) + : value); if (row.isNull(i)) { - tablet.bitMaps[i].mark(rowIndex); + PipeTabletUtils.markNullValue(tablet, rowIndex, i); } } - tablet.rowSize++; if (tablet.rowSize == tablet.getMaxRowNumber()) { collectTabletInsertionEvent(); @@ -95,6 +95,7 @@ public void collectRow(Row row) { private void collectTabletInsertionEvent() { if (tablet != null) { + PipeTabletUtils.compactBitMaps(tablet); tabletInsertionEventList.add( new PipeRawTabletInsertionEvent( tablet, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java index d322291934ffe..261e3c7a8b324 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java @@ -323,6 +323,7 @@ public boolean isAligned() { public Tablet convertToTablet() { if (!shouldParseTimeOrPattern()) { + PipeTabletUtils.compactBitMaps(tablet); return tablet; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletUtils.java new file mode 100644 index 0000000000000..0a6b073b5b6ca --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletUtils.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.event.common.tablet; + +import org.apache.iotdb.db.utils.BitMapUtils; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.UnSupportedDataTypeException; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.time.LocalDate; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public final class PipeTabletUtils { + + private PipeTabletUtils() {} + + public static final class TabletStringInternPool { + + private final Map internedStrings = new HashMap<>(); + + public String intern(final String value) { + if (Objects.isNull(value)) { + return null; + } + + final String internedValue = internedStrings.get(value); + if (Objects.nonNull(internedValue)) { + return internedValue; + } + + internedStrings.put(value, value); + return value; + } + + public void intern(final String[] values) { + if (Objects.isNull(values)) { + return; + } + + for (int i = 0; i < values.length; ++i) { + values[i] = intern(values[i]); + } + } + + public void intern(final List values) { + if (Objects.isNull(values)) { + return; + } + + for (int i = 0; i < values.size(); ++i) { + values.set(i, intern(values.get(i))); + } + } + + public Tablet intern(final Tablet tablet) { + if (Objects.isNull(tablet)) { + return null; + } + + tablet.setDeviceId(intern(tablet.deviceId)); + internMeasurementSchemas(tablet.getSchemas()); + return tablet; + } + + public void internMeasurementSchemas(final List schemas) { + if (Objects.isNull(schemas)) { + return; + } + + for (final MeasurementSchema schema : schemas) { + intern(schema); + } + } + + public MeasurementSchema intern(final MeasurementSchema schema) { + if (Objects.isNull(schema)) { + return null; + } + + schema.setMeasurementId(intern(schema.getMeasurementId())); + schema.setProps(intern(schema.getProps())); + return schema; + } + + private Map intern(final Map props) { + if (Objects.isNull(props) || props.isEmpty()) { + return props; + } + + final Map internedProps = new HashMap<>(props.size()); + for (final Map.Entry entry : props.entrySet()) { + internedProps.put(intern(entry.getKey()), intern(entry.getValue())); + } + return internedProps; + } + } + + public static Tablet internTablet( + final Tablet tablet, final TabletStringInternPool tabletStringInternPool) { + return Objects.nonNull(tabletStringInternPool) ? tabletStringInternPool.intern(tablet) : tablet; + } + + public static void compactBitMaps(final Tablet tablet) { + if (Objects.isNull(tablet)) { + return; + } + tablet.bitMaps = compactBitMaps(tablet.bitMaps, tablet.rowSize); + } + + public static BitMap[] compactBitMaps(final BitMap[] bitMaps, final int rowCount) { + return BitMapUtils.compactBitMaps(bitMaps, rowCount); + } + + public static BitMap[] copyBitMapsOrCreateEmpty(final Tablet tablet) { + final BitMap[] bitMaps = tablet.bitMaps; + return Objects.nonNull(bitMaps) + ? Arrays.copyOf(bitMaps, bitMaps.length) + : new BitMap[getColumnCount(tablet)]; + } + + public static void markNullValue(final Tablet tablet, final int rowIndex, final int columnIndex) { + final BitMap[] bitMaps = ensureBitMaps(tablet, columnIndex + 1); + if (Objects.isNull(bitMaps[columnIndex])) { + bitMaps[columnIndex] = new BitMap(tablet.getMaxRowNumber()); + } + bitMaps[columnIndex].mark(rowIndex); + } + + public static void putTimestamp(final Tablet tablet, final int rowIndex, final long timestamp) { + tablet.timestamps[rowIndex] = timestamp; + tablet.rowSize = Math.max(tablet.rowSize, rowIndex + 1); + } + + public static void putValue( + final Tablet tablet, + final int rowIndex, + final int columnIndex, + final TSDataType dataType, + final Object value) { + switch (dataType) { + case BOOLEAN: + ((boolean[]) tablet.values[columnIndex])[rowIndex] = (Boolean) value; + break; + case INT32: + ((int[]) tablet.values[columnIndex])[rowIndex] = (Integer) value; + break; + case DATE: + ((LocalDate[]) tablet.values[columnIndex])[rowIndex] = (LocalDate) value; + break; + case INT64: + case TIMESTAMP: + ((long[]) tablet.values[columnIndex])[rowIndex] = (Long) value; + break; + case FLOAT: + ((float[]) tablet.values[columnIndex])[rowIndex] = (Float) value; + break; + case DOUBLE: + ((double[]) tablet.values[columnIndex])[rowIndex] = (Double) value; + break; + case TEXT: + case BLOB: + case STRING: + ((Binary[]) tablet.values[columnIndex])[rowIndex] = toBinary(value); + break; + default: + throw new UnSupportedDataTypeException("Unsupported data type: " + dataType); + } + unmarkNullValue(tablet, rowIndex, columnIndex); + } + + private static void unmarkNullValue( + final Tablet tablet, final int rowIndex, final int columnIndex) { + final BitMap[] bitMaps = tablet.bitMaps; + if (Objects.nonNull(bitMaps) + && columnIndex < bitMaps.length + && Objects.nonNull(bitMaps[columnIndex])) { + bitMaps[columnIndex].unmark(rowIndex); + } + } + + private static BitMap[] ensureBitMaps(final Tablet tablet, final int minColumnCount) { + final int columnCount = Math.max(getColumnCount(tablet), minColumnCount); + BitMap[] bitMaps = tablet.bitMaps; + if (Objects.isNull(bitMaps)) { + bitMaps = new BitMap[columnCount]; + tablet.bitMaps = bitMaps; + } else if (bitMaps.length < columnCount) { + final BitMap[] expandedBitMaps = new BitMap[columnCount]; + System.arraycopy(bitMaps, 0, expandedBitMaps, 0, bitMaps.length); + bitMaps = expandedBitMaps; + tablet.bitMaps = bitMaps; + } + return bitMaps; + } + + private static int getColumnCount(final Tablet tablet) { + if (Objects.nonNull(tablet.getSchemas())) { + return tablet.getSchemas().size(); + } + return Objects.nonNull(tablet.values) ? tablet.values.length : 0; + } + + private static Binary toBinary(final Object value) { + if (Objects.isNull(value)) { + return Binary.EMPTY_VALUE; + } + if (value instanceof Binary) { + return (Binary) value; + } + if (value instanceof byte[]) { + return new Binary((byte[]) value); + } + if (value instanceof String) { + return new Binary(((String) value).getBytes(TSFileConfig.STRING_CHARSET)); + } + throw new IllegalArgumentException( + String.format("Expected Binary, byte[] or String, but was %s.", value.getClass())); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java index 133dbb5bff8a0..d8c2bccaa97c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java @@ -198,6 +198,7 @@ private void parse(final InsertRowNode insertRowNode, final PipePattern pattern) } this.rowCount = this.timestampColumn.length; + this.nullValueColumnBitmaps = PipeTabletUtils.compactBitMaps(nullValueColumnBitmaps, rowCount); if (this.rowCount == 0 && LOGGER.isDebugEnabled()) { LOGGER.debug( "InsertRowNode({}) is parsed to zero rows according to the pattern({}) and time range [{}, {}], the corresponding source event({}) will be ignored.", @@ -217,7 +218,6 @@ private void parse(final InsertTabletNode insertTabletNode, final PipePattern pa this.isAligned = insertTabletNode.isAligned(); final long[] originTimestampColumn = insertTabletNode.getTimes(); - final int originRowSize = originTimestampColumn.length; final List rowIndexList = generateRowIndexList(originTimestampColumn); this.timestampColumn = rowIndexList.stream().mapToLong(i -> originTimestampColumn[i]).toArray(); @@ -243,18 +243,7 @@ private void parse(final InsertTabletNode insertTabletNode, final PipePattern pa final String[] originColumnNameStringList = insertTabletNode.getMeasurements(); final TSDataType[] originValueColumnTypes = insertTabletNode.getDataTypes(); final Object[] originValueColumns = insertTabletNode.getColumns(); - final BitMap[] originBitMapList = - (insertTabletNode.getBitMaps() == null - ? IntStream.range(0, originColumnSize) - .boxed() - .map(o -> new BitMap(originRowSize)) - .toArray(BitMap[]::new) - : insertTabletNode.getBitMaps()); - for (int i = 0; i < originBitMapList.length; i++) { - if (originBitMapList[i] == null) { - originBitMapList[i] = new BitMap(originRowSize); - } - } + final BitMap[] originBitMapList = insertTabletNode.getBitMaps(); for (int i = 0; i < originColumnIndex2FilteredColumnIndexMapperList.length; i++) { if (originColumnIndex2FilteredColumnIndexMapperList[i] != null) { @@ -277,7 +266,7 @@ private void parse(final InsertTabletNode insertTabletNode, final PipePattern pa originValueColumns[i], rowIndexList, false, - originBitMapList[i], + getBitMap(originBitMapList, i), bitMap); } this.nullValueColumnBitmaps[filteredColumnIndex] = bitMap; @@ -285,6 +274,7 @@ private void parse(final InsertTabletNode insertTabletNode, final PipePattern pa } this.rowCount = this.timestampColumn.length; + this.nullValueColumnBitmaps = PipeTabletUtils.compactBitMaps(nullValueColumnBitmaps, rowCount); if (rowCount == 0 && LOGGER.isDebugEnabled()) { LOGGER.debug( "InsertTabletNode({}) is parsed to zero rows according to the pattern({}) and time range [{}, {}], the corresponding source event({}) will be ignored.", @@ -338,18 +328,7 @@ private void parse(final Tablet tablet, final boolean isAligned, final PipePatte } final Object[] originValueColumns = tablet.values; // we do not reduce value columns here by origin row size - final BitMap[] originBitMapList = - tablet.bitMaps == null - ? IntStream.range(0, originColumnSize) - .boxed() - .map(o -> new BitMap(tablet.getMaxRowNumber())) - .toArray(BitMap[]::new) - : tablet.bitMaps; // We do not reduce bitmaps here by origin row size - for (int i = 0; i < originBitMapList.length; i++) { - if (originBitMapList[i] == null) { - originBitMapList[i] = new BitMap(tablet.getMaxRowNumber()); - } - } + final BitMap[] originBitMapList = tablet.bitMaps; for (int i = 0; i < originColumnIndex2FilteredColumnIndexMapperList.length; i++) { if (originColumnIndex2FilteredColumnIndexMapperList[i] != null) { @@ -372,7 +351,7 @@ private void parse(final Tablet tablet, final boolean isAligned, final PipePatte originValueColumns[i], rowIndexList, false, - originBitMapList[i], + getBitMap(originBitMapList, i), bitMap); } this.nullValueColumnBitmaps[filteredColumnIndex] = bitMap; @@ -380,6 +359,7 @@ private void parse(final Tablet tablet, final boolean isAligned, final PipePatte } this.rowCount = this.timestampColumn.length; + this.nullValueColumnBitmaps = PipeTabletUtils.compactBitMaps(nullValueColumnBitmaps, rowCount); if (this.rowCount == 0 && LOGGER.isDebugEnabled()) { LOGGER.debug( "Tablet({}) is parsed to zero rows according to the pattern({}) and time range [{}, {}], the corresponding source event({}) will be ignored.", @@ -471,7 +451,7 @@ private static Object filterValueColumnsByRowIndexList( : (int[]) originValueColumn; final int[] valueColumns = new int[rowIndexList.size()]; for (int i = 0; i < rowIndexList.size(); ++i) { - if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { + if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { valueColumns[i] = 0; nullValueColumnBitmap.mark(i); } else { @@ -493,7 +473,7 @@ private static Object filterValueColumnsByRowIndexList( : (LocalDate[]) originValueColumn; for (int i = 0; i < rowIndexList.size(); ++i) { - if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { + if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { valueColumns[i] = EMPTY_LOCALDATE; nullValueColumnBitmap.mark(i); } else { @@ -507,7 +487,7 @@ private static Object filterValueColumnsByRowIndexList( ? new int[] {(int) originValueColumn} : (int[]) originValueColumn; for (int i = 0; i < rowIndexList.size(); ++i) { - if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { + if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { valueColumns[i] = EMPTY_LOCALDATE; nullValueColumnBitmap.mark(i); } else { @@ -527,7 +507,7 @@ private static Object filterValueColumnsByRowIndexList( : (long[]) originValueColumn; final long[] valueColumns = new long[rowIndexList.size()]; for (int i = 0; i < rowIndexList.size(); ++i) { - if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { + if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { valueColumns[i] = 0L; nullValueColumnBitmap.mark(i); } else { @@ -544,7 +524,7 @@ private static Object filterValueColumnsByRowIndexList( : (float[]) originValueColumn; final float[] valueColumns = new float[rowIndexList.size()]; for (int i = 0; i < rowIndexList.size(); ++i) { - if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { + if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { valueColumns[i] = 0F; nullValueColumnBitmap.mark(i); } else { @@ -561,7 +541,7 @@ private static Object filterValueColumnsByRowIndexList( : (double[]) originValueColumn; final double[] valueColumns = new double[rowIndexList.size()]; for (int i = 0; i < rowIndexList.size(); ++i) { - if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { + if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { valueColumns[i] = 0D; nullValueColumnBitmap.mark(i); } else { @@ -578,7 +558,7 @@ private static Object filterValueColumnsByRowIndexList( : (boolean[]) originValueColumn; final boolean[] valueColumns = new boolean[rowIndexList.size()]; for (int i = 0; i < rowIndexList.size(); ++i) { - if (originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { + if (isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { valueColumns[i] = false; nullValueColumnBitmap.mark(i); } else { @@ -599,7 +579,7 @@ private static Object filterValueColumnsByRowIndexList( for (int i = 0; i < rowIndexList.size(); ++i) { if (Objects.isNull(binaryValueColumns[rowIndexList.get(i)]) || Objects.isNull(binaryValueColumns[rowIndexList.get(i)].getValues()) - || originNullValueColumnBitmap.isMarked(rowIndexList.get(i))) { + || isNullValue(originNullValueColumnBitmap, rowIndexList.get(i))) { valueColumns[i] = Binary.EMPTY_VALUE; nullValueColumnBitmap.mark(i); } else { @@ -659,6 +639,14 @@ private void fillNullValue( } } + private static BitMap getBitMap(final BitMap[] bitMaps, final int index) { + return Objects.nonNull(bitMaps) && index < bitMaps.length ? bitMaps[index] : null; + } + + private static boolean isNullValue(final BitMap bitMap, final int rowIndex) { + return Objects.nonNull(bitMap) && bitMap.isMarked(rowIndex); + } + //////////////////////////// process //////////////////////////// public List processRowByRow(final BiConsumer consumer) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java index a8bf19ebf3b17..f57c016e246a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java @@ -32,9 +32,16 @@ import org.apache.iotdb.db.pipe.agent.task.PipeDataNodeTask; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; /** * The {@link PipeTerminateEvent} is an {@link EnrichedEvent} that controls the termination of pipe, @@ -44,21 +51,37 @@ */ public class PipeTerminateEvent extends EnrichedEvent { + private static final Logger LOGGER = LoggerFactory.getLogger(PipeTerminateEvent.class); + private final int dataRegionId; private final boolean shouldMark; + private static final int TERMINATE_EXECUTOR_THREAD_COUNT = + IoTDBDescriptor.getInstance().getConfig().getPipeTaskThreadCount(); + + private static final int TERMINATE_EXECUTOR_QUEUE_SIZE = + Math.max(1024, TERMINATE_EXECUTOR_THREAD_COUNT * 64); + // Do not use call run policy to avoid deadlock - private static final ExecutorService terminateExecutor = - new WrappedThreadPoolExecutor( - 0, - IoTDBDescriptor.getInstance().getConfig().getPipeTaskThreadCount(), - 0L, - TimeUnit.SECONDS, - new ArrayBlockingQueue<>( - IoTDBDescriptor.getInstance().getConfig().getPipeTaskThreadCount()), - new IoTThreadFactory(ThreadName.PIPE_TERMINATE_EXECUTION_POOL.getName()), - ThreadName.PIPE_TERMINATE_EXECUTION_POOL.getName()); + private static final ExecutorService terminateExecutor = createTerminateExecutor(); + + private static final ConcurrentMap + HISTORICAL_TRANSFER_SUMMARY_COUNTER_MAP = new ConcurrentHashMap<>(); + + private static ExecutorService createTerminateExecutor() { + final WrappedThreadPoolExecutor executor = + new WrappedThreadPoolExecutor( + TERMINATE_EXECUTOR_THREAD_COUNT, + TERMINATE_EXECUTOR_THREAD_COUNT, + 60L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(TERMINATE_EXECUTOR_QUEUE_SIZE), + new IoTThreadFactory(ThreadName.PIPE_TERMINATE_EXECUTION_POOL.getName()), + ThreadName.PIPE_TERMINATE_EXECUTION_POOL.getName()); + executor.allowCoreThreadTimeOut(true); + return executor; + } public PipeTerminateEvent( final String pipeName, @@ -117,6 +140,18 @@ public boolean mayEventPathsOverlappedWithPattern() { } public void markCompleted() { + final HistoricalTransferSummary summary = + snapshotAndClearHistoricalTransferSummary(pipeName, creationTime, dataRegionId); + if (Objects.nonNull(summary)) { + LOGGER.info( + "Pipe {}@{}: terminate event committed for historical transfer. creationTime: {}, shouldMark: {}. {}", + pipeName, + dataRegionId, + creationTime, + shouldMark, + summary.toReportMessage()); + } + // To avoid deadlock if (shouldMark) { terminateExecutor.submit( @@ -131,4 +166,159 @@ public String toString() { + " - " + super.toString(); } + + public static void initializeHistoricalTransferSummary( + final String pipeName, + final long creationTime, + final int dataRegionId, + final long extractedHistoricalTsFileCount, + final long extractedHistoricalDeletionCount) { + HISTORICAL_TRANSFER_SUMMARY_COUNTER_MAP + .computeIfAbsent( + new HistoricalTransferKey(pipeName, creationTime, dataRegionId), + ignored -> new HistoricalTransferSummaryCounter()) + .initialize(extractedHistoricalTsFileCount, extractedHistoricalDeletionCount); + } + + public static void markHistoricalTsFileSkipped( + final String pipeName, final long creationTime, final int dataRegionId) { + getOrCreateHistoricalTransferSummaryCounter(pipeName, creationTime, dataRegionId) + .skippedHistoricalTsFileCount + .incrementAndGet(); + } + + public static void markHistoricalTsFileSplit( + final String pipeName, final long creationTime, final int dataRegionId) { + getOrCreateHistoricalTransferSummaryCounter(pipeName, creationTime, dataRegionId) + .splitHistoricalTsFileCount + .incrementAndGet(); + } + + public static void markHistoricalTsFileUnsplit( + final String pipeName, final long creationTime, final int dataRegionId) { + getOrCreateHistoricalTransferSummaryCounter(pipeName, creationTime, dataRegionId) + .unsplitHistoricalTsFileCount + .incrementAndGet(); + } + + public static HistoricalTransferSummary snapshotHistoricalTransferSummary( + final String pipeName, final long creationTime, final int dataRegionId) { + final HistoricalTransferSummaryCounter counter = + HISTORICAL_TRANSFER_SUMMARY_COUNTER_MAP.get( + new HistoricalTransferKey(pipeName, creationTime, dataRegionId)); + return Objects.nonNull(counter) ? counter.snapshot() : null; + } + + public static void clearHistoricalTransferSummary( + final String pipeName, final long creationTime, final int dataRegionId) { + HISTORICAL_TRANSFER_SUMMARY_COUNTER_MAP.remove( + new HistoricalTransferKey(pipeName, creationTime, dataRegionId)); + } + + private static HistoricalTransferSummary snapshotAndClearHistoricalTransferSummary( + final String pipeName, final long creationTime, final int dataRegionId) { + final HistoricalTransferSummaryCounter counter = + HISTORICAL_TRANSFER_SUMMARY_COUNTER_MAP.remove( + new HistoricalTransferKey(pipeName, creationTime, dataRegionId)); + return Objects.nonNull(counter) ? counter.snapshot() : null; + } + + private static HistoricalTransferSummaryCounter getOrCreateHistoricalTransferSummaryCounter( + final String pipeName, final long creationTime, final int dataRegionId) { + return HISTORICAL_TRANSFER_SUMMARY_COUNTER_MAP.computeIfAbsent( + new HistoricalTransferKey(pipeName, creationTime, dataRegionId), + ignored -> new HistoricalTransferSummaryCounter()); + } + + public static final class HistoricalTransferSummary { + + private final long extractedHistoricalTsFileCount; + private final long skippedHistoricalTsFileCount; + private final long splitHistoricalTsFileCount; + private final long unsplitHistoricalTsFileCount; + private final long extractedHistoricalDeletionCount; + + private HistoricalTransferSummary( + final long extractedHistoricalTsFileCount, + final long skippedHistoricalTsFileCount, + final long splitHistoricalTsFileCount, + final long unsplitHistoricalTsFileCount, + final long extractedHistoricalDeletionCount) { + this.extractedHistoricalTsFileCount = extractedHistoricalTsFileCount; + this.skippedHistoricalTsFileCount = skippedHistoricalTsFileCount; + this.splitHistoricalTsFileCount = splitHistoricalTsFileCount; + this.unsplitHistoricalTsFileCount = unsplitHistoricalTsFileCount; + this.extractedHistoricalDeletionCount = extractedHistoricalDeletionCount; + } + + public String toReportMessage() { + return String.format( + "historical summary: extractedTsFileCount=%s, skippedTsFileCount=%s, splitTsFileCount=%s, unsplitTsFileCount=%s, deletionCount=%s", + extractedHistoricalTsFileCount, + skippedHistoricalTsFileCount, + splitHistoricalTsFileCount, + unsplitHistoricalTsFileCount, + extractedHistoricalDeletionCount); + } + } + + private static final class HistoricalTransferSummaryCounter { + + private final AtomicLong extractedHistoricalTsFileCount = new AtomicLong(0); + private final AtomicLong skippedHistoricalTsFileCount = new AtomicLong(0); + private final AtomicLong splitHistoricalTsFileCount = new AtomicLong(0); + private final AtomicLong unsplitHistoricalTsFileCount = new AtomicLong(0); + private final AtomicLong extractedHistoricalDeletionCount = new AtomicLong(0); + + private void initialize( + final long extractedHistoricalTsFileCount, final long extractedHistoricalDeletionCount) { + this.extractedHistoricalTsFileCount.set(extractedHistoricalTsFileCount); + this.skippedHistoricalTsFileCount.set(0); + this.splitHistoricalTsFileCount.set(0); + this.unsplitHistoricalTsFileCount.set(0); + this.extractedHistoricalDeletionCount.set(extractedHistoricalDeletionCount); + } + + private HistoricalTransferSummary snapshot() { + return new HistoricalTransferSummary( + extractedHistoricalTsFileCount.get(), + skippedHistoricalTsFileCount.get(), + splitHistoricalTsFileCount.get(), + unsplitHistoricalTsFileCount.get(), + extractedHistoricalDeletionCount.get()); + } + } + + private static final class HistoricalTransferKey { + + private final String pipeName; + private final long creationTime; + private final int dataRegionId; + + private HistoricalTransferKey( + final String pipeName, final long creationTime, final int dataRegionId) { + this.pipeName = pipeName; + this.creationTime = creationTime; + this.dataRegionId = dataRegionId; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof HistoricalTransferKey)) { + return false; + } + final HistoricalTransferKey that = (HistoricalTransferKey) obj; + return creationTime == that.creationTime + && dataRegionId == that.dataRegionId + && Objects.equals(pipeName, that.pipeName); + } + + @Override + public int hashCode() { + return Objects.hash(pipeName, creationTime, dataRegionId); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java index f099301a63cdd..c9cd2d44af04b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.datastructure.pattern.PipePattern; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; +import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.commons.pipe.resource.ref.PipePhantomReferenceManager.PipeEventResource; import org.apache.iotdb.db.pipe.event.ReferenceTrackableEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; @@ -78,6 +79,10 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent protected final boolean isGeneratedByPipeConsensus; protected final boolean isGeneratedByHistoricalExtractor; + // Realtime TsFile events are created after TsFileProcessor#endFile(), so the file is already + // immutable even if TsFileResource status is still UNCLOSED. + private final boolean isTsFileSealed; + protected final AtomicBoolean isClosed; protected final AtomicReference dataContainer; @@ -91,7 +96,18 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent public PipeTsFileInsertionEvent(final TsFileResource resource, final boolean isLoaded) { // The modFile must be copied before the event is assigned to the listening pipes this( - resource, null, true, isLoaded, false, null, 0, null, null, Long.MIN_VALUE, Long.MAX_VALUE); + resource, + null, + true, + isLoaded, + false, + null, + 0, + null, + null, + Long.MIN_VALUE, + Long.MAX_VALUE, + true); } public PipeTsFileInsertionEvent( @@ -106,7 +122,36 @@ public PipeTsFileInsertionEvent( final PipePattern pattern, final long startTime, final long endTime) { - super(pipeName, creationTime, pipeTaskMeta, pattern, startTime, endTime); + this( + resource, + tsFile, + isWithMod, + isLoaded, + isGeneratedByHistoricalExtractor, + pipeName, + creationTime, + pipeTaskMeta, + pattern, + startTime, + endTime, + false); + } + + private PipeTsFileInsertionEvent( + final TsFileResource resource, + final File tsFile, + final boolean isWithMod, + final boolean isLoaded, + final boolean isGeneratedByHistoricalExtractor, + final String pipeName, + final long creationTime, + final PipeTaskMeta pipeTaskMeta, + final PipePattern pipePattern, + final long startTime, + final long endTime, + final boolean isTsFileSealed) { + super(pipeName, creationTime, pipeTaskMeta, pipePattern, startTime, endTime); + this.resource = resource; // For events created at assigner or historical extractor, the tsFile is get from the resource @@ -123,6 +168,7 @@ public PipeTsFileInsertionEvent( this.isGeneratedByPipe = resource.isGeneratedByPipe(); this.isGeneratedByPipeConsensus = resource.isGeneratedByPipeConsensus(); this.isGeneratedByHistoricalExtractor = isGeneratedByHistoricalExtractor; + this.isTsFileSealed = isTsFileSealed; this.tableNames = tableNames; this.dataContainer = new AtomicReference<>(null); @@ -180,6 +226,10 @@ public boolean waitForTsFileClose() throws InterruptedException { return true; } + if (isTsFileSealed) { + return !resource.isEmpty(); + } + if (!isClosed.get()) { isClosed.set(resource.isClosed()); @@ -215,6 +265,10 @@ public File getTsFile() { return tsFile; } + public String getDatabaseName() { + return Objects.isNull(resource) ? null : resource.getDatabaseName(); + } + public File getModFile() { return modFile; } @@ -359,7 +413,8 @@ public PipeTsFileInsertionEvent shallowCopySelfAndBindPipeTaskMetaForProgressRep pipeTaskMeta, pattern, startTime, - endTime); + endTime, + isTsFileSealed); } @Override @@ -445,8 +500,7 @@ public void consumeTabletInsertionEventsWithRetry( callerName, getTsFile(), tabletEventCount, - retryCount, - e); + retryCount); } else if (LOGGER.isDebugEnabled()) { LOGGER.debug( "{}: failed to allocate memory for parsing TsFile {}, tablet event no. {}, retry count is {}, will keep retrying.", @@ -492,7 +546,11 @@ public Iterable toTabletInsertionEvents(final long timeout "Interrupted when waiting for closing TsFile %s.", resource.getTsFilePath()) : String.format( "Parse TsFile %s error. Because: %s", resource.getTsFilePath(), e.getMessage()); - LOGGER.warn(errorMsg, e); + if (e instanceof PipeRuntimeOutOfMemoryCriticalException) { + PipeLogger.log(LOGGER::warn, errorMsg); + } else { + PipeLogger.log(LOGGER::warn, e, errorMsg); + } throw new PipeException(errorMsg); } } @@ -516,28 +574,29 @@ private void waitForResourceEnough4Parsing(final long timeoutMs) throws Interrup final double waitTimeSeconds = (currentTime - startTime) / 1000.0; if (elapsedRecordTimeSeconds > 10.0) { LOGGER.info( - "Wait for resource enough for parsing {} for {} seconds.", + "Wait for memory enough for parsing {} for {} seconds.", resource != null ? resource.getTsFilePath() : "tsfile", waitTimeSeconds); lastRecordTime = currentTime; } else if (LOGGER.isDebugEnabled()) { LOGGER.debug( - "Wait for resource enough for parsing {} for {} seconds.", + "Wait for memory enough for parsing {} for {} seconds.", resource != null ? resource.getTsFilePath() : "tsfile", waitTimeSeconds); } if (waitTimeSeconds * 1000 > timeoutMs) { // should contain 'TimeoutException' in exception message - throw new PipeException( - String.format("TimeoutException: Waited %s seconds", waitTimeSeconds)); + throw new PipeRuntimeOutOfMemoryCriticalException( + String.format( + "TimeoutException: Waited %s seconds for memory to parse TsFile", waitTimeSeconds)); } } final long currentTime = System.currentTimeMillis(); final double waitTimeSeconds = (currentTime - startTime) / 1000.0; LOGGER.info( - "Wait for resource enough for parsing {} for {} seconds.", + "Wait for memory enough for parsing {} for {} seconds.", resource != null ? resource.getTsFilePath() : "tsfile", waitTimeSeconds); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainer.java index 279996690ff92..1eb620e84bea5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/TsFileInsertionDataContainer.java @@ -39,6 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; public abstract class TsFileInsertionDataContainer implements AutoCloseable { @@ -69,13 +70,15 @@ public abstract class TsFileInsertionDataContainer implements AutoCloseable { protected Iterable tabletInsertionIterable; protected TsFileInsertionDataContainer( + final File tsFile, final String pipeName, final long creationTime, final PipePattern pattern, final long startTime, final long endTime, final PipeTaskMeta pipeTaskMeta, - final EnrichedEvent sourceEvent) { + final EnrichedEvent sourceEvent, + final boolean isWithMod) { this.pipeName = pipeName; this.creationTime = creationTime; @@ -93,6 +96,19 @@ protected TsFileInsertionDataContainer( PipeDataNodeResourceManager.memory() .forceAllocateForTabletWithRetry( IoTDBDescriptor.getInstance().getConfig().getPipeDataStructureTabletSizeInBytes()); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "TsFile {} has initialized {}, pipeName: {}, creation time: {}, pattern: {}, startTime: {}, endTime: {}, withMod: {}", + tsFile, + getClass().getSimpleName(), + pipeName, + creationTime, + pattern, + startTime, + endTime, + isWithMod); + } } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/query/TsFileInsertionQueryDataContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/query/TsFileInsertionQueryDataContainer.java index 15732fd0e8db2..e1fa58f5a0f68 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/query/TsFileInsertionQueryDataContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/query/TsFileInsertionQueryDataContainer.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils.TabletStringInternPool; import org.apache.iotdb.db.pipe.event.common.tsfile.container.TsFileInsertionDataContainer; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.util.ModsOperationUtil; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; @@ -70,6 +71,7 @@ public class TsFileInsertionQueryDataContainer extends TsFileInsertionDataContai private final Iterator>> deviceMeasurementsMapIterator; private final Map deviceIsAlignedMap; private final Map measurementDataTypeMap; + private final TabletStringInternPool tabletStringInternPool = new TabletStringInternPool(); @TestOnly public TsFileInsertionQueryDataContainer( @@ -114,7 +116,16 @@ public TsFileInsertionQueryDataContainer( final Map deviceIsAlignedMap, final boolean isWithMod) throws IOException { - super(pipeName, creationTime, pattern, startTime, endTime, pipeTaskMeta, sourceEvent); + super( + tsFile, + pipeName, + creationTime, + pattern, + startTime, + endTime, + pipeTaskMeta, + sourceEvent, + isWithMod); try { currentModifications = @@ -376,7 +387,8 @@ public boolean hasNext() { entry.getValue(), timeFilterExpression, allocatedMemoryBlockForTablet, - currentModifications); + currentModifications, + tabletStringInternPool); } catch (final Exception e) { close(); throw new PipeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/query/TsFileInsertionQueryDataTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/query/TsFileInsertionQueryDataTabletIterator.java index e16c7113da354..2e81f4aa335a0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/query/TsFileInsertionQueryDataTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/query/TsFileInsertionQueryDataTabletIterator.java @@ -20,6 +20,8 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.container.query; import org.apache.iotdb.commons.path.PatternTreeMap; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils.TabletStringInternPool; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.util.ModsOperationUtil; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; @@ -37,6 +39,7 @@ import org.apache.tsfile.read.expression.IExpression; import org.apache.tsfile.read.expression.QueryExpression; import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -57,6 +60,7 @@ public class TsFileInsertionQueryDataTabletIterator implements Iterator private final String deviceId; private final List measurements; + private final List schemas; private final IExpression timeFilterExpression; @@ -76,20 +80,28 @@ public class TsFileInsertionQueryDataTabletIterator implements Iterator final List measurements, final IExpression timeFilterExpression, final PipeMemoryBlock allocatedBlockForTablet, - final PatternTreeMap currentModifications) + final PatternTreeMap currentModifications, + final TabletStringInternPool tabletStringInternPool) throws IOException { this.tsFileReader = tsFileReader; this.measurementDataTypeMap = measurementDataTypeMap; - this.deviceId = deviceId; + this.deviceId = tabletStringInternPool.intern(deviceId); this.measurements = measurements.stream() .filter( measurement -> // time column in aligned time-series should not be a query column measurement != null && !measurement.isEmpty()) + .map(tabletStringInternPool::intern) .sorted() .collect(Collectors.toList()); + this.schemas = new ArrayList<>(); + for (final String measurement : this.measurements) { + final TSDataType dataType = + measurementDataTypeMap.get(this.deviceId + TsFileConstant.PATH_SEPARATOR + measurement); + schemas.add(new MeasurementSchema(measurement, dataType)); + } this.timeFilterExpression = timeFilterExpression; @@ -99,7 +111,7 @@ public class TsFileInsertionQueryDataTabletIterator implements Iterator this.measurementModsList = ModsOperationUtil.initializeMeasurementMods( - deviceId, this.measurements, currentModifications); + this.deviceId, this.measurements, currentModifications); } private QueryDataSet buildQueryDataSet() throws IOException { @@ -133,18 +145,9 @@ public Tablet next() { } private Tablet buildNextTablet() throws IOException { - final List schemas = new ArrayList<>(); - for (final String measurement : measurements) { - final TSDataType dataType = - measurementDataTypeMap.get(deviceId + TsFileConstant.PATH_SEPARATOR + measurement); - schemas.add(new MeasurementSchema(measurement, dataType)); - } - Tablet tablet = null; if (!queryDataSet.hasNext()) { - tablet = new Tablet(deviceId, schemas, 1); - tablet.initBitMaps(); - return tablet; + return new Tablet(deviceId, schemas, 1); } boolean isFirstRow = true; @@ -156,7 +159,6 @@ private Tablet buildNextTablet() throws IOException { Pair rowCountAndMemorySize = PipeMemoryWeightUtil.calculateTabletRowCountAndMemory(rowRecord); tablet = new Tablet(deviceId, schemas, rowCountAndMemorySize.getLeft()); - tablet.initBitMaps(); if (allocatedBlockForTablet.getMemoryUsageInBytes() < rowCountAndMemorySize.getRight()) { PipeDataNodeResourceManager.memory() .forceResize(allocatedBlockForTablet, rowCountAndMemorySize.getRight()); @@ -172,27 +174,30 @@ private Tablet buildNextTablet() throws IOException { final int fieldSize = fields.size(); for (int i = 0; i < fieldSize; i++) { final Field field = fields.get(i); - final String measurement = measurements.get(i); + final TSDataType dataType = schemas.get(i).getType(); // Check if this value is deleted by mods if (field == null || ModsOperationUtil.isDelete(rowRecord.getTimestamp(), measurementModsList.get(i))) { - tablet.bitMaps[i].mark(rowIndex); + if (dataType != null && dataType.isBinary()) { + PipeTabletUtils.putValue(tablet, rowIndex, i, dataType, Binary.EMPTY_VALUE); + } + PipeTabletUtils.markNullValue(tablet, rowIndex, i); } else { - tablet.addValue(measurement, rowIndex, field.getObjectValue(schemas.get(i).getType())); + PipeTabletUtils.putValue( + tablet, rowIndex, i, dataType, field.getObjectValue(schemas.get(i).getType())); isNeedFillTime = true; } } if (isNeedFillTime) { - tablet.addTimestamp(rowIndex, rowRecord.getTimestamp()); + PipeTabletUtils.putTimestamp(tablet, rowIndex, rowRecord.getTimestamp()); } - tablet.rowSize++; - if (tablet.rowSize == tablet.getMaxRowNumber()) { break; } } + PipeTabletUtils.compactBitMaps(tablet); return tablet; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/AlignedSinglePageWholeChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/AlignedSinglePageWholeChunkReader.java index 740a1523d279f..9c3b3514c830e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/AlignedSinglePageWholeChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/AlignedSinglePageWholeChunkReader.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.pipe.event.common.tsfile.container.scan; +import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.encoding.decoder.Decoder; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.header.ChunkHeader; @@ -29,18 +30,21 @@ import org.apache.tsfile.read.reader.chunk.AbstractChunkReader; import org.apache.tsfile.read.reader.chunk.ChunkReader; import org.apache.tsfile.read.reader.page.AlignedPageReader; +import org.apache.tsfile.read.reader.page.LazyLoadPageData; import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * The {@link AlignedSinglePageWholeChunkReader} is used to read a whole single page aligned chunk * with need to pass in the statistics. */ -public class AlignedSinglePageWholeChunkReader extends AbstractChunkReader { +public class AlignedSinglePageWholeChunkReader extends AbstractChunkReader + implements EstimatedMemoryChunkReader { // chunk header of the time column private final ChunkHeader timeChunkHeader; @@ -53,12 +57,15 @@ public class AlignedSinglePageWholeChunkReader extends AbstractChunkReader { private final List valueChunkDataBufferList = new ArrayList<>(); // deleted intervals of all the sub sensors private final List> valueDeleteIntervalsList = new ArrayList<>(); + private final long pageEstimatedMemoryUsageInBytes; public AlignedSinglePageWholeChunkReader(Chunk timeChunk, List valueChunkList) throws IOException { super(Long.MIN_VALUE, null); this.timeChunkHeader = timeChunk.getHeader(); this.timeChunkDataBuffer = timeChunk.getData(); + this.pageEstimatedMemoryUsageInBytes = + calculateMaxPageEstimatedMemoryUsageInBytesWithBatchData(timeChunk, valueChunkList); valueChunkList.forEach( chunk -> { @@ -124,7 +131,7 @@ private AlignedPageReader constructAlignedPageReader( ChunkReader.deserializePageData(timePageHeader, timeChunkDataBuffer, timeChunkHeader); List valuePageHeaderList = new ArrayList<>(); - List valuePageDataList = new ArrayList<>(); + LazyLoadPageData[] valuePageDataArray = new LazyLoadPageData[rawValuePageHeaderList.size()]; List valueDataTypeList = new ArrayList<>(); List valueDecoderList = new ArrayList<>(); @@ -135,15 +142,21 @@ private AlignedPageReader constructAlignedPageReader( if (valuePageHeader == null || valuePageHeader.getUncompressedSize() == 0) { // Empty Page valuePageHeaderList.add(null); - valuePageDataList.add(null); + valuePageDataArray[i] = null; valueDataTypeList.add(null); valueDecoderList.add(null); } else { ChunkHeader valueChunkHeader = valueChunkHeaderList.get(i); + int currentPagePosition = valueChunkDataBufferList.get(i).position(); + valueChunkDataBufferList + .get(i) + .position(currentPagePosition + valuePageHeader.getCompressedSize()); valuePageHeaderList.add(valuePageHeader); - valuePageDataList.add( - ChunkReader.deserializePageData( - valuePageHeader, valueChunkDataBufferList.get(i), valueChunkHeader)); + valuePageDataArray[i] = + new LazyLoadPageData( + valueChunkDataBufferList.get(i).array(), + currentPagePosition, + IUnCompressor.getUnCompressor(valueChunkHeader.getCompressionType())); valueDataTypeList.add(valueChunkHeader.getDataType()); valueDecoderList.add( Decoder.getDecoderByType( @@ -160,11 +173,90 @@ private AlignedPageReader constructAlignedPageReader( timePageData, defaultTimeDecoder, valuePageHeaderList, - valuePageDataList, + valuePageDataArray, valueDataTypeList, valueDecoderList, queryFilter); alignedPageReader.setDeleteIntervalList(valueDeleteIntervalsList); return alignedPageReader; } + + @Override + public long getCurrentPageEstimatedMemoryUsageInBytes() { + return pageEstimatedMemoryUsageInBytes; + } + + public static long calculatePageEstimatedMemoryUsageInBytes( + final Chunk timeChunk, final List valueChunkList) throws IOException { + final ByteBuffer timeChunkDataBuffer = timeChunk.getData().duplicate(); + long estimatedMemoryUsageInBytes = + PageHeader.deserializeFrom(timeChunkDataBuffer, (Statistics) null) + .getUncompressedSize(); + + for (final Chunk valueChunk : valueChunkList) { + if (valueChunk == null) { + continue; + } + + final ByteBuffer valueChunkDataBuffer = valueChunk.getData().duplicate(); + estimatedMemoryUsageInBytes += + PageHeader.deserializeFrom( + valueChunkDataBuffer, (Statistics) null) + .getUncompressedSize(); + } + + return estimatedMemoryUsageInBytes; + } + + public static long calculateMaxPageEstimatedMemoryUsageInBytesWithBatchData( + final Chunk timeChunk, final List valueChunkList) throws IOException { + final List pageEstimatedMemoryUsageInBytesList = + calculatePageEstimatedMemoryUsageInBytesWithBatchDataList(timeChunk, valueChunkList); + return pageEstimatedMemoryUsageInBytesList.isEmpty() + ? 0 + : pageEstimatedMemoryUsageInBytesList.get(0); + } + + public static List calculatePageEstimatedMemoryUsageInBytesWithBatchDataList( + final Chunk timeChunk, final List valueChunkList) throws IOException { + final ByteBuffer timeChunkDataBuffer = timeChunk.getData().duplicate(); + final List valueChunkDataBufferList = new ArrayList<>(valueChunkList.size()); + for (final Chunk valueChunk : valueChunkList) { + valueChunkDataBufferList.add( + Objects.isNull(valueChunk) ? null : valueChunk.getData().duplicate()); + } + + final List pageEstimatedMemoryUsageInBytesList = new ArrayList<>(); + while (timeChunkDataBuffer.remaining() > 0) { + long pageUncompressedSizeInBytes = 0; + final PageHeader timePageHeader = + SinglePageWholeChunkReader.deserializePageHeader( + timeChunkDataBuffer, timeChunk.getHeader()); + pageUncompressedSizeInBytes += timePageHeader.getUncompressedSize(); + SinglePageWholeChunkReader.skipCompressedPageData(timeChunkDataBuffer, timePageHeader); + + final List valueDataTypeList = new ArrayList<>(valueChunkList.size()); + for (int i = 0; i < valueChunkList.size(); ++i) { + final Chunk valueChunk = valueChunkList.get(i); + final ByteBuffer valueChunkDataBuffer = valueChunkDataBufferList.get(i); + if (Objects.isNull(valueChunk) || Objects.isNull(valueChunkDataBuffer)) { + valueDataTypeList.add(null); + continue; + } + + final PageHeader valuePageHeader = + SinglePageWholeChunkReader.deserializePageHeader( + valueChunkDataBuffer, valueChunk.getHeader()); + pageUncompressedSizeInBytes += valuePageHeader.getUncompressedSize(); + valueDataTypeList.add(valueChunk.getHeader().getDataType()); + SinglePageWholeChunkReader.skipCompressedPageData(valueChunkDataBuffer, valuePageHeader); + } + pageEstimatedMemoryUsageInBytesList.add( + SinglePageWholeChunkReader.estimatePageMemoryUsageInBytesWithBatchData( + pageUncompressedSizeInBytes, + SinglePageWholeChunkReader.getPageRowCount(timePageHeader, timeChunk), + valueDataTypeList)); + } + return SinglePageWholeChunkReader.toSuffixMaxList(pageEstimatedMemoryUsageInBytesList); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/EstimatedMemoryChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/EstimatedMemoryChunkReader.java new file mode 100644 index 0000000000000..dc1f2501bdf7b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/EstimatedMemoryChunkReader.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.event.common.tsfile.container.scan; + +interface EstimatedMemoryChunkReader { + + long getCurrentPageEstimatedMemoryUsageInBytes(); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/MemoryControlledChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/MemoryControlledChunkReader.java new file mode 100644 index 0000000000000..6a74ffc54fcec --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/MemoryControlledChunkReader.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.event.common.tsfile.container.scan; + +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.reader.IChunkReader; +import org.apache.tsfile.read.reader.IPageReader; + +import java.io.IOException; +import java.util.List; + +class MemoryControlledChunkReader implements IChunkReader, EstimatedMemoryChunkReader { + + private final IChunkReader delegate; + private final List pageEstimatedMemoryUsageInBytesList; + private int pageIndex; + + MemoryControlledChunkReader( + final IChunkReader delegate, final List pageEstimatedMemoryUsageInBytesList) { + this.delegate = delegate; + this.pageEstimatedMemoryUsageInBytesList = pageEstimatedMemoryUsageInBytesList; + } + + @Override + public long getCurrentPageEstimatedMemoryUsageInBytes() { + return pageIndex < pageEstimatedMemoryUsageInBytesList.size() + ? pageEstimatedMemoryUsageInBytesList.get(pageIndex) + : 0; + } + + @Override + public boolean hasNextSatisfiedPage() throws IOException { + return delegate.hasNextSatisfiedPage(); + } + + @Override + public BatchData nextPageData() throws IOException { + try { + return delegate.nextPageData(); + } finally { + ++pageIndex; + } + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + @Override + public List loadPageReaderList() throws IOException { + return delegate.loadPageReaderList(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/SinglePageWholeChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/SinglePageWholeChunkReader.java index 3f1aadbba0ccb..f41a6861120bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/SinglePageWholeChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/SinglePageWholeChunkReader.java @@ -21,27 +21,38 @@ import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.reader.chunk.AbstractChunkReader; +import org.apache.tsfile.read.reader.page.LazyLoadPageData; import org.apache.tsfile.read.reader.page.PageReader; +import org.apache.tsfile.utils.RamUsageEstimator; import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; -public class SinglePageWholeChunkReader extends AbstractChunkReader { +public class SinglePageWholeChunkReader extends AbstractChunkReader + implements EstimatedMemoryChunkReader { private final ChunkHeader chunkHeader; private final ByteBuffer chunkDataBuffer; + private final long pageEstimatedMemoryUsageInBytes; public SinglePageWholeChunkReader(Chunk chunk) throws IOException { super(Long.MIN_VALUE, null); this.chunkHeader = chunk.getHeader(); this.chunkDataBuffer = chunk.getData(); - + this.pageEstimatedMemoryUsageInBytes = + calculateMaxPageEstimatedMemoryUsageInBytesWithBatchData(chunk); initAllPageReaders(); } @@ -56,15 +67,212 @@ private void initAllPageReaders() throws IOException { } private PageReader constructPageReader(PageHeader pageHeader) throws IOException { + final int currentPagePosition = chunkDataBuffer.position(); + chunkDataBuffer.position(currentPagePosition + pageHeader.getCompressedSize()); return new PageReader( pageHeader, - deserializePageData(pageHeader, chunkDataBuffer, chunkHeader), + new LazyLoadPageData( + chunkDataBuffer.array(), + currentPagePosition, + IUnCompressor.getUnCompressor(chunkHeader.getCompressionType())), chunkHeader.getDataType(), Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()), defaultTimeDecoder, null); } + @Override + public long getCurrentPageEstimatedMemoryUsageInBytes() { + return pageEstimatedMemoryUsageInBytes; + } + + public static long calculatePageEstimatedMemoryUsageInBytes(final Chunk chunk) + throws IOException { + final ByteBuffer chunkDataBuffer = chunk.getData().duplicate(); + final PageHeader pageHeader = deserializePageHeader(chunkDataBuffer, chunk.getHeader()); + return pageHeader.getUncompressedSize(); + } + + public static long calculateMaxPageEstimatedMemoryUsageInBytes(final Chunk chunk) + throws IOException { + final ByteBuffer chunkDataBuffer = chunk.getData().duplicate(); + long maxPageEstimatedMemoryUsageInBytes = 0; + while (chunkDataBuffer.remaining() > 0) { + final PageHeader pageHeader = deserializePageHeader(chunkDataBuffer, chunk.getHeader()); + maxPageEstimatedMemoryUsageInBytes = + Math.max(maxPageEstimatedMemoryUsageInBytes, pageHeader.getUncompressedSize()); + skipCompressedPageData(chunkDataBuffer, pageHeader); + } + return maxPageEstimatedMemoryUsageInBytes; + } + + public static long calculateMaxPageEstimatedMemoryUsageInBytesWithBatchData(final Chunk chunk) + throws IOException { + final List pageEstimatedMemoryUsageInBytesList = + calculatePageEstimatedMemoryUsageInBytesWithBatchDataList(chunk); + return pageEstimatedMemoryUsageInBytesList.isEmpty() + ? 0 + : pageEstimatedMemoryUsageInBytesList.get(0); + } + + public static List calculatePageEstimatedMemoryUsageInBytesWithBatchDataList( + final Chunk chunk) throws IOException { + final ByteBuffer chunkDataBuffer = chunk.getData().duplicate(); + final List pageEstimatedMemoryUsageInBytesList = new ArrayList<>(); + while (chunkDataBuffer.remaining() > 0) { + final PageHeader pageHeader = deserializePageHeader(chunkDataBuffer, chunk.getHeader()); + pageEstimatedMemoryUsageInBytesList.add( + estimatePageMemoryUsageInBytesWithBatchData( + pageHeader, chunk, Collections.singletonList(chunk.getHeader().getDataType()))); + skipCompressedPageData(chunkDataBuffer, pageHeader); + } + return toSuffixMaxList(pageEstimatedMemoryUsageInBytesList); + } + + static PageHeader deserializePageHeader( + final ByteBuffer chunkDataBuffer, final ChunkHeader chunkHeader) throws IOException { + return isSinglePageChunk(chunkHeader) + ? PageHeader.deserializeFrom(chunkDataBuffer, (Statistics) null) + : PageHeader.deserializeFrom(chunkDataBuffer, chunkHeader.getDataType()); + } + + static boolean isSinglePageChunk(final ChunkHeader chunkHeader) { + return (chunkHeader.getChunkType() & 0x3F) == MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER; + } + + static void skipCompressedPageData( + final ByteBuffer chunkDataBuffer, final PageHeader pageHeader) { + chunkDataBuffer.position(chunkDataBuffer.position() + pageHeader.getCompressedSize()); + } + + static List toSuffixMaxList(final List pageEstimatedMemoryUsageInBytesList) { + long suffixMaxPageEstimatedMemoryUsageInBytes = 0; + for (int i = pageEstimatedMemoryUsageInBytesList.size() - 1; i >= 0; --i) { + suffixMaxPageEstimatedMemoryUsageInBytes = + Math.max( + suffixMaxPageEstimatedMemoryUsageInBytes, pageEstimatedMemoryUsageInBytesList.get(i)); + pageEstimatedMemoryUsageInBytesList.set(i, suffixMaxPageEstimatedMemoryUsageInBytes); + } + return pageEstimatedMemoryUsageInBytesList; + } + + static long estimatePageMemoryUsageInBytesWithBatchData( + final PageHeader timePageHeader, + final Chunk timeChunk, + final List valueDataTypeList) { + return estimatePageMemoryUsageInBytesWithBatchData( + timePageHeader.getUncompressedSize(), + getPageRowCount(timePageHeader, timeChunk), + valueDataTypeList); + } + + static int getPageRowCount(final PageHeader pageHeader, final Chunk chunk) { + if (isSinglePageChunk(chunk.getHeader())) { + return Objects.isNull(chunk.getChunkStatistic()) + ? 0 + : saturateToInt(chunk.getChunkStatistic().getCount()); + } + return saturateToInt(pageHeader.getNumOfValues()); + } + + private static int saturateToInt(final long value) { + return (int) Math.min(Integer.MAX_VALUE, value); + } + + static long estimatePageMemoryUsageInBytesWithBatchData( + final long pageUncompressedSizeInBytes, + final int rowCount, + final List valueDataTypeList) { + return pageUncompressedSizeInBytes + + estimateBatchDataMemoryUsageInBytes(rowCount, valueDataTypeList); + } + + private static long estimateBatchDataMemoryUsageInBytes( + final int rowCount, final List valueDataTypeList) { + final int valueCount = valueDataTypeList.size(); + final long segmentCount = Math.max(1, (rowCount + 15L) / 16); + long estimatedMemoryUsageInBytes = RamUsageEstimator.sizeOfLongArray(16) * segmentCount; + + if (valueCount == 1) { + estimatedMemoryUsageInBytes += + estimateSingleValueArrayMemoryUsageInBytes(rowCount, valueDataTypeList.get(0)); + } else if (valueCount > 1) { + estimatedMemoryUsageInBytes += RamUsageEstimator.sizeOfObjectArray(16) * segmentCount; + estimatedMemoryUsageInBytes += + (long) rowCount + * (RamUsageEstimator.sizeOfObjectArray(valueCount) + + estimateVectorValueMemoryUsageInBytes(valueDataTypeList)); + } + + return estimatedMemoryUsageInBytes; + } + + private static long estimateSingleValueArrayMemoryUsageInBytes( + final int rowCount, final TSDataType dataType) { + final long segmentCount = Math.max(1, (rowCount + 15L) / 16); + if (Objects.isNull(dataType)) { + return 0; + } + + switch (dataType) { + case BOOLEAN: + return RamUsageEstimator.sizeOfBooleanArray(16) * segmentCount; + case INT32: + case DATE: + return RamUsageEstimator.sizeOfIntArray(16) * segmentCount; + case INT64: + case TIMESTAMP: + return RamUsageEstimator.sizeOfLongArray(16) * segmentCount; + case FLOAT: + return RamUsageEstimator.sizeOfFloatArray(16) * segmentCount; + case DOUBLE: + return RamUsageEstimator.sizeOfDoubleArray(16) * segmentCount; + case TEXT: + case BLOB: + case STRING: + return RamUsageEstimator.sizeOfObjectArray(16) * segmentCount; + default: + return 0; + } + } + + private static long estimateVectorValueMemoryUsageInBytes( + final List valueDataTypeList) { + long estimatedMemoryUsageInBytes = 0; + for (final TSDataType dataType : valueDataTypeList) { + if (Objects.isNull(dataType)) { + continue; + } + + estimatedMemoryUsageInBytes += + RamUsageEstimator.alignObjectSize( + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + + estimateTsPrimitiveTypeValueMemoryUsageInBytes(dataType)); + } + return estimatedMemoryUsageInBytes; + } + + private static long estimateTsPrimitiveTypeValueMemoryUsageInBytes(final TSDataType dataType) { + switch (dataType) { + case BOOLEAN: + return 1; + case INT32: + case DATE: + case FLOAT: + return Integer.BYTES; + case INT64: + case TIMESTAMP: + case DOUBLE: + return Long.BYTES; + case TEXT: + case BLOB: + case STRING: + return RamUsageEstimator.NUM_BYTES_OBJECT_REF; + default: + return 0; + } + } + ///////////////////////////////////////////////////////////////////////////////////////////////// // util methods ///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java index 271bc317d0f96..ac1fdb94db77e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java @@ -26,6 +26,8 @@ import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils.TabletStringInternPool; import org.apache.iotdb.db.pipe.event.common.tsfile.container.TsFileInsertionDataContainer; import org.apache.iotdb.db.pipe.event.common.tsfile.parser.util.ModsOperationUtil; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; @@ -93,15 +95,17 @@ public class TsFileInsertionScanDataContainer extends TsFileInsertionDataContain private String currentDevice; private boolean currentIsAligned; private final List currentMeasurements = new ArrayList<>(); + private final TabletStringInternPool tabletStringInternPool = new TabletStringInternPool(); private final List modsInfos = new ArrayList<>(); // Cached time chunk private final List timeChunkList = new ArrayList<>(); private final List isMultiPageList = new ArrayList<>(); + private final List timeChunkPageMemorySizeList = new ArrayList<>(); private final Map measurementIndexMap = new HashMap<>(); private int lastIndex = -1; - private ChunkHeader firstChunkHeader4NextSequentialValueChunks; + private Chunk firstChunk4NextSequentialValueChunks; private byte lastMarker = Byte.MIN_VALUE; @@ -116,7 +120,16 @@ public TsFileInsertionScanDataContainer( final EnrichedEvent sourceEvent, final boolean isWithMod) throws IOException { - super(pipeName, creationTime, pattern, startTime, endTime, pipeTaskMeta, sourceEvent); + super( + tsFile, + pipeName, + creationTime, + pattern, + startTime, + endTime, + pipeTaskMeta, + sourceEvent, + isWithMod); this.startTime = startTime; this.endTime = endTime; @@ -262,7 +275,6 @@ private Tablet getNextTablet() { if (!data.hasCurrent()) { tablet = new Tablet(currentDevice, currentMeasurements, 1); - tablet.initBitMaps(); // Ignore the memory cost of tablet PipeDataNodeResourceManager.memory().forceResize(allocatedMemoryBlockForTablet, 0); return tablet; @@ -278,7 +290,6 @@ private Tablet getNextTablet() { PipeMemoryWeightUtil.calculateTabletRowCountAndMemory(data); tablet = new Tablet(currentDevice, currentMeasurements, rowCountAndMemorySize.getLeft()); - tablet.initBitMaps(); if (allocatedMemoryBlockForTablet.getMemoryUsageInBytes() < rowCountAndMemorySize.getRight()) { PipeDataNodeResourceManager.memory() @@ -290,15 +301,13 @@ private Tablet getNextTablet() { final int rowIndex = tablet.rowSize; if (putValueToColumns(data, tablet, rowIndex)) { - tablet.addTimestamp(rowIndex, data.currentTime()); + PipeTabletUtils.putTimestamp(tablet, rowIndex, data.currentTime()); } - - tablet.rowSize++; } data.next(); while (!data.hasCurrent() && chunkReader.hasNextSatisfiedPage()) { - data = chunkReader.nextPageData(); + data = nextPageData(); } if (tablet != null && tablet.rowSize == tablet.getMaxRowNumber()) { @@ -308,13 +317,13 @@ private Tablet getNextTablet() { if (tablet == null) { tablet = new Tablet(currentDevice, currentMeasurements, 1); - tablet.initBitMaps(); } // Switch chunk reader iff current chunk is all consumed if (!data.hasCurrent()) { prepareData(); } + PipeTabletUtils.compactBitMaps(tablet); return tablet; } catch (final Exception e) { close(); @@ -334,91 +343,130 @@ private void prepareData() throws IOException { } do { - data = chunkReader.nextPageData(); - long size = PipeMemoryWeightUtil.calculateBatchDataRamBytesUsed(data); - if (allocatedMemoryBlockForBatchData.getMemoryUsageInBytes() < size) { - PipeDataNodeResourceManager.memory().forceResize(allocatedMemoryBlockForBatchData, size); - } + data = nextPageData(); } while (!data.hasCurrent() && chunkReader.hasNextSatisfiedPage()); } while (!data.hasCurrent()); } + private BatchData nextPageData() throws IOException { + resizePageDataMemoryForCurrentPageIfNeeded(); + final BatchData nextData = chunkReader.nextPageData(); + resizePageDataMemoryIfNeeded(PipeMemoryWeightUtil.calculateBatchDataRamBytesUsed(nextData)); + return nextData; + } + + private void resizePageDataMemoryForCurrentPageIfNeeded() { + if (!(chunkReader instanceof EstimatedMemoryChunkReader)) { + return; + } + + final long estimatedMemoryUsageInBytes = + ((EstimatedMemoryChunkReader) chunkReader).getCurrentPageEstimatedMemoryUsageInBytes(); + resizePageDataMemoryIfNeeded(estimatedMemoryUsageInBytes); + } + + private void resizePageDataMemoryIfNeeded(final long estimatedMemoryUsageInBytes) { + if (allocatedMemoryBlockForBatchData.getMemoryUsageInBytes() < estimatedMemoryUsageInBytes) { + PipeDataNodeResourceManager.memory() + .forceResize(allocatedMemoryBlockForBatchData, estimatedMemoryUsageInBytes); + } + } + private boolean putValueToColumns(final BatchData data, final Tablet tablet, final int rowIndex) { - final Object[] columns = tablet.values; boolean isNeedFillTime = false; if (data.getDataType() == TSDataType.VECTOR) { - for (int i = 0; i < columns.length; ++i) { + for (int i = 0; i < tablet.getSchemas().size(); ++i) { final TsPrimitiveType primitiveType = data.getVector()[i]; + final TSDataType type = tablet.getSchemas().get(i).getType(); if (Objects.isNull(primitiveType) || ModsOperationUtil.isDelete(data.currentTime(), modsInfos.get(i))) { - tablet.bitMaps[i].mark(rowIndex); - final TSDataType type = tablet.getSchemas().get(i).getType(); if (type == TSDataType.TEXT || type == TSDataType.BLOB || type == TSDataType.STRING) { - ((Binary[]) columns[i])[rowIndex] = Binary.EMPTY_VALUE; - } - if (type == TSDataType.DATE) { - ((LocalDate[]) columns[i])[rowIndex] = EMPTY_DATE; + PipeTabletUtils.putValue(tablet, rowIndex, i, type, Binary.EMPTY_VALUE); } + PipeTabletUtils.markNullValue(tablet, rowIndex, i); continue; } isNeedFillTime = true; - switch (tablet.getSchemas().get(i).getType()) { + switch (type) { case BOOLEAN: - ((boolean[]) columns[i])[rowIndex] = primitiveType.getBoolean(); + PipeTabletUtils.putValue(tablet, rowIndex, i, type, primitiveType.getBoolean()); break; case INT32: - ((int[]) columns[i])[rowIndex] = primitiveType.getInt(); + PipeTabletUtils.putValue(tablet, rowIndex, i, type, primitiveType.getInt()); break; case DATE: - ((LocalDate[]) columns[i])[rowIndex] = - DateUtils.parseIntToLocalDate(primitiveType.getInt()); + PipeTabletUtils.putValue( + tablet, rowIndex, i, type, DateUtils.parseIntToLocalDate(primitiveType.getInt())); break; case INT64: case TIMESTAMP: - ((long[]) columns[i])[rowIndex] = primitiveType.getLong(); + PipeTabletUtils.putValue(tablet, rowIndex, i, type, primitiveType.getLong()); break; case FLOAT: - ((float[]) columns[i])[rowIndex] = primitiveType.getFloat(); + PipeTabletUtils.putValue(tablet, rowIndex, i, type, primitiveType.getFloat()); break; case DOUBLE: - ((double[]) columns[i])[rowIndex] = primitiveType.getDouble(); + PipeTabletUtils.putValue(tablet, rowIndex, i, type, primitiveType.getDouble()); break; case TEXT: case BLOB: case STRING: - ((Binary[]) columns[i])[rowIndex] = primitiveType.getBinary(); + final Binary binary = primitiveType.getBinary(); + PipeTabletUtils.putValue( + tablet, + rowIndex, + i, + type, + Objects.isNull(binary) || Objects.isNull(binary.getValues()) + ? Binary.EMPTY_VALUE + : binary); break; default: throw new UnSupportedDataTypeException("UnSupported" + primitiveType.getDataType()); } } } else { + if (!modsInfos.isEmpty() + && ModsOperationUtil.isDelete(data.currentTime(), modsInfos.get(0))) { + return false; + } + isNeedFillTime = true; - switch (tablet.getSchemas().get(0).getType()) { + final TSDataType type = tablet.getSchemas().get(0).getType(); + switch (type) { case BOOLEAN: - ((boolean[]) columns[0])[rowIndex] = data.getBoolean(); + PipeTabletUtils.putValue(tablet, rowIndex, 0, type, data.getBoolean()); break; case INT32: - ((int[]) columns[0])[rowIndex] = data.getInt(); + PipeTabletUtils.putValue(tablet, rowIndex, 0, type, data.getInt()); break; case DATE: - ((LocalDate[]) columns[0])[rowIndex] = DateUtils.parseIntToLocalDate(data.getInt()); + PipeTabletUtils.putValue( + tablet, rowIndex, 0, type, DateUtils.parseIntToLocalDate(data.getInt())); break; case INT64: case TIMESTAMP: - ((long[]) columns[0])[rowIndex] = data.getLong(); + PipeTabletUtils.putValue(tablet, rowIndex, 0, type, data.getLong()); break; case FLOAT: - ((float[]) columns[0])[rowIndex] = data.getFloat(); + PipeTabletUtils.putValue(tablet, rowIndex, 0, type, data.getFloat()); break; case DOUBLE: - ((double[]) columns[0])[rowIndex] = data.getDouble(); + PipeTabletUtils.putValue(tablet, rowIndex, 0, type, data.getDouble()); break; case TEXT: case BLOB: case STRING: - ((Binary[]) columns[0])[rowIndex] = data.getBinary(); + final Binary binary = data.getBinary(); + PipeTabletUtils.putValue( + tablet, + rowIndex, + 0, + type, + Objects.isNull(binary) || Objects.isNull(binary.getValues()) + ? Binary.EMPTY_VALUE + : binary); break; default: throw new UnSupportedDataTypeException("UnSupported" + data.getDataType()); @@ -430,6 +478,7 @@ private boolean putValueToColumns(final BatchData data, final Tablet tablet, fin private void moveToNextChunkReader() throws IOException, IllegalStateException { ChunkHeader chunkHeader; long valueChunkSize = 0; + long valueChunkPageMemorySize = 0; final List valueChunkList = new ArrayList<>(); currentMeasurements.clear(); modsInfos.clear(); @@ -440,7 +489,12 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { } byte marker; - while ((marker = lastMarker != Byte.MIN_VALUE ? lastMarker : tsFileSequenceReader.readMarker()) + while ((marker = + lastMarker != Byte.MIN_VALUE + ? lastMarker + : Objects.nonNull(firstChunk4NextSequentialValueChunks) + ? toValueChunkMarker(firstChunk4NextSequentialValueChunks.getHeader()) + : tsFileSequenceReader.readMarker()) != MetaMarker.SEPARATOR) { lastMarker = Byte.MIN_VALUE; switch (marker) { @@ -464,10 +518,15 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { if ((chunkHeader.getChunkType() & TsFileConstant.TIME_COLUMN_MASK) == TsFileConstant.TIME_COLUMN_MASK) { - timeChunkList.add( + final Chunk timeChunk = new Chunk( - chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize()))); - isMultiPageList.add(marker == MetaMarker.TIME_CHUNK_HEADER); + chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize())); + final boolean isMultiPage = marker == MetaMarker.TIME_CHUNK_HEADER; + timeChunkList.add(timeChunk); + isMultiPageList.add(isMultiPage); + timeChunkPageMemorySizeList.add( + SinglePageWholeChunkReader.calculateMaxPageEstimatedMemoryUsageInBytes( + timeChunk)); break; } @@ -513,26 +572,32 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { Chunk chunk = new Chunk( chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize())); + final List pageEstimatedMemoryUsageInBytesList = + SinglePageWholeChunkReader + .calculatePageEstimatedMemoryUsageInBytesWithBatchDataList(chunk); chunkReader = currentIsMultiPage - ? new ChunkReader(chunk, filter) + ? new MemoryControlledChunkReader( + new ChunkReader(chunk, filter), pageEstimatedMemoryUsageInBytesList) : new SinglePageWholeChunkReader(chunk); currentIsAligned = false; + final String measurementID = + tabletStringInternPool.intern(chunkHeader.getMeasurementID()); currentMeasurements.add( - new MeasurementSchema(chunkHeader.getMeasurementID(), chunkHeader.getDataType())); + new MeasurementSchema(measurementID, chunkHeader.getDataType())); modsInfos.addAll( ModsOperationUtil.initializeMeasurementMods( - currentDevice, - Collections.singletonList(chunkHeader.getMeasurementID()), - currentModifications)); + currentDevice, Collections.singletonList(measurementID), currentModifications)); return; } case MetaMarker.VALUE_CHUNK_HEADER: case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: { - if (Objects.isNull(firstChunkHeader4NextSequentialValueChunks)) { - long currentChunkHeaderOffset = tsFileSequenceReader.position() - 1; + Chunk chunk; + long currentValueChunkPageMemorySize = 0; + if (Objects.isNull(firstChunk4NextSequentialValueChunks)) { + final long currentChunkHeaderOffset = tsFileSequenceReader.position() - 1; chunkHeader = tsFileSequenceReader.readChunkHeader(marker); final long nextMarkerOffset = @@ -572,9 +637,11 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { } // Increase value index + final String measurementID = + tabletStringInternPool.intern(chunkHeader.getMeasurementID()); final int valueIndex = measurementIndexMap.compute( - chunkHeader.getMeasurementID(), + measurementID, (measurement, index) -> Objects.nonNull(index) ? index + 1 : 0); // Emit when encountered non-sequential value chunk, or the chunk size exceeds @@ -583,51 +650,62 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { if (chunkHeader.getDataSize() == 0) { break; } + chunk = + new Chunk( + chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize())); + currentValueChunkPageMemorySize = calculateMaxPageMemorySize(chunk); boolean needReturn = false; final long timeChunkSize = lastIndex >= 0 ? PipeMemoryWeightUtil.calculateChunkRamBytesUsed( timeChunkList.get(lastIndex)) : 0; + final long timeChunkPageMemorySize = + lastIndex >= 0 ? timeChunkPageMemorySizeList.get(lastIndex) : 0; if (lastIndex >= 0) { if (valueIndex != lastIndex) { needReturn = recordAlignedChunk(valueChunkList, marker); } else { final long chunkSize = timeChunkSize + valueChunkSize; + final long pageMemorySize = timeChunkPageMemorySize + valueChunkPageMemorySize; if (chunkSize + chunkHeader.getDataSize() - > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) { - if (valueChunkList.size() == 1 - && chunkSize > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) { - PipeDataNodeResourceManager.memory() - .forceResize(allocatedMemoryBlockForChunk, chunkSize); - } + > allocatedMemoryBlockForChunk.getMemoryUsageInBytes() + || timeChunkPageMemorySize > 0 + && currentValueChunkPageMemorySize > 0 + && pageMemorySize + currentValueChunkPageMemorySize + > getPageDataMemoryLimitInBytes()) { needReturn = recordAlignedChunk(valueChunkList, marker); } } } lastIndex = valueIndex; if (needReturn) { - firstChunkHeader4NextSequentialValueChunks = chunkHeader; + firstChunk4NextSequentialValueChunks = chunk; return; } + resizeChunkMemoryBlockIfFirstValueChunkExceedsLimit(valueChunkList, chunkHeader); + resizePageDataMemoryBlockIfFirstValueChunkExceedsLimit( + valueChunkList, currentValueChunkPageMemorySize); } else { - chunkHeader = firstChunkHeader4NextSequentialValueChunks; - firstChunkHeader4NextSequentialValueChunks = null; + chunk = firstChunk4NextSequentialValueChunks; + chunkHeader = chunk.getHeader(); + firstChunk4NextSequentialValueChunks = null; + currentValueChunkPageMemorySize = calculateMaxPageMemorySize(chunk); + resizeChunkMemoryBlockIfFirstValueChunkExceedsLimit(valueChunkList, chunkHeader); + resizePageDataMemoryBlockIfFirstValueChunkExceedsLimit( + valueChunkList, currentValueChunkPageMemorySize); } - Chunk chunk = - new Chunk( - chunkHeader, tsFileSequenceReader.readChunk(-1, chunkHeader.getDataSize())); - valueChunkSize += chunkHeader.getDataSize(); + valueChunkPageMemorySize += currentValueChunkPageMemorySize; valueChunkList.add(chunk); + final String measurementID = + tabletStringInternPool.intern(chunkHeader.getMeasurementID()); currentMeasurements.add( - new MeasurementSchema(chunkHeader.getMeasurementID(), chunkHeader.getDataType())); + new MeasurementSchema(measurementID, chunkHeader.getDataType())); modsInfos.addAll( ModsOperationUtil.initializeMeasurementMods( - currentDevice, - Collections.singletonList(chunkHeader.getMeasurementID()), - currentModifications)); + currentDevice, Collections.singletonList(measurementID), currentModifications)); break; } case MetaMarker.CHUNK_GROUP_HEADER: @@ -643,9 +721,13 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { lastIndex = -1; timeChunkList.clear(); isMultiPageList.clear(); + timeChunkPageMemorySizeList.clear(); measurementIndexMap.clear(); - currentDevice = pattern.mayOverlapWithDevice(deviceID) ? deviceID : null; + currentDevice = + pattern.mayOverlapWithDevice(deviceID) + ? tabletStringInternPool.intern(deviceID) + : null; break; } case MetaMarker.OPERATION_INDEX_RANGE: @@ -664,15 +746,37 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { } } + private long getPageDataMemoryLimitInBytes() { + return PipeConfig.getInstance().getPipeMaxReaderChunkSize(); + } + private boolean recordAlignedChunk(final List valueChunkList, final byte marker) throws IOException { if (!valueChunkList.isEmpty()) { final Chunk timeChunk = timeChunkList.get(lastIndex); timeChunk.getData().rewind(); currentIsMultiPage = isMultiPageList.get(lastIndex); + if (!currentIsMultiPage) { + resizePageDataMemoryIfNeeded( + AlignedSinglePageWholeChunkReader.calculatePageEstimatedMemoryUsageInBytes( + timeChunk, valueChunkList)); + } + final List pageEstimatedMemoryUsageInBytesList = + currentIsMultiPage + ? AlignedSinglePageWholeChunkReader + .calculatePageEstimatedMemoryUsageInBytesWithBatchDataList( + timeChunk, valueChunkList) + : Collections.emptyList(); + final long maxPageEstimatedMemoryUsageInBytes = + pageEstimatedMemoryUsageInBytesList.isEmpty() + ? 0 + : pageEstimatedMemoryUsageInBytesList.get(0); + resizePageDataMemoryIfNeeded(maxPageEstimatedMemoryUsageInBytes); chunkReader = currentIsMultiPage - ? new AlignedChunkReader(timeChunk, valueChunkList, filter) + ? new MemoryControlledChunkReader( + new AlignedChunkReader(timeChunk, valueChunkList, filter), + pageEstimatedMemoryUsageInBytesList) : new AlignedSinglePageWholeChunkReader(timeChunk, valueChunkList); currentIsAligned = true; lastMarker = marker; @@ -681,6 +785,52 @@ private boolean recordAlignedChunk(final List valueChunkList, final byte return false; } + private void resizeChunkMemoryBlockIfFirstValueChunkExceedsLimit( + final List valueChunkList, final ChunkHeader valueChunkHeader) { + if (!valueChunkList.isEmpty() || lastIndex < 0) { + return; + } + + final long chunkSize = + PipeMemoryWeightUtil.calculateChunkRamBytesUsed(timeChunkList.get(lastIndex)) + + valueChunkHeader.getDataSize(); + if (chunkSize > allocatedMemoryBlockForChunk.getMemoryUsageInBytes()) { + PipeDataNodeResourceManager.memory().forceResize(allocatedMemoryBlockForChunk, chunkSize); + } + } + + private void resizePageDataMemoryBlockIfFirstValueChunkExceedsLimit( + final List valueChunkList, final long valueChunkPageMemorySize) { + if (!valueChunkList.isEmpty() || lastIndex < 0 || valueChunkPageMemorySize <= 0) { + return; + } + + final long timeChunkPageMemorySize = timeChunkPageMemorySizeList.get(lastIndex); + if (timeChunkPageMemorySize <= 0) { + return; + } + + final long pageMemorySize = timeChunkPageMemorySize + valueChunkPageMemorySize; + if (pageMemorySize > getPageDataMemoryLimitInBytes()) { + PipeDataNodeResourceManager.memory() + .forceResize(allocatedMemoryBlockForBatchData, pageMemorySize); + } + } + + private long calculateMaxPageMemorySize(final Chunk chunk) throws IOException { + return SinglePageWholeChunkReader.calculateMaxPageEstimatedMemoryUsageInBytes(chunk); + } + + private boolean isSinglePageValueChunk(final ChunkHeader chunkHeader) { + return (chunkHeader.getChunkType() & 0x3F) == MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER; + } + + private byte toValueChunkMarker(final ChunkHeader chunkHeader) { + return isSinglePageValueChunk(chunkHeader) + ? MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER + : MetaMarker.VALUE_CHUNK_HEADER; + } + @Override public void close() { super.close(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEvent.java index c8474e360da78..48ab8259fe4b0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/realtime/PipeRealtimeEvent.java @@ -140,6 +140,16 @@ public ProgressIndex getProgressIndex() { return event.getProgressIndex(); } + @Override + public void skipReportOnCommit() { + event.skipReportOnCommit(); + } + + @Override + public boolean isShouldReportOnCommit() { + return event.isShouldReportOnCommit(); + } + @Override public void skipParsingPattern() { event.skipParsingPattern(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java index 38cfb579622a5..6ff3de4d9495c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/overview/PipeDataNodeRemainingEventAndTimeOperator.java @@ -138,14 +138,11 @@ public double getRemainingTime() { + rawTabletEventCount.get() + insertNodeEventCount.get(); - dataRegionCommitMeter.updateAndGet( - meter -> { - if (Objects.nonNull(meter)) { - lastDataRegionCommitSmoothingValue = - pipeRemainingTimeCommitRateAverageTime.getMeterRate(meter); - } - return meter; - }); + final Meter dataRegionMeter = dataRegionCommitMeter.get(); + if (Objects.nonNull(dataRegionMeter)) { + lastDataRegionCommitSmoothingValue = + pipeRemainingTimeCommitRateAverageTime.getMeterRate(dataRegionMeter); + } final double dataRegionRemainingTime; if (totalDataRegionWriteEventCount <= 0) { dataRegionRemainingTime = 0; @@ -162,14 +159,11 @@ public double getRemainingTime() { .reduce(Long::sum) .orElse(0L); - schemaRegionCommitMeter.updateAndGet( - meter -> { - if (Objects.nonNull(meter)) { - lastSchemaRegionCommitSmoothingValue = - pipeRemainingTimeCommitRateAverageTime.getMeterRate(meter); - } - return meter; - }); + final Meter schemaRegionMeter = schemaRegionCommitMeter.get(); + if (Objects.nonNull(schemaRegionMeter)) { + lastSchemaRegionCommitSmoothingValue = + pipeRemainingTimeCommitRateAverageTime.getMeterRate(schemaRegionMeter); + } final double schemaRegionRemainingTime; if (totalSchemaRegionWriteEventCount <= 0) { schemaRegionRemainingTime = 0; @@ -199,23 +193,17 @@ void register(final IoTDBSchemaRegionSource source) { //////////////////////////// Rate //////////////////////////// void markDataRegionCommit() { - dataRegionCommitMeter.updateAndGet( - meter -> { - if (Objects.nonNull(meter)) { - meter.mark(); - } - return meter; - }); + final Meter meter = dataRegionCommitMeter.get(); + if (Objects.nonNull(meter)) { + meter.mark(); + } } void markSchemaRegionCommit() { - schemaRegionCommitMeter.updateAndGet( - meter -> { - if (Objects.nonNull(meter)) { - meter.mark(); - } - return meter; - }); + final Meter meter = schemaRegionCommitMeter.get(); + if (Objects.nonNull(meter)) { + meter.mark(); + } } void markTsFileCollectInvocationCount(final long collectInvocationCount) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java index 16a20a5f509aa..340ac085a010c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java @@ -74,6 +74,7 @@ public class TwoStageCountProcessor implements PipeProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(TwoStageCountProcessor.class); + private static final String LEGACY_PROCESSOR_OUTPUT_SERIES_KEY = "processor.output.series"; private String pipeName; private long creationTime; @@ -98,10 +99,17 @@ public class TwoStageCountProcessor implements PipeProcessor { @Override public void validate(PipeParameterValidator validator) throws Exception { - validator.validateRequiredAttribute(PipeProcessorConstant.PROCESSOR_OUTPUT_SERIES_KEY); + validator.validateSynonymAttributes( + Collections.singletonList(PipeProcessorConstant.PROCESSOR_OUTPUT_SERIES_KEY), + Collections.singletonList(LEGACY_PROCESSOR_OUTPUT_SERIES_KEY), + true); final String rawOutputSeries = - validator.getParameters().getString(PipeProcessorConstant.PROCESSOR_OUTPUT_SERIES_KEY); + validator + .getParameters() + .getStringByKeys( + PipeProcessorConstant.PROCESSOR_OUTPUT_SERIES_KEY, + LEGACY_PROCESSOR_OUTPUT_SERIES_KEY); try { PathUtils.isLegalPath(rawOutputSeries); } catch (IllegalPathException e) { @@ -119,8 +127,7 @@ public void customize(PipeParameters parameters, PipeProcessorRuntimeConfigurati regionId = runtimeEnvironment.getRegionId(); pipeTaskMeta = runtimeEnvironment.getPipeTaskMeta(); - outputSeries = - new PartialPath(parameters.getString(PipeProcessorConstant.PROCESSOR_OUTPUT_SERIES_KEY)); + outputSeries = parseOutputSeries(parameters); if (Objects.nonNull(pipeTaskMeta) && Objects.nonNull(pipeTaskMeta.getProgressIndex())) { if (pipeTaskMeta.getProgressIndex() instanceof MinimumProgressIndex) { @@ -152,6 +159,13 @@ public void customize(PipeParameters parameters, PipeProcessorRuntimeConfigurati twoStageAggregateSender = new TwoStageAggregateSender(pipeName, creationTime); } + static PartialPath parseOutputSeries(final PipeParameters parameters) + throws IllegalPathException { + return new PartialPath( + parameters.getStringByKeys( + PipeProcessorConstant.PROCESSOR_OUTPUT_SERIES_KEY, LEGACY_PROCESSOR_OUTPUT_SERIES_KEY)); + } + @Override public void process(TabletInsertionEvent tabletInsertionEvent, EventCollector eventCollector) throws Exception { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiver.java index 8658d12b6a89f..278c1ccaaefc5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiver.java @@ -178,6 +178,11 @@ private void handleReq(final AirGapPseudoTPipeTransferRequest req, final long st if (System.currentTimeMillis() - startTime < PipeConfig.getInstance().getPipeAirGapRetryMaxMs()) { handleReq(req, startTime); + } else { + LOGGER.warn( + "Pipe air gap receiver {}: Temporary unavailable retry timed out, returning FAIL to sender.", + receiverId); + fail(); } } else { LOGGER.warn( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgent.java index ae7ab22308a09..cc85e2f4f100e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgent.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.pipe.receiver.PipeReceiverFilePathUtils; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -51,6 +52,7 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; +import java.nio.file.Paths; import java.time.ZoneId; import java.util.Map; import java.util.Objects; @@ -249,10 +251,12 @@ private SyncIdentityInfo getCurrentSyncIdentityInfo() { * @param tsFilePipeData pipeData * @param fileDir path of file data dir */ - private void handleTsFilePipeData(TsFilePipeData tsFilePipeData, String fileDir) { - String tsFileName = tsFilePipeData.getTsFileName(); - File dir = new File(fileDir); - File[] targetFiles = + private void handleTsFilePipeData(final TsFilePipeData tsFilePipeData, final String fileDir) + throws IOException { + final String tsFileName = tsFilePipeData.getTsFileName(); + final File tsFile = resolveFileInFileDataDir(fileDir, tsFileName); + final File dir = tsFile.getParentFile(); + final File[] targetFiles = dir.listFiles((dir1, name) -> name.startsWith(tsFileName) && name.endsWith(PATCH_SUFFIX)); if (targetFiles != null) { for (File targetFile : targetFiles) { @@ -289,13 +293,21 @@ public TSStatus transportFile(TSyncTransportMetaInfo metaInfo, ByteBuffer buff) LOGGER.debug( "Invoke transportData method from client ip = {}", identityInfo.getRemoteAddress()); - String fileDir = getFileDataDir(identityInfo); - String fileName = metaInfo.fileName; - long startIndex = metaInfo.startIndex; - File file = new File(fileDir, fileName + PATCH_SUFFIX); + final String fileDir = getFileDataDir(identityInfo); + final String fileName = metaInfo.fileName; + final long startIndex = metaInfo.startIndex; + final File file; + final File fileWithoutPatch; + try { + fileWithoutPatch = resolveFileInFileDataDir(fileDir, fileName); + file = resolveFileInFileDataDir(fileDir, fileName + PATCH_SUFFIX); + } catch (final IOException e) { + LOGGER.warn(e.getMessage()); + return RpcUtils.getStatus(TSStatusCode.SYNC_FILE_ERROR, e.getMessage()); + } // step2. check startIndex - IndexCheckResult result = checkStartIndexValid(new File(fileDir, fileName), startIndex); + final IndexCheckResult result = checkStartIndexValid(fileWithoutPatch, startIndex); if (!result.isResult()) { return RpcUtils.getStatus(TSStatusCode.SYNC_FILE_REDIRECTION_ERROR, result.getIndex()); } @@ -307,9 +319,9 @@ public TSStatus transportFile(TSyncTransportMetaInfo metaInfo, ByteBuffer buff) byte[] byteArray = new byte[length]; buff.get(byteArray); randomAccessFile.write(byteArray); - recordStartIndex(new File(fileDir, fileName), startIndex + length); + recordStartIndex(fileWithoutPatch, startIndex + length); LOGGER.debug("Sync {} start at {} to {} is done.", fileName, startIndex, startIndex + length); - } catch (IOException e) { + } catch (final IOException e) { LOGGER.error(e.getMessage()); return RpcUtils.getStatus(TSStatusCode.SYNC_FILE_ERROR, e.getMessage()); } @@ -317,7 +329,21 @@ public TSStatus transportFile(TSyncTransportMetaInfo metaInfo, ByteBuffer buff) return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, ""); } - private IndexCheckResult checkStartIndexValid(File file, long startIndex) { + private static File resolveFileInFileDataDir(final String fileDir, final String fileName) + throws IOException { + if (StringUtils.isEmpty(fileName)) { + throw new IOException("Illegal fileName: " + fileName); + } + + final String illegalError = FileUtils.getIllegalError4Directory(fileName); + if (Objects.nonNull(illegalError)) { + throw new IOException("Illegal fileName: " + fileName + ", " + illegalError); + } + + return PipeReceiverFilePathUtils.resolveFilePath(Paths.get(fileDir), fileName).toFile(); + } + + private IndexCheckResult checkStartIndexValid(final File file, final long startIndex) { // get local index from memory map long localIndex = getCurrentFileStartIndex(file.getAbsolutePath()); // get local index from file diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java index f88510a0ca85d..8459e22f1eed3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/loader/TsFileLoader.java @@ -54,7 +54,7 @@ public TsFileLoader(File tsFile, String database) { @Override public void load() { try { - LoadTsFileStatement statement = new LoadTsFileStatement(tsFile.getAbsolutePath()); + LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath()); statement.setDeleteAfterLoad(true); statement.setConvertOnTypeMismatch(true); statement.setDatabaseLevel(parseSgLevel()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java index 334c3530414f3..8a5c4ad060fa6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiver.java @@ -52,6 +52,7 @@ import org.apache.iotdb.db.pipe.receiver.visitor.PipeStatementTSStatusVisitor; import org.apache.iotdb.db.pipe.receiver.visitor.PipeStatementToBatchVisitor; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; +import org.apache.iotdb.db.pipe.resource.log.PipePeriodicalLogReducer; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV1Req; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV2Req; @@ -459,29 +460,31 @@ protected String getSenderPort() { protected TSStatus loadFileV1(final PipeTransferFileSealReqV1 req, final String fileAbsolutePath) throws IOException { return isUsingAsyncLoadTsFileStrategy.get() - ? loadTsFileAsync(Collections.singletonList(fileAbsolutePath)) - : loadTsFileSync(fileAbsolutePath); + ? loadTsFileAsync(null, Collections.singletonList(fileAbsolutePath)) + : loadTsFileSync(null, fileAbsolutePath); } @Override protected TSStatus loadFileV2( final PipeTransferFileSealReqV2 req, final List fileAbsolutePaths) throws IOException, IllegalPathException { - return req instanceof PipeTransferTsFileSealWithModReq - // TsFile's absolute path will be the second element - ? (isUsingAsyncLoadTsFileStrategy.get() - ? loadTsFileAsync(fileAbsolutePaths) - : loadTsFileSync(fileAbsolutePaths.get(1))) - : loadSchemaSnapShot(req.getParameters(), fileAbsolutePaths); + if (req instanceof PipeTransferTsFileSealWithModReq) { + final String dataBaseName = + ((PipeTransferTsFileSealWithModReq) req).getDatabaseNameByTsFileName(); + return isUsingAsyncLoadTsFileStrategy.get() + ? loadTsFileAsync(dataBaseName, fileAbsolutePaths) + : loadTsFileSync(dataBaseName, fileAbsolutePaths.get(req.getFileNames().size() - 1)); + } + return loadSchemaSnapShot(req.getParameters(), fileAbsolutePaths); } - private TSStatus loadTsFileAsync(final List absolutePaths) throws IOException { + private TSStatus loadTsFileAsync(final String dataBaseName, final List absolutePaths) + throws IOException { final Map loadAttributes = - ActiveLoadPathHelper.buildAttributes( - null, + buildLoadTsFileAttributesForAsync( + dataBaseName, shouldConvertDataTypeOnTypeMismatch, validateTsFile.get(), - null, shouldMarkAsPipeRequest.get()); if (!ActiveLoadUtil.loadFilesToActiveDir(loadAttributes, absolutePaths, true)) { throw new PipeException("Load active listening pipe dir is not set."); @@ -489,16 +492,45 @@ private TSStatus loadTsFileAsync(final List absolutePaths) throws IOExce return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } - private TSStatus loadTsFileSync(final String fileAbsolutePath) throws FileNotFoundException { - final LoadTsFileStatement statement = new LoadTsFileStatement(fileAbsolutePath); + static Map buildLoadTsFileAttributesForAsync( + final String dataBaseName, + final boolean shouldConvertDataTypeOnTypeMismatch, + final boolean validateTsFile, + final boolean shouldMarkAsPipeRequest) { + return ActiveLoadPathHelper.buildAttributes( + dataBaseName, + LoadTsFileStatement.getDatabaseLevelByTreeDatabase(dataBaseName), + shouldConvertDataTypeOnTypeMismatch, + validateTsFile || shouldConvertDataTypeOnTypeMismatch, + null, + shouldMarkAsPipeRequest); + } + private TSStatus loadTsFileSync(final String dataBaseName, final String fileAbsolutePath) + throws FileNotFoundException { + return executeStatementAndClassifyExceptions( + buildLoadTsFileStatementForSync( + dataBaseName, + fileAbsolutePath, + validateTsFile.get(), + shouldConvertDataTypeOnTypeMismatch)); + } + + static LoadTsFileStatement buildLoadTsFileStatementForSync( + final String dataBaseName, + final String fileAbsolutePath, + final boolean validateTsFile, + final boolean shouldConvertDataTypeOnTypeMismatch) + throws FileNotFoundException { + final LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(fileAbsolutePath); statement.setDeleteAfterLoad(true); - statement.setConvertOnTypeMismatch(true); - statement.setVerifySchema(validateTsFile.get()); + statement.setConvertOnTypeMismatch(shouldConvertDataTypeOnTypeMismatch); + statement.setVerifySchema(validateTsFile || shouldConvertDataTypeOnTypeMismatch); statement.setAutoCreateDatabase( IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled()); - - return executeStatementAndClassifyExceptions(statement); + statement.setDatabase(dataBaseName); + statement.updateDatabaseLevelByTreeDatabase(); + return statement; } private TSStatus loadSchemaSnapShot( @@ -705,12 +737,7 @@ private TSStatus executeStatementAndClassifyExceptions( return STATEMENT_STATUS_VISITOR.process(statement, result); } } catch (final Exception e) { - PipeLogger.log( - LOGGER::warn, - e, - "Receiver id = %s: Exception encountered while executing statement %s: ", - receiverId.get(), - statement.getPipeLoggingString()); + logStatementExceptionIfNecessary(statement, e); return STATEMENT_EXCEPTION_VISITOR.process(statement, e); } finally { if (Objects.nonNull(allocatedMemoryBlock)) { @@ -720,15 +747,52 @@ private TSStatus executeStatementAndClassifyExceptions( } } + private void logStatementExceptionIfNecessary(final Statement statement, final Exception e) { + if (shouldLogStatementException(receiverId.get(), statement, e)) { + PipeLogger.log( + LOGGER::warn, + e, + "Receiver id = %s: Exception encountered while executing statement %s: ", + receiverId.get(), + Objects.isNull(statement) ? null : statement.getPipeLoggingString()); + } + } + + static boolean shouldLogStatementException( + final long receiverId, final Statement statement, final Exception e) { + // Use the reducer cache as a gate. The actual stack trace is logged only when it passes. + return PipePeriodicalLogReducer.log( + message -> {}, + "Receiver id = %s, statement = %s, exception = %s, message = %s", + receiverId, + Objects.isNull(statement) ? null : statement.getPipeLoggingString(), + e.getClass().getName(), + e.getMessage()); + } + private TSStatus executeStatementWithRetryOnDataTypeMismatch(final Statement statement) { if (statement == null) { return RpcUtils.getStatus( TSStatusCode.PIPE_TRANSFER_EXECUTE_STATEMENT_ERROR, "Execute null statement."); } + // Execute insert statements through the conversion wrapper first to avoid writing a partial + // row/tablet before the type mismatch is converted. + if (shouldConvertDataTypeOnTypeMismatch && statement instanceof InsertBaseStatement) { + final Optional convertedStatus = + statement.accept( + statementDataTypeConvertExecutionVisitor, + new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + if (convertedStatus.isPresent()) { + return convertedStatus.get(); + } + } + final TSStatus status = executeStatement(statement); - // Try to convert the data type and the status code is not success + // Try to convert data type if the status code is not success. Insert statements normally return + // above after the first converted execution. The retry path is kept for load and fallback + // cases. return shouldConvertDataTypeOnTypeMismatch && ((statement instanceof InsertBaseStatement && ((InsertBaseStatement) statement).hasFailedMeasurements()) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementExceptionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementExceptionVisitor.java index 098c983977a65..7b8246e3dab7a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementExceptionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementExceptionVisitor.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.db.exception.load.LoadRuntimeOutOfMemoryException; import org.apache.iotdb.db.exception.sql.SemanticException; @@ -48,6 +49,13 @@ public class PipeStatementExceptionVisitor extends StatementVisitor { @Override public TSStatus visitNode(final StatementNode node, final Exception context) { + if (context instanceof IoTDBRuntimeException + && ((IoTDBRuntimeException) context).getErrorCode() + == TSStatusCode.DATABASE_NOT_EXIST.getStatusCode()) { + return new TSStatus( + TSStatusCode.PIPE_RECEIVER_PARALLEL_OR_USER_CONFLICT_EXCEPTION.getStatusCode()) + .setMessage(context.getMessage()); + } return new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode()) .setMessage(context.getMessage()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java index b58d934988f1a..009daea6c243b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.java @@ -28,33 +28,270 @@ import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; public class PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner { private static final Logger LOGGER = LoggerFactory.getLogger(PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner.class); + private static final String STALE_PIPE_DIR_SUFFIX = ".startup-cleaning-"; + private static final String PERIODICAL_CLEANUP_JOB_ID = + "PipeDataNodeHardlinkOrCopiedFileDirStartupCleaner#cleanTsFileDir()"; + private static final long DELETE_MAX_PATH_COUNT_PER_ROUND = 100_000L; + private static final long DELETE_MAX_TIME_PER_ROUND_MS = 1_000L; /** * Delete the data directory and all of its subdirectories that contain the * PipeConfig.PIPE_TSFILE_DIR_NAME directory. */ - public static void clean() { - cleanTsFileDir(); + public static void clean(final PeriodicalJobRegistrar periodicalJobRegistrar) { + cleanTsFileDir(periodicalJobRegistrar); cleanSnapshotDir(); } - private static void cleanTsFileDir() { + private static void cleanTsFileDir(final PeriodicalJobRegistrar periodicalJobRegistrar) { + final String pipeHardlinkBaseDirName = PipeConfig.getInstance().getPipeHardlinkBaseDirName(); + final List stalePipeDirs = new ArrayList<>(); for (final String dataDir : IoTDBDescriptor.getInstance().getConfig().getDataDirs()) { - final File pipeHardLinkDir = - new File( - dataDir + File.separator + PipeConfig.getInstance().getPipeHardlinkBaseDirName()); + final File localDataDir = new File(dataDir); + collectInterruptedStalePipeDirs(localDataDir, pipeHardlinkBaseDirName, stalePipeDirs); + + final File pipeHardLinkDir = new File(localDataDir, pipeHardlinkBaseDirName); if (pipeHardLinkDir.isDirectory()) { - LOGGER.info( - "Pipe hardlink dir found, deleting it: {}, result: {}", - pipeHardLinkDir, - FileUtils.deleteQuietly(pipeHardLinkDir)); + moveAsideAndCollect(pipeHardLinkDir, pipeHardlinkBaseDirName, stalePipeDirs); } } + registerPeriodicalCleanupJob(periodicalJobRegistrar, stalePipeDirs); + } + + private static void collectInterruptedStalePipeDirs( + final File localDataDir, + final String pipeHardlinkBaseDirName, + final List stalePipeDirs) { + final File[] stalePipeDirFiles = + localDataDir.listFiles( + file -> + file.isDirectory() + && file.getName().startsWith(pipeHardlinkBaseDirName + STALE_PIPE_DIR_SUFFIX)); + if (stalePipeDirFiles == null) { + return; + } + + for (final File stalePipeDir : stalePipeDirFiles) { + LOGGER.info( + "Stale pipe hardlink dir found, registering it for throttled periodical deletion: {}", + stalePipeDir); + stalePipeDirs.add(stalePipeDir); + } + } + + private static void moveAsideAndCollect( + final File pipeHardLinkDir, + final String pipeHardlinkBaseDirName, + final List stalePipeDirs) { + try { + final File stalePipeDir = moveAside(pipeHardLinkDir, pipeHardlinkBaseDirName); + LOGGER.info( + "Pipe hardlink dir found, moved it from {} to {} for throttled periodical deletion.", + pipeHardLinkDir, + stalePipeDir); + stalePipeDirs.add(stalePipeDir); + } catch (final IOException e) { + LOGGER.warn( + "Failed to move pipe hardlink dir {} for periodical deletion, deleting it synchronously.", + pipeHardLinkDir, + e); + LOGGER.info( + "Pipe hardlink dir found, deleting it: {}, result: {}", + pipeHardLinkDir, + FileUtils.deleteQuietly(pipeHardLinkDir)); + } + } + + private static File moveAside(final File pipeHardLinkDir, final String pipeHardlinkBaseDirName) + throws IOException { + final File parentDir = pipeHardLinkDir.getParentFile(); + if (parentDir == null) { + throw new IOException("Failed to get parent dir of " + pipeHardLinkDir); + } + + final long timestamp = System.currentTimeMillis(); + for (int i = 0; ; ++i) { + final File stalePipeDir = + new File( + parentDir, pipeHardlinkBaseDirName + STALE_PIPE_DIR_SUFFIX + timestamp + "-" + i); + if (!stalePipeDir.exists()) { + Files.move(pipeHardLinkDir.toPath(), stalePipeDir.toPath()); + return stalePipeDir; + } + } + } + + private static void registerPeriodicalCleanupJob( + final PeriodicalJobRegistrar periodicalJobRegistrar, final List stalePipeDirs) { + if (stalePipeDirs.isEmpty()) { + return; + } + + periodicalJobRegistrar.register( + PERIODICAL_CLEANUP_JOB_ID, + new PeriodicalStalePipeDirCleaner(stalePipeDirs)::cleanOneRound, + PipeConfig.getInstance().getPipeSubtaskExecutorCronHeartbeatEventIntervalSeconds()); + } + + private static CleanupRoundResult deleteQuietlyWithThrottle(final File stalePipeDir) { + if (!stalePipeDir.exists()) { + return CleanupRoundResult.finished(); + } + + final AtomicBoolean deleteResult = new AtomicBoolean(true); + final AtomicLong deletedPathCount = new AtomicLong(0); + final long deadlineNanos = System.nanoTime() + DELETE_MAX_TIME_PER_ROUND_MS * 1_000_000L; + try { + Files.walkFileTree( + stalePipeDir.toPath(), + new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory( + final Path dir, final BasicFileAttributes attrs) { + return shouldStop(deletedPathCount, deadlineNanos) + ? FileVisitResult.TERMINATE + : FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) { + return deletePath(file, deleteResult, deletedPathCount, deadlineNanos); + } + + @Override + public FileVisitResult visitFileFailed(final Path file, final IOException exc) { + deleteResult.set(false); + return deletePath(file, deleteResult, deletedPathCount, deadlineNanos); + } + + @Override + public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) { + if (exc != null) { + deleteResult.set(false); + } + return deletePath(dir, deleteResult, deletedPathCount, deadlineNanos); + } + }); + } catch (final IOException e) { + deleteResult.set(false); + } + + return new CleanupRoundResult( + deletedPathCount.get(), deleteResult.get() && !stalePipeDir.exists(), deleteResult.get()); + } + + private static FileVisitResult deletePath( + final Path path, + final AtomicBoolean deleteResult, + final AtomicLong deletedPathCount, + final long deadlineNanos) { + if (shouldStop(deletedPathCount, deadlineNanos)) { + return FileVisitResult.TERMINATE; + } + + try { + if (Files.deleteIfExists(path)) { + deletedPathCount.incrementAndGet(); + } + } catch (final IOException e) { + deleteResult.set(false); + } + return shouldStop(deletedPathCount, deadlineNanos) + ? FileVisitResult.TERMINATE + : FileVisitResult.CONTINUE; + } + + private static boolean shouldStop(final AtomicLong deletedPathCount, final long deadlineNanos) { + return Thread.currentThread().isInterrupted() + || deletedPathCount.get() >= DELETE_MAX_PATH_COUNT_PER_ROUND + || System.nanoTime() >= deadlineNanos; + } + + @FunctionalInterface + public interface PeriodicalJobRegistrar { + + void register(String id, Runnable periodicalJob, long intervalInSeconds); + } + + private static class PeriodicalStalePipeDirCleaner { + + private final List stalePipeDirs; + private int currentDirIndex; + private boolean finished; + + private PeriodicalStalePipeDirCleaner(final List stalePipeDirs) { + this.stalePipeDirs = stalePipeDirs; + currentDirIndex = 0; + finished = false; + } + + private void cleanOneRound() { + if (finished) { + return; + } + + long deletedPathCount = 0; + while (currentDirIndex < stalePipeDirs.size()) { + final File stalePipeDir = stalePipeDirs.get(currentDirIndex); + final CleanupRoundResult result = deleteQuietlyWithThrottle(stalePipeDir); + deletedPathCount += result.deletedPathCount; + + if (result.finished) { + LOGGER.info( + "Finished deleting stale pipe hardlink dir {} by periodical job, result: {}", + stalePipeDir, + result.success); + ++currentDirIndex; + continue; + } + + if (deletedPathCount > 0 || !result.success) { + LOGGER.info( + "Periodically deleted {} paths from stale pipe hardlink dirs, current dir: {}, " + + "current round result: {}", + deletedPathCount, + stalePipeDir, + result.success); + } + return; + } + + finished = true; + LOGGER.info("Finished deleting all stale pipe hardlink dirs by periodical job."); + } + } + + private static class CleanupRoundResult { + + private final long deletedPathCount; + private final boolean finished; + private final boolean success; + + private CleanupRoundResult( + final long deletedPathCount, final boolean finished, final boolean success) { + this.deletedPathCount = deletedPathCount; + this.finished = finished; + this.success = success; + } + + private static CleanupRoundResult finished() { + return new CleanupRoundResult(0, true, true); + } } private static void cleanSnapshotDir() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java index c29eb15371ce2..9616777b021f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java @@ -26,6 +26,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -44,10 +45,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; public class InsertNodeMemoryEstimator { @@ -58,7 +63,6 @@ public class InsertNodeMemoryEstimator { private static final String INSERT_ROWS_NODE = "InsertRowsNode"; private static final String INSERT_ROWS_OF_ONE_DEVICE_NODE = "InsertRowsOfOneDeviceNode"; private static final String INSERT_MULTI_TABLETS_NODE = "InsertMultiTabletsNode"; - private static final String RELATIONAL_INSERT_ROW_NODE = "RelationalInsertRowNode"; private static final long NUM_BYTES_OBJECT_REF = RamUsageEstimator.NUM_BYTES_OBJECT_REF; private static final long NUM_BYTES_OBJECT_HEADER = RamUsageEstimator.NUM_BYTES_OBJECT_HEADER; @@ -158,174 +162,104 @@ public static long sizeOf(final InsertNode insertNode) { // =============================InsertNode================================== private static long calculateFullInsertNodeSize(final InsertNode node) { + return calculateFullInsertNodeSize(node, null); + } + + private static long calculateFullInsertNodeSize( + final InsertNode node, final Set deduplicatedObjects) { long size = 0; + // PlanNodeId + size += sizeOfPlanNodeId(node.getPlanNodeId(), deduplicatedObjects); // PartialPath - size += sizeOfPartialPath(node.getDevicePath()); + size += sizeOfPartialPath(node.getDevicePath(), deduplicatedObjects); // MeasurementSchemas - size += sizeOfMeasurementSchemas(node.getMeasurementSchemas()); + size += sizeOfMeasurementSchemas(node.getMeasurementSchemas(), deduplicatedObjects); // Measurement - size += sizeOfStringArray(node.getMeasurements()); + size += sizeOfStringArray(node.getMeasurements(), deduplicatedObjects); // dataTypes - size += RamUsageEstimator.shallowSizeOf(node.getDataTypes()); + size += sizeOfShallowObject(node.getDataTypes(), deduplicatedObjects); // deviceID if (node.isDeviceIDExists()) { - size += sizeOfIDeviceID(node.getDeviceID()); + size += sizeOfIDeviceID(node.getDeviceID(), deduplicatedObjects); } // dataRegionReplicaSet - size += sizeOfTRegionReplicaSet(node.getRegionReplicaSet()); + size += sizeOfTRegionReplicaSet(node.getRegionReplicaSet(), deduplicatedObjects); // progressIndex - size += sizeOfProgressIndex(node.getProgressIndex()); - return size; - } - - private static long calculateInsertNodeSizeExcludingSchemas(final InsertNode node) { - // Measurement - long size = 2 * RamUsageEstimator.shallowSizeOf(node.getMeasurementSchemas()); - // dataTypes - size += RamUsageEstimator.shallowSizeOf(node.getDataTypes()); - // deviceID - if (node.isDeviceIDExists()) { - size += sizeOfIDeviceID(node.getDeviceID()); - } - // dataRegionReplicaSet - size += sizeOfTRegionReplicaSet(node.getRegionReplicaSet()); - // progressIndex - size += sizeOfProgressIndex(node.getProgressIndex()); + size += sizeOfProgressIndex(node.getProgressIndex(), deduplicatedObjects); return size; } private static long sizeOfInsertTabletNode(final InsertTabletNode node) { - long size = INSERT_TABLET_NODE_SIZE; - size += calculateFullInsertNodeSize(node); - size += RamUsageEstimator.sizeOf(node.getTimes()); - size += sizeOfBitMapArray(node.getBitMaps()); - size += sizeOfColumns(node.getColumns(), node.getMeasurementSchemas()); - final List range = node.getRange(); - if (range != null) { - size += NUM_BYTES_OBJECT_HEADER + SIZE_OF_INT * range.size(); - } - return size; + return sizeOfInsertTabletNode(node, newDeduplicatedObjectSet()); } - private static long calculateInsertTabletNodeSizeExcludingSchemas(final InsertTabletNode node) { + private static long sizeOfInsertTabletNode( + final InsertTabletNode node, final Set deduplicatedObjects) { long size = INSERT_TABLET_NODE_SIZE; - - size += calculateInsertNodeSizeExcludingSchemas(node); - + size += calculateFullInsertNodeSize(node, deduplicatedObjects); size += RamUsageEstimator.sizeOf(node.getTimes()); - size += sizeOfBitMapArray(node.getBitMaps()); - size += sizeOfColumns(node.getColumns(), node.getMeasurementSchemas()); - - final List range = node.getRange(); - if (range != null) { - size += NUM_BYTES_OBJECT_HEADER + SIZE_OF_INT * range.size(); - } + size += sizeOfIntegerList(node.getRange()); return size; } private static long sizeOfInsertRowNode(final InsertRowNode node) { - long size = INSERT_ROW_NODE_SIZE; - size += calculateFullInsertNodeSize(node); - size += sizeOfValues(node.getValues(), node.getMeasurementSchemas()); - return size; + return sizeOfInsertRowNode(node, newDeduplicatedObjectSet()); } - private static long calculateInsertRowNodeExcludingSchemas(final InsertRowNode node) { + private static long sizeOfInsertRowNode( + final InsertRowNode node, final Set deduplicatedObjects) { long size = INSERT_ROW_NODE_SIZE; - size += calculateInsertNodeSizeExcludingSchemas(node); + size += calculateFullInsertNodeSize(node, deduplicatedObjects); size += sizeOfValues(node.getValues(), node.getMeasurementSchemas()); return size; } private static long sizeOfInsertRowsNode(final InsertRowsNode node) { + final Set deduplicatedObjects = newDeduplicatedObjectSet(); long size = INSERT_ROWS_NODE_SIZE; - size += calculateFullInsertNodeSize(node); - final List rows = node.getInsertRowNodeList(); - final List indexList = node.getInsertRowNodeIndexList(); - if (rows != null && !rows.isEmpty()) { - // InsertRowNodeList - size += NUM_BYTES_OBJECT_HEADER; - size += - (calculateInsertRowNodeExcludingSchemas(rows.get(0)) + NUM_BYTES_OBJECT_REF) - * rows.size(); - size += sizeOfPartialPath(rows.get(0).getDevicePath()); - size += sizeOfMeasurementSchemas(rows.get(0).getMeasurementSchemas()); - // InsertRowNodeIndexList - size += NUM_BYTES_OBJECT_HEADER; - size += (long) indexList.size() * (SIZE_OF_INT + NUM_BYTES_OBJECT_REF); - } + size += calculateFullInsertNodeSize(node, deduplicatedObjects); + size += sizeOfInsertRowNodeList(node.getInsertRowNodeList(), deduplicatedObjects); + size += sizeOfIntegerList(node.getInsertRowNodeIndexList()); + size += sizeOfResults(node.getResults()); return size; } private static long sizeOfInsertRowsOfOneDeviceNode(final InsertRowsOfOneDeviceNode node) { + final Set deduplicatedObjects = newDeduplicatedObjectSet(); long size = INSERT_ROWS_OF_ONE_DEVICE_NODE_SIZE; - size += calculateFullInsertNodeSize(node); - final List rows = node.getInsertRowNodeList(); - final List indexList = node.getInsertRowNodeIndexList(); - if (rows != null && !rows.isEmpty()) { - // InsertRowNodeList - size += NUM_BYTES_OBJECT_HEADER; - size += - (calculateInsertRowNodeExcludingSchemas(rows.get(0)) + NUM_BYTES_OBJECT_REF) - * rows.size(); - size += sizeOfPartialPath(rows.get(0).getDevicePath()); - size += sizeOfMeasurementSchemas(rows.get(0).getMeasurementSchemas()); - // InsertRowNodeIndexList - size += NUM_BYTES_OBJECT_HEADER; - size += (long) indexList.size() * (SIZE_OF_INT + NUM_BYTES_OBJECT_REF); - } - // results - size += NUM_BYTES_OBJECT_HEADER; - for (Map.Entry entry : node.getResults().entrySet()) { - size += - Integer.BYTES - + sizeOfTSStatus(entry.getValue()) - + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY; - } + size += calculateFullInsertNodeSize(node, deduplicatedObjects); + size += sizeOfInsertRowNodeList(node.getInsertRowNodeList(), deduplicatedObjects); + size += sizeOfIntegerList(node.getInsertRowNodeIndexList()); + size += sizeOfResults(node.getResults()); return size; } private static long sizeOfInsertMultiTabletsNode(final InsertMultiTabletsNode node) { + final Set deduplicatedObjects = newDeduplicatedObjectSet(); long size = INSERT_MULTI_TABLETS_NODE_SIZE; - size += calculateFullInsertNodeSize(node); - // dataTypes - size += RamUsageEstimator.shallowSizeOf(node.getDataTypes()); - - final List rows = node.getInsertTabletNodeList(); - final List indexList = node.getParentInsertTabletNodeIndexList(); - if (rows != null && !rows.isEmpty()) { - // InsertTabletNodeList - size += NUM_BYTES_OBJECT_HEADER; - size += - (calculateInsertTabletNodeSizeExcludingSchemas(rows.get(0)) + NUM_BYTES_OBJECT_REF) - * rows.size(); - size += sizeOfPartialPath(rows.get(0).getDevicePath()); - size += sizeOfMeasurementSchemas(rows.get(0).getMeasurementSchemas()); - // ParentInsertTabletNodeIndexList - size += NUM_BYTES_OBJECT_HEADER; - size += (long) indexList.size() * (SIZE_OF_INT + NUM_BYTES_OBJECT_REF); - } - // results - if (node.getResults() != null) { - size += NUM_BYTES_OBJECT_HEADER; - for (Map.Entry entry : node.getResults().entrySet()) { - size += - Integer.BYTES - + sizeOfTSStatus(entry.getValue()) - + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY; - } - } + size += calculateFullInsertNodeSize(node, deduplicatedObjects); + size += sizeOfInsertTabletNodeList(node.getInsertTabletNodeList(), deduplicatedObjects); + size += sizeOfIntegerList(node.getParentInsertTabletNodeIndexList()); + size += sizeOfResults(node.getResults()); return size; } // ============================Device And Measurement=================================== public static long sizeOfPartialPath(final PartialPath partialPath) { + return sizeOfPartialPath(partialPath, null); + } + + private static long sizeOfPartialPath( + final PartialPath partialPath, final Set deduplicatedObjects) { if (partialPath == null) { return 0L; } + if (!shouldCountObject(partialPath, deduplicatedObjects)) { + return 0L; + } long size = PARTIAL_PATH_SIZE; final String[] nodes = partialPath.getNodes(); if (nodes != null) { @@ -338,22 +272,38 @@ public static long sizeOfPartialPath(final PartialPath partialPath) { } public static long sizeOfMeasurementSchemas(final MeasurementSchema[] measurementSchemas) { + return sizeOfMeasurementSchemas(measurementSchemas, null); + } + + private static long sizeOfMeasurementSchemas( + final MeasurementSchema[] measurementSchemas, final Set deduplicatedObjects) { if (measurementSchemas == null) { return 0L; } + if (!shouldCountObject(measurementSchemas, deduplicatedObjects)) { + return 0L; + } long size = RamUsageEstimator.alignObjectSize( NUM_BYTES_ARRAY_HEADER + NUM_BYTES_OBJECT_REF * measurementSchemas.length); for (MeasurementSchema measurementSchema : measurementSchemas) { - size += sizeOfMeasurementSchema(measurementSchema); + size += sizeOfMeasurementSchema(measurementSchema, deduplicatedObjects); } return size; } - private static long sizeOfMeasurementSchema(final MeasurementSchema measurementSchema) { + public static long sizeOfMeasurementSchema(final MeasurementSchema measurementSchema) { + return sizeOfMeasurementSchema(measurementSchema, null); + } + + private static long sizeOfMeasurementSchema( + final MeasurementSchema measurementSchema, final Set deduplicatedObjects) { if (measurementSchema == null) { return 0L; } + if (!shouldCountObject(measurementSchema, deduplicatedObjects)) { + return 0L; + } // Header + primitive + reference long size = MEASUREMENT_SCHEMA_SIZE; // measurementId @@ -361,7 +311,7 @@ private static long sizeOfMeasurementSchema(final MeasurementSchema measurementS // props final Map props = measurementSchema.getProps(); if (props != null) { - size += NUM_BYTES_OBJECT_HEADER; + size += RamUsageEstimator.shallowSizeOf(props); for (Map.Entry entry : props.entrySet()) { size += RamUsageEstimator.sizeOf(entry.getKey()) @@ -373,16 +323,31 @@ private static long sizeOfMeasurementSchema(final MeasurementSchema measurementS return size; } - private static long sizeOfIDeviceID(final IDeviceID deviceID) { - return Objects.nonNull(deviceID) ? deviceID.ramBytesUsed() : 0L; + public static long sizeOfIDeviceID(final IDeviceID deviceID) { + return sizeOfIDeviceID(deviceID, null); + } + + private static long sizeOfIDeviceID( + final IDeviceID deviceID, final Set deduplicatedObjects) { + return Objects.nonNull(deviceID) && shouldCountObject(deviceID, deduplicatedObjects) + ? deviceID.ramBytesUsed() + : 0L; } // =============================Thrift================================== private static long sizeOfTRegionReplicaSet(final TRegionReplicaSet tRegionReplicaSet) { + return sizeOfTRegionReplicaSet(tRegionReplicaSet, null); + } + + private static long sizeOfTRegionReplicaSet( + final TRegionReplicaSet tRegionReplicaSet, final Set deduplicatedObjects) { if (tRegionReplicaSet == null) { return 0L; } + if (!shouldCountObject(tRegionReplicaSet, deduplicatedObjects)) { + return 0L; + } // Memory alignment of basic types and reference types in structures long size = T_REGION_REPLICA_SET_SIZE; // Memory calculation in reference type, cannot get exact value, roughly estimate @@ -390,9 +355,9 @@ private static long sizeOfTRegionReplicaSet(final TRegionReplicaSet tRegionRepli size += sizeOfTConsensusGroupId(); } if (tRegionReplicaSet.isSetDataNodeLocations()) { - size += NUM_BYTES_OBJECT_HEADER; + size += sizeOfObjectList(tRegionReplicaSet.getDataNodeLocations()); for (TDataNodeLocation tDataNodeLocation : tRegionReplicaSet.getDataNodeLocations()) { - size += sizeOfTDataNodeLocation(tDataNodeLocation); + size += sizeOfTDataNodeLocation(tDataNodeLocation, deduplicatedObjects); } } return size; @@ -403,25 +368,39 @@ private static long sizeOfTConsensusGroupId() { return T_CONSENSUS_GROUP_ID_SIZE; } - private static long sizeOfTDataNodeLocation(final TDataNodeLocation tDataNodeLocation) { + private static long sizeOfTDataNodeLocation( + final TDataNodeLocation tDataNodeLocation, final Set deduplicatedObjects) { if (tDataNodeLocation == null) { return 0L; } + if (!shouldCountObject(tDataNodeLocation, deduplicatedObjects)) { + return 0L; + } long size = T_DATA_NODE_LOCATION_SIZE; - size += sizeOfTEndPoint(tDataNodeLocation.getClientRpcEndPoint()); - size += sizeOfTEndPoint(tDataNodeLocation.getInternalEndPoint()); - size += sizeOfTEndPoint(tDataNodeLocation.getMPPDataExchangeEndPoint()); - size += sizeOfTEndPoint(tDataNodeLocation.getDataRegionConsensusEndPoint()); - size += sizeOfTEndPoint(tDataNodeLocation.getSchemaRegionConsensusEndPoint()); + size += sizeOfTEndPoint(tDataNodeLocation.getClientRpcEndPoint(), deduplicatedObjects); + size += sizeOfTEndPoint(tDataNodeLocation.getInternalEndPoint(), deduplicatedObjects); + size += sizeOfTEndPoint(tDataNodeLocation.getMPPDataExchangeEndPoint(), deduplicatedObjects); + size += + sizeOfTEndPoint(tDataNodeLocation.getDataRegionConsensusEndPoint(), deduplicatedObjects); + size += + sizeOfTEndPoint(tDataNodeLocation.getSchemaRegionConsensusEndPoint(), deduplicatedObjects); return size; } private static long sizeOfTEndPoint(final TEndPoint tEndPoint) { + return sizeOfTEndPoint(tEndPoint, null); + } + + private static long sizeOfTEndPoint( + final TEndPoint tEndPoint, final Set deduplicatedObjects) { if (tEndPoint == null) { return 0L; } + if (!shouldCountObject(tEndPoint, deduplicatedObjects)) { + return 0L; + } // objectHeader + ip + port long size = T_END_POINT_SIZE; @@ -430,18 +409,32 @@ private static long sizeOfTEndPoint(final TEndPoint tEndPoint) { } private static long sizeOfTSStatus(final TSStatus tSStatus) { + return sizeOfTSStatus(tSStatus, null); + } + + private static long sizeOfTSStatus( + final TSStatus tSStatus, final Set deduplicatedObjects) { if (tSStatus == null) { return 0L; } + if (!shouldCountObject(tSStatus, deduplicatedObjects)) { + return 0L; + } long size = TS_STATUS_SIZE; // message if (tSStatus.isSetMessage()) { size += RamUsageEstimator.sizeOf(tSStatus.message); } - // ignore subStatus + // subStatus + if (tSStatus.getSubStatus() != null) { + size += sizeOfObjectList(tSStatus.getSubStatus()); + for (TSStatus subStatus : tSStatus.getSubStatus()) { + size += sizeOfTSStatus(subStatus, deduplicatedObjects); + } + } // redirectNode if (tSStatus.isSetRedirectNode()) { - size += sizeOfTEndPoint(tSStatus.redirectNode); + size += sizeOfTEndPoint(tSStatus.redirectNode, deduplicatedObjects); } return size; } @@ -449,7 +442,14 @@ private static long sizeOfTSStatus(final TSStatus tSStatus) { // =============================ProgressIndex================================== private static long sizeOfProgressIndex(final ProgressIndex progressIndex) { - return Objects.nonNull(progressIndex) ? progressIndex.ramBytesUsed() : 0L; + return sizeOfProgressIndex(progressIndex, null); + } + + private static long sizeOfProgressIndex( + final ProgressIndex progressIndex, final Set deduplicatedObjects) { + return Objects.nonNull(progressIndex) && shouldCountObject(progressIndex, deduplicatedObjects) + ? progressIndex.ramBytesUsed() + : 0L; } // =============================Write================================== @@ -617,4 +617,96 @@ public static long sizeOfValues( } return size; } + + private static long sizeOfPlanNodeId( + final PlanNodeId planNodeId, final Set deduplicatedObjects) { + return planNodeId != null && shouldCountObject(planNodeId, deduplicatedObjects) + ? planNodeId.ramBytesUsed() + : 0L; + } + + private static long sizeOfStringArray( + final String[] strings, final Set deduplicatedObjects) { + return strings != null && shouldCountObject(strings, deduplicatedObjects) + ? RamUsageEstimator.sizeOf(strings) + : 0L; + } + + private static long sizeOfShallowObject( + final Object object, final Set deduplicatedObjects) { + return object != null && shouldCountObject(object, deduplicatedObjects) + ? RamUsageEstimator.shallowSizeOf(object) + : 0L; + } + + private static long sizeOfInsertRowNodeList( + final List rows, final Set deduplicatedObjects) { + if (rows == null) { + return 0L; + } + long size = sizeOfObjectList(rows); + for (InsertRowNode row : rows) { + size += sizeOfInsertRowNode(row, deduplicatedObjects); + } + return size; + } + + private static long sizeOfInsertTabletNodeList( + final List tablets, final Set deduplicatedObjects) { + if (tablets == null) { + return 0L; + } + long size = sizeOfObjectList(tablets); + for (InsertTabletNode tablet : tablets) { + size += sizeOfInsertTabletNode(tablet, deduplicatedObjects); + } + return size; + } + + private static long sizeOfObjectList(final List list) { + if (list == null) { + return 0L; + } + long size = RamUsageEstimator.shallowSizeOf(list); + if (list instanceof ArrayList) { + size += + RamUsageEstimator.alignObjectSize( + NUM_BYTES_ARRAY_HEADER + NUM_BYTES_OBJECT_REF * list.size()); + } + return size; + } + + private static long sizeOfIntegerList(final List integers) { + if (integers == null) { + return 0L; + } + long size = sizeOfObjectList(integers); + for (Integer ignored : integers) { + size += SIZE_OF_INT; + } + return size; + } + + private static long sizeOfResults(final Map results) { + if (results == null) { + return 0L; + } + long size = RamUsageEstimator.shallowSizeOf(results); + for (Map.Entry entry : results.entrySet()) { + size += + SIZE_OF_INT + + sizeOfTSStatus(entry.getValue()) + + RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY; + } + return size; + } + + private static Set newDeduplicatedObjectSet() { + return Collections.newSetFromMap(new IdentityHashMap<>()); + } + + private static boolean shouldCountObject( + final Object object, final Set deduplicatedObjects) { + return object != null && (deduplicatedObjects == null || deduplicatedObjects.add(object)); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlock.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlock.java index 07f5b904523c8..62069a2fb2504 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlock.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlock.java @@ -24,6 +24,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -62,6 +63,7 @@ public void setMemoryUsageInBytes(final long memoryUsageInBytes) { public PipeMemoryBlock setShrinkMethod(final LongUnaryOperator shrinkMethod) { this.shrinkMethod.set(shrinkMethod); + pipeMemoryManager.addShrinkableBlock(this); return this; } @@ -72,6 +74,7 @@ public PipeMemoryBlock setShrinkCallback(final BiConsumer shrinkCall public PipeMemoryBlock setExpandMethod(final LongUnaryOperator extendMethod) { this.expandMethod.set(extendMethod); + pipeMemoryManager.addExpandableBlock(this); return this; } @@ -177,6 +180,12 @@ public void close() { if (lock.tryLock(50, TimeUnit.MICROSECONDS)) { try { pipeMemoryManager.release(this); + if (Objects.nonNull(shrinkMethod.get())) { + pipeMemoryManager.removeShrinkableBlock(this); + } + if (Objects.nonNull(expandMethod.get())) { + pipeMemoryManager.removeExpandableBlock(this); + } if (isInterrupted) { LOGGER.warn("{} is released after thread interruption.", this); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java index 87be4d5fb62f4..0d49487f7504f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java @@ -56,6 +56,8 @@ public class PipeMemoryManager { // Only non-zero memory blocks will be added to this set. private final Set allocatedBlocks = new HashSet<>(); + private final Set shrinkableBlocks = new HashSet<>(); + private final Set expandableBlocks = new HashSet<>(); public PipeMemoryManager() { PipeDataNodeAgent.runtime() @@ -531,8 +533,9 @@ private PipeMemoryBlock registerMemoryBlock(long sizeInBytes, PipeMemoryBlockTyp return returnedMemoryBlock; } + // Single-threaded logic private boolean tryShrinkUntilFreeMemorySatisfy(long sizeInBytes) { - final List shuffledBlocks = new ArrayList<>(allocatedBlocks); + final List shuffledBlocks = new ArrayList<>(shrinkableBlocks); Collections.shuffle(shuffledBlocks); while (true) { @@ -551,46 +554,64 @@ private boolean tryShrinkUntilFreeMemorySatisfy(long sizeInBytes) { } } + void addShrinkableBlock(final PipeMemoryBlock block) { + shrinkableBlocks.add(block); + } + + void removeShrinkableBlock(final PipeMemoryBlock block) { + shrinkableBlocks.remove(block); + } + public synchronized void tryExpandAllAndCheckConsistency() { - allocatedBlocks.forEach(PipeMemoryBlock::expand); + expandableBlocks.forEach(PipeMemoryBlock::expand); + + if (LOGGER.isDebugEnabled()) { + final long blockSum = + allocatedBlocks.stream().mapToLong(PipeMemoryBlock::getMemoryUsageInBytes).sum(); + if (blockSum != usedMemorySizeInBytes) { + LOGGER.debug( + "tryExpandAllAndCheckConsistency: memory usage is not consistent with allocated blocks," + + " usedMemorySizeInBytes is {} but sum of all blocks is {}", + usedMemorySizeInBytes, + blockSum); + } - long blockSum = - allocatedBlocks.stream().mapToLong(PipeMemoryBlock::getMemoryUsageInBytes).sum(); - if (blockSum != usedMemorySizeInBytes) { - LOGGER.warn( - "tryExpandAllAndCheckConsistency: memory usage is not consistent with allocated blocks," - + " usedMemorySizeInBytes is {} but sum of all blocks is {}", - usedMemorySizeInBytes, - blockSum); - } + final long tabletBlockSum = + allocatedBlocks.stream() + .filter(PipeTabletMemoryBlock.class::isInstance) + .mapToLong(PipeMemoryBlock::getMemoryUsageInBytes) + .sum(); + if (tabletBlockSum != usedMemorySizeInBytesOfTablets) { + LOGGER.debug( + "tryExpandAllAndCheckConsistency: memory usage of tablets is not consistent with allocated blocks," + + " usedMemorySizeInBytesOfTablets is {} but sum of all tablet blocks is {}", + usedMemorySizeInBytesOfTablets, + tabletBlockSum); + } - long tabletBlockSum = - allocatedBlocks.stream() - .filter(PipeTabletMemoryBlock.class::isInstance) - .mapToLong(PipeMemoryBlock::getMemoryUsageInBytes) - .sum(); - if (tabletBlockSum != usedMemorySizeInBytesOfTablets) { - LOGGER.warn( - "tryExpandAllAndCheckConsistency: memory usage of tablets is not consistent with allocated blocks," - + " usedMemorySizeInBytesOfTablets is {} but sum of all tablet blocks is {}", - usedMemorySizeInBytesOfTablets, - tabletBlockSum); - } - - long tsFileBlockSum = - allocatedBlocks.stream() - .filter(PipeTsFileMemoryBlock.class::isInstance) - .mapToLong(PipeMemoryBlock::getMemoryUsageInBytes) - .sum(); - if (tsFileBlockSum != usedMemorySizeInBytesOfTsFiles) { - LOGGER.warn( - "tryExpandAllAndCheckConsistency: memory usage of tsfiles is not consistent with allocated blocks," - + " usedMemorySizeInBytesOfTsFiles is {} but sum of all tsfile blocks is {}", - usedMemorySizeInBytesOfTsFiles, - tsFileBlockSum); + final long tsFileBlockSum = + allocatedBlocks.stream() + .filter(PipeTsFileMemoryBlock.class::isInstance) + .mapToLong(PipeMemoryBlock::getMemoryUsageInBytes) + .sum(); + if (tsFileBlockSum != usedMemorySizeInBytesOfTsFiles) { + LOGGER.debug( + "tryExpandAllAndCheckConsistency: memory usage of tsfiles is not consistent with allocated blocks," + + " usedMemorySizeInBytesOfTsFiles is {} but sum of all tsfile blocks is {}", + usedMemorySizeInBytesOfTsFiles, + tsFileBlockSum); + } } } + void addExpandableBlock(final PipeMemoryBlock block) { + expandableBlocks.add(block); + } + + void removeExpandableBlock(final PipeMemoryBlock block) { + expandableBlocks.remove(block); + } + public synchronized void release(PipeMemoryBlock block) { if (!PIPE_MEMORY_MANAGEMENT_ENABLED || block == null || block.isReleased()) { return; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java index a707f554c5184..a22522666c2ff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryWeightUtil.java @@ -179,14 +179,14 @@ private static Pair calculateTabletRowCountAndMemoryBySize( return new Pair<>(1, 0); } - // Calculate row number according to the max size of a pipe tablet. - // "-100" is the estimated size of other data structures in a pipe tablet. + // Calculate row number according to the max size of a pipe tablet. "100" is the estimated size + // of other data structures in a pipe tablet. // "*8" converts bytes to bits, because the bitmap size is 1 bit per schema. - // Here we estimate the max use of int sizeLimit = - Math.min( - IoTDBDescriptor.getInstance().getConfig().getPipeDataStructureTabletSizeInBytes(), - (int) (inputNum * rowBytesUsed * 1.2)); + (int) + Math.min( + IoTDBDescriptor.getInstance().getConfig().getPipeDataStructureTabletSizeInBytes(), + Math.min(Integer.MAX_VALUE, 100 + inputNum * (double) rowBytesUsed * 1.2)); int rowNumber = 8 * (sizeLimit - 100) / (8 * rowBytesUsed + schemaCount); rowNumber = Math.max(1, rowNumber); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeAsyncClientManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeAsyncClientManager.java index 435f890f3a2e2..e2f6dbec5693a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeAsyncClientManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeAsyncClientManager.java @@ -193,11 +193,12 @@ public AsyncPipeDataTransferServiceClient borrowClient(final TEndPoint endPoint) return client; } } catch (final Exception e) { - LOGGER.warn( - "failed to borrow client {}:{} for cached leader.", + PipeLogger.log( + LOGGER::warn, + e, + "Failed to borrow client %s:%s for cached leader.", endPoint.getIp(), - endPoint.getPort(), - e); + endPoint.getPort()); } return borrowClient(); @@ -342,11 +343,12 @@ public void onError(final Exception e) { client.close(); client.invalidateAll(); } catch (final Exception e) { - LOGGER.warn( - "Failed to close client {}:{} after handshake failure when the manager is closed.", + PipeLogger.log( + LOGGER::warn, + e, + "Failed to close client %s:%s after handshake failure when the manager is closed.", targetNodeUrl.getIp(), - targetNodeUrl.getPort(), - e); + targetNodeUrl.getPort()); } } client.setShouldReturnSelf(true); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventBatch.java index 96bddd0d672f0..7058b88575015 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventBatch.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.pipe.sink.payload.evolvable.batch; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; @@ -154,10 +155,15 @@ public synchronized void close() { * Discard all events of the given pipe. This method only clears the reference count of the events * and discard them, but do not modify other objects (such as buffers) for simplicity. */ - public synchronized void discardEventsOfPipe(final String pipeNameToDrop, final int regionId) { + public synchronized void discardEventsOfPipe( + final String pipeNameToDrop, final long creationTimeToDrop, final int regionId) { + discardEventsOfPipe(new CommitterKey(pipeNameToDrop, creationTimeToDrop, regionId, -1)); + } + + public synchronized void discardEventsOfPipe(final CommitterKey committerKey) { events.removeIf( event -> { - if (pipeNameToDrop.equals(event.getPipeName()) && regionId == event.getRegionId()) { + if (isEventFromPipe(event, committerKey)) { event.clearReferenceCount(IoTDBDataRegionAsyncSink.class.getName()); return true; } @@ -165,6 +171,14 @@ public synchronized void discardEventsOfPipe(final String pipeNameToDrop, final }); } + private static boolean isEventFromPipe( + final EnrichedEvent event, final CommitterKey committerKey) { + return committerKey.getPipeName().equals(event.getPipeName()) + && committerKey.getCreationTime() == event.getCreationTime() + && committerKey.getRegionId() == event.getRegionId() + && (committerKey.getRestartTimes() < 0 || committerKey.equals(event.getCommitterKey())); + } + public synchronized void decreaseEventsReferenceCount( final String holderMessage, final boolean shouldReport) { events.forEach(event -> event.decreaseReferenceCount(holderMessage, shouldReport)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java index 36ca10daa725a..2e69b23927743 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventTsFileBatch.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; import org.apache.iotdb.db.pipe.sink.util.PipeTabletEventSorter; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; @@ -361,7 +362,7 @@ private Tablet tryBestToAggregateTablets( // Aggregate the current tablet's data aggregatedSchemas.addAll(tablet.getSchemas()); aggregatedValues.addAll(Arrays.asList(tablet.values)); - aggregatedBitMaps.addAll(Arrays.asList(tablet.bitMaps)); + aggregatedBitMaps.addAll(Arrays.asList(PipeTabletUtils.copyBitMapsOrCreateEmpty(tablet))); // Remove the aggregated tablet tablets.pollFirst(); } else { @@ -563,7 +564,7 @@ private void writeTabletsIntoOneFile( .map(schema -> (MeasurementSchema) schema) .toArray(MeasurementSchema[]::new); Object[] values = Arrays.copyOf(tablet.values, tablet.values.length); - BitMap[] bitMaps = Arrays.copyOf(tablet.bitMaps, tablet.bitMaps.length); + BitMap[] bitMaps = PipeTabletUtils.copyBitMapsOrCreateEmpty(tablet); // convert date value to int refer to // org.apache.iotdb.db.storageengine.dataregion.memtable.WritableMemChunk.writeNonAlignedTablet diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTransferBatchReqBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTransferBatchReqBuilder.java index 9fa706e985f1e..52827ac000885 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTransferBatchReqBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTransferBatchReqBuilder.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.sink.payload.evolvable.batch; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; @@ -38,10 +39,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_FORMAT_HYBRID_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_FORMAT_KEY; @@ -83,8 +84,7 @@ public class PipeTransferBatchReqBuilder implements AutoCloseable { // If the leader cache is enabled, the batch will be divided by the leader endpoint, // each endpoint has a batch. // This is only used in plain batch since tsfile does not return redirection info. - private final Map endPointToBatch = - new ConcurrentHashMap<>(); + private final Map endPointToBatch = new HashMap<>(); public PipeTransferBatchReqBuilder(final PipeParameters parameters) { final boolean usingTsFileBatch = @@ -177,35 +177,48 @@ public synchronized void onEvent(final TabletInsertionEvent event) public synchronized List> getAllNonEmptyAndShouldEmitBatches() { final List> nonEmptyAndShouldEmitBatches = - new ArrayList<>(); + new ArrayList<>(endPointToBatch.size() + 1); if (!defaultBatch.isEmpty() && defaultBatch.shouldEmit()) { nonEmptyAndShouldEmitBatches.add(new Pair<>(null, defaultBatch)); } - endPointToBatch.forEach( - (endPoint, batch) -> { - if (!batch.isEmpty() && batch.shouldEmit()) { - nonEmptyAndShouldEmitBatches.add(new Pair<>(endPoint, batch)); - } - }); + for (final Map.Entry entry : endPointToBatch.entrySet()) { + final PipeTabletEventPlainBatch batch = entry.getValue(); + if (!batch.isEmpty() && batch.shouldEmit()) { + nonEmptyAndShouldEmitBatches.add(new Pair<>(entry.getKey(), batch)); + } + } return nonEmptyAndShouldEmitBatches; } - public boolean isEmpty() { - return defaultBatch.isEmpty() - && endPointToBatch.values().stream().allMatch(PipeTabletEventPlainBatch::isEmpty); + public synchronized boolean isEmpty() { + if (!defaultBatch.isEmpty()) { + return false; + } + for (final PipeTabletEventPlainBatch batch : endPointToBatch.values()) { + if (!batch.isEmpty()) { + return false; + } + } + return true; + } + + public synchronized void discardEventsOfPipe( + final String pipeNameToDrop, final long creationTimeToDrop, final int regionId) { + discardEventsOfPipe(new CommitterKey(pipeNameToDrop, creationTimeToDrop, regionId, -1)); } - public synchronized void discardEventsOfPipe(final String pipeNameToDrop, final int regionId) { - defaultBatch.discardEventsOfPipe(pipeNameToDrop, regionId); - endPointToBatch.values().forEach(batch -> batch.discardEventsOfPipe(pipeNameToDrop, regionId)); + public synchronized void discardEventsOfPipe(final CommitterKey committerKey) { + defaultBatch.discardEventsOfPipe(committerKey); + endPointToBatch.values().forEach(batch -> batch.discardEventsOfPipe(committerKey)); } - public int size() { + public synchronized int size() { try { - return defaultBatch.events.size() - + endPointToBatch.values().stream() - .map(batch -> batch.events.size()) - .reduce(0, Integer::sum); + int size = defaultBatch.events.size(); + for (final PipeTabletEventPlainBatch batch : endPointToBatch.values()) { + size += batch.events.size(); + } + return size; } catch (final Exception e) { LOGGER.warn( "Failed to get the size of PipeTransferBatchReqBuilder, return 0. Exception: {}", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBatchReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBatchReq.java index 94a838ee0ad69..266894060dcb8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBatchReq.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletBatchReq.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils.TabletStringInternPool; import org.apache.iotdb.db.queryengine.plan.planner.plan.PlanFragment; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; @@ -34,7 +35,6 @@ import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.ReadWriteIOUtils; -import org.apache.tsfile.write.record.Tablet; import java.io.DataOutputStream; import java.io.IOException; @@ -130,6 +130,7 @@ public static PipeTransferTabletBatchReq toTPipeTransferReq( public static PipeTransferTabletBatchReq fromTPipeTransferReq( final TPipeTransferReq transferReq) { final PipeTransferTabletBatchReq batchReq = new PipeTransferTabletBatchReq(); + final TabletStringInternPool tabletStringInternPool = new TabletStringInternPool(); // Binary req, for rolling upgrading ReadWriteIOUtils.readInt(transferReq.body); @@ -144,8 +145,7 @@ public static PipeTransferTabletBatchReq fromTPipeTransferReq( size = ReadWriteIOUtils.readInt(transferReq.body); for (int i = 0; i < size; ++i) { batchReq.tabletReqs.add( - PipeTransferTabletRawReq.toTPipeTransferRawReq( - Tablet.deserialize(transferReq.body), ReadWriteIOUtils.readBool(transferReq.body))); + PipeTransferTabletRawReq.toTPipeTransferRawReq(transferReq.body, tabletStringInternPool)); } batchReq.version = transferReq.version; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReq.java index 47bf4d4489700..60619fd426816 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReq.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReq.java @@ -22,6 +22,8 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils.TabletStringInternPool; import org.apache.iotdb.db.pipe.sink.util.PipeTabletEventSorter; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; @@ -82,6 +84,17 @@ public static PipeTransferTabletRawReq toTPipeTransferRawReq( return tabletReq; } + public static PipeTransferTabletRawReq toTPipeTransferRawReq( + final ByteBuffer buffer, final TabletStringInternPool tabletStringInternPool) { + final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq(); + + tabletReq.tablet = + PipeTabletUtils.internTablet(Tablet.deserialize(buffer), tabletStringInternPool); + tabletReq.isAligned = ReadWriteIOUtils.readBool(buffer); + + return tabletReq; + } + /////////////////////////////// Thrift /////////////////////////////// public static PipeTransferTabletRawReq toTPipeTransferReq( @@ -105,10 +118,8 @@ public static PipeTransferTabletRawReq toTPipeTransferReq( } public static PipeTransferTabletRawReq fromTPipeTransferReq(final TPipeTransferReq transferReq) { - final PipeTransferTabletRawReq tabletReq = new PipeTransferTabletRawReq(); - - tabletReq.tablet = Tablet.deserialize(transferReq.body); - tabletReq.isAligned = ReadWriteIOUtils.readBool(transferReq.body); + final PipeTransferTabletRawReq tabletReq = + toTPipeTransferRawReq(transferReq.body, new TabletStringInternPool()); tabletReq.version = transferReq.version; tabletReq.type = transferReq.type; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTsFileSealWithModReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTsFileSealWithModReq.java index 28959a1a0903c..7d0aa99cb13ea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTsFileSealWithModReq.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTsFileSealWithModReq.java @@ -25,7 +25,9 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.Map; public class PipeTransferTsFileSealWithModReq extends PipeTransferFileSealReqV2 { @@ -38,17 +40,59 @@ protected PipeRequestType getPlanType() { return PipeRequestType.TRANSFER_TS_FILE_SEAL_WITH_MOD; } + private static final String DATABASE_NAME_KEY_PREFIX = "DATABASE_NAME_"; + + public String getDatabaseNameByTsFileName() { + return getParameters() == null + ? null + : getParameters() + .get( + generateDatabaseNameWithFileNameKey(getFileNames().get(getFileNames().size() - 1))); + } + + private static String generateDatabaseNameWithFileNameKey(final String fileName) { + return DATABASE_NAME_KEY_PREFIX + fileName; + } + + private static Map generateDatabaseNameParameter( + final String tsFileName, final String dataBaseName) { + return dataBaseName == null + ? new HashMap<>() + : Collections.singletonMap(generateDatabaseNameWithFileNameKey(tsFileName), dataBaseName); + } + /////////////////////////////// Thrift /////////////////////////////// public static PipeTransferTsFileSealWithModReq toTPipeTransferReq( String modFileName, long modFileLength, String tsFileName, long tsFileLength) throws IOException { + return toTPipeTransferReq(modFileName, modFileLength, tsFileName, tsFileLength, null); + } + + public static PipeTransferTsFileSealWithModReq toTPipeTransferReq( + final String modFileName, + final long modFileLength, + final String tsFileName, + final long tsFileLength, + final String dataBaseName) + throws IOException { return (PipeTransferTsFileSealWithModReq) new PipeTransferTsFileSealWithModReq() .convertToTPipeTransferReq( Arrays.asList(modFileName, tsFileName), Arrays.asList(modFileLength, tsFileLength), - new HashMap<>()); + generateDatabaseNameParameter(tsFileName, dataBaseName)); + } + + public static PipeTransferTsFileSealWithModReq toTPipeTransferReq( + final String tsFileName, final long tsFileLength, final String dataBaseName) + throws IOException { + return (PipeTransferTsFileSealWithModReq) + new PipeTransferTsFileSealWithModReq() + .convertToTPipeTransferReq( + Collections.singletonList(tsFileName), + Collections.singletonList(tsFileLength), + generateDatabaseNameParameter(tsFileName, dataBaseName)); } public static PipeTransferTsFileSealWithModReq fromTPipeTransferReq(TPipeTransferReq req) { @@ -61,11 +105,31 @@ public static PipeTransferTsFileSealWithModReq fromTPipeTransferReq(TPipeTransfe public static byte[] toTPipeTransferBytes( String modFileName, long modFileLength, String tsFileName, long tsFileLength) throws IOException { + return toTPipeTransferBytes(modFileName, modFileLength, tsFileName, tsFileLength, null); + } + + public static byte[] toTPipeTransferBytes( + final String modFileName, + final long modFileLength, + final String tsFileName, + final long tsFileLength, + final String dataBaseName) + throws IOException { return new PipeTransferTsFileSealWithModReq() .convertToTPipeTransferSnapshotSealBytes( Arrays.asList(modFileName, tsFileName), Arrays.asList(modFileLength, tsFileLength), - new HashMap<>()); + generateDatabaseNameParameter(tsFileName, dataBaseName)); + } + + public static byte[] toTPipeTransferBytes( + final String tsFileName, final long tsFileLength, final String dataBaseName) + throws IOException { + return new PipeTransferTsFileSealWithModReq() + .convertToTPipeTransferSnapshotSealBytes( + Collections.singletonList(tsFileName), + Collections.singletonList(tsFileLength), + generateDatabaseNameParameter(tsFileName, dataBaseName)); } /////////////////////////////// Object /////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSink.java index 6396c710abd01..b2af96a37c5f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSink.java @@ -20,8 +20,10 @@ package org.apache.iotdb.db.pipe.sink.protocol.airgap; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.sink.limiter.TsFileSendRateLimiter; +import org.apache.iotdb.commons.utils.RetryUtils; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; import org.apache.iotdb.db.pipe.event.common.schema.PipeSchemaRegionWritePlanEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; @@ -30,6 +32,10 @@ import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.metric.overview.PipeResourceMetrics; import org.apache.iotdb.db.pipe.metric.sink.PipeDataRegionSinkMetrics; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.batch.PipeTabletEventBatch; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.batch.PipeTabletEventPlainBatch; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.batch.PipeTabletEventTsFileBatch; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.batch.PipeTransferBatchReqBuilder; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletInsertNodeReq; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletRawReq; import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTsFilePieceReq; @@ -38,6 +44,7 @@ import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTsFileSealWithModReq; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException; +import org.apache.iotdb.metrics.type.Histogram; import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.pipe.api.event.Event; @@ -46,13 +53,26 @@ import org.apache.iotdb.pipe.api.exception.PipeConnectionException; import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.commons.io.FileUtils; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.DataOutputStream; import java.io.File; import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.file.NoSuchFileException; import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.Objects; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_ENABLE_SEND_TSFILE_LIMIT; @@ -63,6 +83,7 @@ public class IoTDBDataRegionAirGapSink extends IoTDBDataNodeAirGapSink { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBDataRegionAirGapSink.class); + private PipeTransferBatchReqBuilder tabletBatchBuilder; private boolean enableSendTsFileLimit; @Override @@ -71,6 +92,10 @@ public void customize( throws Exception { super.customize(parameters, configuration); + if (isTabletBatchModeEnabled) { + tabletBatchBuilder = new PipeTransferBatchReqBuilder(parameters); + } + enableSendTsFileLimit = parameters.getBooleanOrDefault( Arrays.asList(SINK_ENABLE_SEND_TSFILE_LIMIT, CONNECTOR_ENABLE_SEND_TSFILE_LIMIT), @@ -97,7 +122,10 @@ public void transfer(final TabletInsertionEvent tabletInsertionEvent) throws Exc // When receiver encountered packet loss, the transfer will time out // We need to restore the transfer quickly by retry under this circumstance socket.setSoTimeout(PIPE_CONFIG.getPipeAirGapSinkTabletTimeoutMs()); - if (tabletInsertionEvent instanceof PipeInsertNodeTabletInsertionEvent) { + if (isTabletBatchModeEnabled) { + tabletBatchBuilder.onEvent(tabletInsertionEvent); + doTransferWrapper(socket); + } else if (tabletInsertionEvent instanceof PipeInsertNodeTabletInsertionEvent) { doTransferWrapper(socket, (PipeInsertNodeTabletInsertionEvent) tabletInsertionEvent); } else { doTransferWrapper(socket, (PipeRawTabletInsertionEvent) tabletInsertionEvent); @@ -136,6 +164,9 @@ public void transfer(final TsFileInsertionEvent tsFileInsertionEvent) throws Exc final AirGapSocket socket = sockets.get(socketIndex); try { + if (isTabletBatchModeEnabled && !tabletBatchBuilder.isEmpty()) { + doTransferWrapper(socket); + } doTransferWrapper(socket, (PipeTsFileInsertionEvent) tsFileInsertionEvent); } catch (final IOException e) { isSocketAlive.set(socketIndex, false); @@ -156,8 +187,18 @@ public void transfer(final Event event) throws Exception { try { if (event instanceof PipeSchemaRegionWritePlanEvent) { + if (isTabletBatchModeEnabled && !tabletBatchBuilder.isEmpty()) { + doTransferWrapper(socket); + } doTransferWrapper(socket, (PipeSchemaRegionWritePlanEvent) event); - } else if (!(event instanceof PipeHeartbeatEvent || event instanceof PipeTerminateEvent)) { + return; + } + + if (isTabletBatchModeEnabled && !tabletBatchBuilder.isEmpty()) { + doTransferWrapper(socket); + } + + if (!(event instanceof PipeHeartbeatEvent || event instanceof PipeTerminateEvent)) { LOGGER.warn( "IoTDBDataRegionAirGapConnector does not support transferring generic event: {}.", event); @@ -173,6 +214,63 @@ public void transfer(final Event event) throws Exception { } } + private void doTransferWrapper(final AirGapSocket socket) + throws IOException, WriteProcessException { + for (final Pair nonEmptyAndShouldEmitBatch : + tabletBatchBuilder.getAllNonEmptyAndShouldEmitBatches()) { + final PipeTabletEventBatch batch = nonEmptyAndShouldEmitBatch.getRight(); + if (batch instanceof PipeTabletEventPlainBatch) { + doTransfer(socket, (PipeTabletEventPlainBatch) batch); + } else if (batch instanceof PipeTabletEventTsFileBatch) { + doTransfer(socket, (PipeTabletEventTsFileBatch) batch); + } else { + LOGGER.warn("Unsupported batch type {}.", batch.getClass()); + } + batch.decreaseEventsReferenceCount(IoTDBDataRegionAirGapSink.class.getName(), true); + batch.onSuccess(); + } + } + + private void doTransfer( + final AirGapSocket socket, final PipeTabletEventPlainBatch batchToTransfer) + throws IOException { + if (!sendBatch( + socket, + toTPipeTransferBytes(batchToTransfer.toTPipeTransferReq()), + batchToTransfer.getPipe2BytesAccumulated())) { + final String errorMessage = + String.format("Transfer PipeTransferTabletBatchReq error. Socket: %s.", socket); + receiverStatusHandler.handle( + new TSStatus(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode()) + .setMessage(errorMessage), + errorMessage, + batchToTransfer.deepCopyEvents().toString()); + } + } + + private void doTransfer( + final AirGapSocket socket, final PipeTabletEventTsFileBatch batchToTransfer) + throws IOException, WriteProcessException { + final List sealedFiles = batchToTransfer.sealTsFiles(); + final Map, Double> pipe2WeightMap = batchToTransfer.deepCopyPipe2WeightMap(); + + for (final File tsFile : sealedFiles) { + doTransfer(pipe2WeightMap, socket, tsFile, null, null, tsFile.getName()); + try { + RetryUtils.retryOnException( + () -> { + FileUtils.delete(tsFile); + return null; + }); + } catch (final NoSuchFileException e) { + LOGGER.info("The file {} is not found, may already be deleted.", tsFile); + } catch (final Exception e) { + LOGGER.warn( + "Failed to delete batch file {}, this file should be deleted manually later", tsFile); + } + } + } + private void doTransferWrapper( final AirGapSocket socket, final PipeInsertNodeTabletInsertionEvent pipeInsertNodeTabletInsertionEvent) @@ -271,50 +369,157 @@ private void doTransferWrapper( private void doTransfer( final AirGapSocket socket, final PipeTsFileInsertionEvent pipeTsFileInsertionEvent) throws PipeException, IOException { - final String pipeName = pipeTsFileInsertionEvent.getPipeName(); - final long creationTime = pipeTsFileInsertionEvent.getCreationTime(); - final File tsFile = pipeTsFileInsertionEvent.getTsFile(); + doTransfer( + Collections.singletonMap( + new Pair<>( + pipeTsFileInsertionEvent.getPipeName(), pipeTsFileInsertionEvent.getCreationTime()), + 1.0), + socket, + pipeTsFileInsertionEvent.getTsFile(), + pipeTsFileInsertionEvent.isWithMod() && supportModsIfIsDataNodeReceiver + ? pipeTsFileInsertionEvent.getModFile() + : null, + pipeTsFileInsertionEvent.getDatabaseName(), + pipeTsFileInsertionEvent.toString()); + } + + private void doTransfer( + final Map, Double> pipe2WeightMap, + final AirGapSocket socket, + final File tsFile, + final File modFile, + final String dataBaseName, + final String receiverStatusContext) + throws PipeException, IOException { final String errorMessage = String.format("Seal file %s error. Socket %s.", tsFile, socket); - // 1. Transfer file piece by piece, and mod if needed - if (pipeTsFileInsertionEvent.isWithMod() && supportModsIfIsDataNodeReceiver) { - final File modFile = pipeTsFileInsertionEvent.getModFile(); - transferFilePieces(pipeName, creationTime, modFile, socket, true); - transferFilePieces(pipeName, creationTime, tsFile, socket, true); - // 2. Transfer file seal signal with mod, which means the file is transferred completely - if (!send( - pipeName, - creationTime, + if (Objects.nonNull(modFile)) { + transferFilePieces(pipe2WeightMap, modFile, socket, true); + transferFilePieces(pipe2WeightMap, tsFile, socket, true); + if (!sendWeighted( socket, PipeTransferTsFileSealWithModReq.toTPipeTransferBytes( - modFile.getName(), modFile.length(), tsFile.getName(), tsFile.length()))) { + modFile.getName(), modFile.length(), tsFile.getName(), tsFile.length(), dataBaseName), + pipe2WeightMap)) { receiverStatusHandler.handle( new TSStatus(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(errorMessage), errorMessage, - pipeTsFileInsertionEvent.toString()); + receiverStatusContext); } else { LOGGER.info("Successfully transferred file {}.", tsFile); } } else { - transferFilePieces(pipeName, creationTime, tsFile, socket, false); - // 2. Transfer file seal signal without mod, which means the file is transferred completely - if (!send( - pipeName, - creationTime, + transferFilePieces(pipe2WeightMap, tsFile, socket, false); + if (!sendWeighted( socket, - PipeTransferTsFileSealReq.toTPipeTransferBytes(tsFile.getName(), tsFile.length()))) { + dataBaseName == null + ? PipeTransferTsFileSealReq.toTPipeTransferBytes(tsFile.getName(), tsFile.length()) + : PipeTransferTsFileSealWithModReq.toTPipeTransferBytes( + tsFile.getName(), tsFile.length(), dataBaseName), + pipe2WeightMap)) { receiverStatusHandler.handle( new TSStatus(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(errorMessage), errorMessage, - pipeTsFileInsertionEvent.toString()); + receiverStatusContext); } else { LOGGER.info("Successfully transferred file {}.", tsFile); } } } + private void transferFilePieces( + final Map, Double> pipe2WeightMap, + final File file, + final AirGapSocket socket, + final boolean isMultiFile) + throws PipeException, IOException { + final int readFileBufferSize = PIPE_CONFIG.getPipeSinkReadFileBufferSize(); + final byte[] readBuffer = new byte[readFileBufferSize]; + long position = 0; + try (final RandomAccessFile reader = new RandomAccessFile(file, "r")) { + while (true) { + mayLimitRateAndRecordIO(readFileBufferSize); + final int readLength = reader.read(readBuffer); + if (readLength == -1) { + break; + } + + final byte[] payload = + readLength == readFileBufferSize + ? readBuffer + : Arrays.copyOfRange(readBuffer, 0, readLength); + if (!sendWeighted( + socket, + isMultiFile + ? getTransferMultiFilePieceBytes(file.getName(), position, payload) + : getTransferSingleFilePieceBytes(file.getName(), position, payload), + pipe2WeightMap)) { + final String errorMessage = + String.format("Transfer file %s error. Socket %s.", file, socket); + receiverStatusHandler.handle( + new TSStatus(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode()) + .setMessage(errorMessage), + errorMessage, + file.toString()); + } else { + position += readLength; + } + } + } + } + + private boolean sendBatch( + final AirGapSocket socket, + byte[] bytes, + final Map, Long> pipe2BytesAccumulated) + throws IOException { + final long uncompressedSize = bytes.length; + bytes = compressIfNeeded(bytes); + + final double compressionRatio = + uncompressedSize == 0 ? 1 : (double) bytes.length / uncompressedSize; + for (final Map.Entry, Long> entry : pipe2BytesAccumulated.entrySet()) { + rateLimitIfNeeded( + entry.getKey().getLeft(), + entry.getKey().getRight(), + socket.getEndPoint(), + (long) (entry.getValue() * compressionRatio)); + } + return sendBytes(socket, bytes); + } + + private boolean sendWeighted( + final AirGapSocket socket, byte[] bytes, final Map, Double> pipe2WeightMap) + throws IOException { + bytes = compressIfNeeded(bytes); + + for (final Map.Entry, Double> entry : pipe2WeightMap.entrySet()) { + rateLimitIfNeeded( + entry.getKey().getLeft(), + entry.getKey().getRight(), + socket.getEndPoint(), + (long) (bytes.length * entry.getValue())); + } + return sendBytes(socket, bytes); + } + + private byte[] toTPipeTransferBytes(final TPipeTransferReq req) throws IOException { + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(req.version, outputStream); + ReadWriteIOUtils.write(req.type, outputStream); + + final ByteBuffer bodyBuffer = req.body.duplicate(); + final byte[] body = new byte[bodyBuffer.remaining()]; + bodyBuffer.get(body); + outputStream.write(body); + + return byteArrayOutputStream.toByteArray(); + } + } + @Override protected void mayLimitRateAndRecordIO(final long requiredBytes) { PipeResourceMetrics.getInstance().recordDiskIO(requiredBytes); @@ -343,4 +548,65 @@ protected byte[] compressIfNeeded(final byte[] reqInBytes) throws IOException { } return super.compressIfNeeded(reqInBytes); } + + @Override + public synchronized void discardEventsOfPipe( + final String pipeNameToDrop, final long creationTimeToDrop, final int regionId) { + discardEventsOfPipe(new CommitterKey(pipeNameToDrop, creationTimeToDrop, regionId, -1)); + } + + @Override + public synchronized void discardEventsOfPipe(final CommitterKey committerKey) { + if (Objects.nonNull(tabletBatchBuilder)) { + tabletBatchBuilder.discardEventsOfPipe(committerKey); + } + } + + public int getBatchSize() { + return Objects.nonNull(tabletBatchBuilder) ? tabletBatchBuilder.size() : 0; + } + + @Override + public void close() { + if (tabletBatchBuilder != null) { + tabletBatchBuilder.close(); + } + + super.close(); + } + + @Override + public void setTabletBatchSizeHistogram(Histogram tabletBatchSizeHistogram) { + if (tabletBatchBuilder != null) { + tabletBatchBuilder.setTabletBatchSizeHistogram(tabletBatchSizeHistogram); + } + } + + @Override + public void setTsFileBatchSizeHistogram(Histogram tsFileBatchSizeHistogram) { + if (tabletBatchBuilder != null) { + tabletBatchBuilder.setTsFileBatchSizeHistogram(tsFileBatchSizeHistogram); + } + } + + @Override + public void setTabletBatchTimeIntervalHistogram(Histogram tabletBatchTimeIntervalHistogram) { + if (tabletBatchBuilder != null) { + tabletBatchBuilder.setTabletBatchTimeIntervalHistogram(tabletBatchTimeIntervalHistogram); + } + } + + @Override + public void setTsFileBatchTimeIntervalHistogram(Histogram tsFileBatchTimeIntervalHistogram) { + if (tabletBatchBuilder != null) { + tabletBatchBuilder.setTsFileBatchTimeIntervalHistogram(tsFileBatchTimeIntervalHistogram); + } + } + + @Override + public void setBatchEventSizeHistogram(Histogram eventSizeHistogram) { + if (tabletBatchBuilder != null) { + tabletBatchBuilder.setEventSizeHistogram(eventSizeHistogram); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBSchemaRegionAirGapSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBSchemaRegionAirGapSink.java index 8280709446a3b..6b7787ba16282 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBSchemaRegionAirGapSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBSchemaRegionAirGapSink.java @@ -47,13 +47,13 @@ public class IoTDBSchemaRegionAirGapSink extends IoTDBDataNodeAirGapSink { @Override public void transfer(final TabletInsertionEvent tabletInsertionEvent) throws Exception { throw new UnsupportedOperationException( - "IoTDBSchemaRegionAirGapConnector can't transfer TabletInsertionEvent."); + "IoTDBSchemaRegionAirGapSink can't transfer TabletInsertionEvent."); } @Override public void transfer(final TsFileInsertionEvent tsFileInsertionEvent) throws Exception { throw new UnsupportedOperationException( - "IoTDBSchemaRegionAirGapConnector can't transfer TsFileInsertionEvent."); + "IoTDBSchemaRegionAirGapSink can't transfer TsFileInsertionEvent."); } @Override @@ -68,8 +68,7 @@ public void transfer(final Event event) throws Exception { doTransferWrapper(socket, (PipeSchemaRegionSnapshotEvent) event); } else if (!(event instanceof PipeHeartbeatEvent)) { LOGGER.warn( - "IoTDBSchemaRegionAirGapConnector does not support transferring generic event: {}.", - event); + "IoTDBSchemaRegionAirGapSink does not support transferring generic event: {}.", event); } } catch (final IOException e) { isSocketAlive.set(socketIndex, false); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/legacy/IoTDBLegacyPipeSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/legacy/IoTDBLegacyPipeSink.java index 4328c758d3934..0a090f99b8df6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/legacy/IoTDBLegacyPipeSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/legacy/IoTDBLegacyPipeSink.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.client.property.ThriftClientProperty; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.exception.pipe.PipeRuntimeCriticalException; import org.apache.iotdb.commons.pipe.config.PipeConfig; @@ -50,6 +51,9 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; +import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; +import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion; import org.apache.iotdb.service.rpc.thrift.TSyncIdentityInfo; import org.apache.iotdb.service.rpc.thrift.TSyncTransportMetaInfo; import org.apache.iotdb.session.pool.SessionPool; @@ -64,6 +68,7 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; +import java.time.ZoneId; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -224,6 +229,7 @@ public void handshake() throws Exception { useSSL, trustStore, trustStorePwd); + openClientSession(); final TSyncIdentityInfo identityInfo = new TSyncIdentityInfo( pipeName, System.currentTimeMillis(), syncConnectorVersion, databaseName); @@ -254,6 +260,26 @@ public void handshake() throws Exception { .build(); } + private void openClientSession() throws TException { + final TSOpenSessionReq openSessionReq = new TSOpenSessionReq(); + openSessionReq.setClient_protocol(TSProtocolVersion.IOTDB_SERVICE_PROTOCOL_V3); + openSessionReq.setUsername(user); + openSessionReq.setPassword(password); + openSessionReq.setZoneId(ZoneId.systemDefault().toString()); + openSessionReq.putToConfiguration("version", IoTDBConstant.ClientVersion.V_1_0.toString()); + openSessionReq.putToConfiguration("sql_dialect", "tree"); + + final TSOpenSessionResp openSessionResp = client.openSession(openSessionReq); + if (openSessionResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + final String errorMsg = + String.format( + "Failed to login to receiver %s:%s for legacy pipe transfer because %s", + ipAddress, port, openSessionResp.getStatus().getMessage()); + LOGGER.warn(errorMsg); + throw new PipeRuntimeCriticalException(errorMsg); + } + } + @Override public void heartbeat() throws Exception { // do nothing diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/OpcUaSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/OpcUaSink.java index e198506122474..654395efe8d7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/OpcUaSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/OpcUaSink.java @@ -63,6 +63,8 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_IOTDB_USERNAME_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_IOTDB_USER_DEFAULT_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_IOTDB_USER_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_OPC_UA_DEBOUNCE_TIME_MS_DEFAULT_VALUE; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_OPC_UA_DEBOUNCE_TIME_MS_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_OPC_UA_DEFAULT_QUALITY_BAD_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_OPC_UA_DEFAULT_QUALITY_GOOD_VALUE; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.CONNECTOR_OPC_UA_DEFAULT_QUALITY_KEY; @@ -101,6 +103,7 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.SINK_IOTDB_PASSWORD_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.SINK_IOTDB_USERNAME_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.SINK_IOTDB_USER_KEY; +import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.SINK_OPC_UA_DEBOUNCE_TIME_MS_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.SINK_OPC_UA_DEFAULT_QUALITY_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.SINK_OPC_UA_ENABLE_ANONYMOUS_ACCESS_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.SINK_OPC_UA_HISTORIZING_KEY; @@ -278,6 +281,10 @@ private void customizeServer(final PipeParameters parameters) { if (securityPolicies.isEmpty()) { throw new PipeException("The security policy cannot be empty."); } + final long debounceTimeMs = + parameters.getLongOrDefault( + Arrays.asList(CONNECTOR_OPC_UA_DEBOUNCE_TIME_MS_KEY, SINK_OPC_UA_DEBOUNCE_TIME_MS_KEY), + CONNECTOR_OPC_UA_DEBOUNCE_TIME_MS_DEFAULT_VALUE); synchronized (SERVER_KEY_TO_REFERENCE_COUNT_AND_NAME_SPACE_MAP) { serverKey = httpsBindPort + ":" + tcpBindPort; @@ -297,7 +304,8 @@ private void customizeServer(final PipeParameters parameters) { .setPassword(password) .setSecurityDir(securityDir) .setEnableAnonymousAccess(enableAnonymousAccess) - .setSecurityPolicies(securityPolicies); + .setSecurityPolicies(securityPolicies) + .setDebounceTimeMs(debounceTimeMs); final OpcUaServer newServer = builder.build(); nameSpace = new OpcUaNameSpace(newServer, builder); nameSpace.startup(); @@ -311,7 +319,8 @@ private void customizeServer(final PipeParameters parameters) { password, securityDir, enableAnonymousAccess, - securityPolicies); + securityPolicies, + debounceTimeMs); return oldValue; } } catch (final PipeException e) { @@ -518,42 +527,38 @@ public interface ThrowingConsumer { @Override public void close() throws Exception { - if (serverKey == null) { - return; - } - - synchronized (SERVER_KEY_TO_REFERENCE_COUNT_AND_NAME_SPACE_MAP) { - final Pair pair = - SERVER_KEY_TO_REFERENCE_COUNT_AND_NAME_SPACE_MAP.get(serverKey); - if (pair == null) { - return; - } + if (serverKey != null) { + synchronized (SERVER_KEY_TO_REFERENCE_COUNT_AND_NAME_SPACE_MAP) { + final Pair pair = + SERVER_KEY_TO_REFERENCE_COUNT_AND_NAME_SPACE_MAP.get(serverKey); + if (pair == null) { + return; + } - if (pair.getLeft().decrementAndGet() <= 0) { - try { - pair.getRight().shutdown(); - } finally { - SERVER_KEY_TO_REFERENCE_COUNT_AND_NAME_SPACE_MAP.remove(serverKey); + if (pair.getLeft().decrementAndGet() <= 0) { + try { + pair.getRight().shutdown(); + } finally { + SERVER_KEY_TO_REFERENCE_COUNT_AND_NAME_SPACE_MAP.remove(serverKey); + } } } } - if (nodeUrl == null) { - return; - } - - synchronized (CLIENT_KEY_TO_REFERENCE_COUNT_AND_CLIENT_MAP) { - final Pair pair = - CLIENT_KEY_TO_REFERENCE_COUNT_AND_CLIENT_MAP.get(nodeUrl); - if (pair == null) { - return; - } + if (nodeUrl != null) { + synchronized (CLIENT_KEY_TO_REFERENCE_COUNT_AND_CLIENT_MAP) { + final Pair pair = + CLIENT_KEY_TO_REFERENCE_COUNT_AND_CLIENT_MAP.get(nodeUrl); + if (pair == null) { + return; + } - if (pair.getLeft().decrementAndGet() <= 0) { - try { - pair.getRight().disconnect(); - } finally { - CLIENT_KEY_TO_REFERENCE_COUNT_AND_CLIENT_MAP.remove(nodeUrl); + if (pair.getLeft().decrementAndGet() <= 0) { + try { + pair.getRight().disconnect(); + } finally { + CLIENT_KEY_TO_REFERENCE_COUNT_AND_CLIENT_MAP.remove(nodeUrl); + } } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/client/IoTDBOpcUaClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/client/IoTDBOpcUaClient.java index 2019c0fe83336..7595b75747e1e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/client/IoTDBOpcUaClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/client/IoTDBOpcUaClient.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.pipe.sink.protocol.opcua.server.OpcUaNameSpace; import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -56,9 +57,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.concurrent.ExecutionException; @@ -130,7 +132,6 @@ private void transferTabletRowForClientServerModel( StatusCode currentQuality = sink.getDefaultQuality(); Object value = null; long timestamp = 0; - NodeId nodeId = null; NodeId opcDataType = null; for (int i = 0; i < measurementSchemas.size(); ++i) { @@ -153,17 +154,43 @@ private void transferTabletRowForClientServerModel( "When the 'with-quality' mode is enabled, the measurement must be either \"value-name\" or \"quality-name\""); continue; } - nodeId = new NodeId(NAME_SPACE_INDEX, String.join("/", segments)); final long utcTimestamp = timestampToUtc(timestamps.get(timestamps.size() > 1 ? i : 0)); - value = values.get(i); - timestamp = utcTimestamp; - opcDataType = convertToOpcDataType(type); + if (Objects.isNull(sink.getValueName())) { + writeValue( + values.get(i), + utcTimestamp, + convertToOpcDataType(type), + currentQuality, + segments, + name); + } else { + value = values.get(i); + timestamp = utcTimestamp; + opcDataType = convertToOpcDataType(type); + } } if (Objects.isNull(value)) { return; } + writeValue(value, timestamp, opcDataType, currentQuality, segments, null); + } + + private void writeValue( + final Object value, + final long timestamp, + final NodeId opcDataType, + final StatusCode currentQuality, + final String[] segments, + final @Nullable String name) + throws Exception { + final NodeId nodeId = + new NodeId( + NAME_SPACE_INDEX, + Objects.nonNull(name) + ? String.join("/", segments) + "/" + name + : String.join("/", segments)); final Variant variant = new Variant(value); final DataValue dataValue = new DataValue(variant, currentQuality, new DateTime(timestamp), new DateTime()); @@ -171,36 +198,41 @@ private void transferTabletRowForClientServerModel( if (writeStatus.getValue() == StatusCodes.Bad_NodeIdUnknown) { final AddNodesResponse addStatus = - client.addNodes(getNodesToAdd(segments, opcDataType, variant)).get(); + client.addNodes(getNodesToAdd(segments, name, opcDataType, variant)).get(); for (final AddNodesResult result : addStatus.getResults()) { if (!result.getStatusCode().equals(StatusCode.GOOD) && !(result.getStatusCode().getValue() == StatusCodes.Bad_NodeIdExists)) { throw new PipeException( "Failed to create nodes after transfer data value, creation status: " + addStatus - + getErrorString(segments, opcDataType, value, writeStatus)); + + getErrorString(segments, name, opcDataType, value, writeStatus)); } } writeStatus = client.writeValue(nodeId, dataValue).get(); if (writeStatus.getValue() != StatusCode.GOOD.getValue()) { throw new PipeException( "Failed to transfer dataValue after successfully created nodes" - + getErrorString(segments, opcDataType, value, writeStatus)); + + getErrorString(segments, name, opcDataType, value, writeStatus)); } } else if (writeStatus.getValue() != StatusCode.GOOD.getValue()) { throw new PipeException( "Failed to transfer dataValue" - + getErrorString(segments, opcDataType, value, writeStatus)); + + getErrorString(segments, name, opcDataType, value, writeStatus)); } } private static String getErrorString( final String[] segments, + final @Nullable String name, final NodeId dataType, final Object value, final StatusCode writeStatus) { - return ", segments: " - + Arrays.toString(segments) + return ", measurement: " + + (Objects.nonNull(name) + ? String.join(TsFileConstant.PATH_SEPARATOR, segments) + + TsFileConstant.PATH_SEPARATOR + + name + : String.join(TsFileConstant.PATH_SEPARATOR, segments)) + ", dataType: " + dataType + ", value: " @@ -210,7 +242,10 @@ private static String getErrorString( } public List getNodesToAdd( - final String[] segments, final NodeId opcDataType, final Variant initialValue) { + final String[] segments, + final @Nullable String name, + final NodeId opcDataType, + final Variant initialValue) { final List addNodesItems = new ArrayList<>(); final StringBuilder sb = new StringBuilder(segments[0]); ExpandedNodeId curNodeId = new NodeId(NAME_SPACE_INDEX, segments[0]).expanded(); @@ -226,7 +261,7 @@ public List getNodesToAdd( Identifiers.FolderType.expanded())); // segments.length >= 3 - for (int i = 1; i < segments.length - 1; ++i) { + for (int i = 1; i < (Objects.nonNull(name) ? segments.length : segments.length - 1); ++i) { sb.append("/").append(segments[i]); final ExpandedNodeId nextId = new NodeId(NAME_SPACE_INDEX, sb.toString()).expanded(); addNodesItems.add( @@ -242,7 +277,7 @@ public List getNodesToAdd( curNodeId = nextId; } - final String measurementName = segments[segments.length - 1]; + final String measurementName = Objects.nonNull(name) ? name : segments[segments.length - 1]; sb.append("/").append(measurementName); addNodesItems.add( new AddNodesItem( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaNameSpace.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaNameSpace.java index 43144cbdd15b5..713a87b2e350f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaNameSpace.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaNameSpace.java @@ -45,7 +45,7 @@ import org.eclipse.milo.opcua.sdk.server.nodes.UaFolderNode; import org.eclipse.milo.opcua.sdk.server.nodes.UaNode; import org.eclipse.milo.opcua.sdk.server.nodes.UaVariableNode; -import org.eclipse.milo.opcua.sdk.server.util.SubscriptionModel; +import org.eclipse.milo.opcua.stack.core.AttributeId; import org.eclipse.milo.opcua.stack.core.Identifiers; import org.eclipse.milo.opcua.stack.core.UaException; import org.eclipse.milo.opcua.stack.core.security.SecurityPolicy; @@ -55,6 +55,7 @@ import org.eclipse.milo.opcua.stack.core.types.builtin.NodeId; import org.eclipse.milo.opcua.stack.core.types.builtin.StatusCode; import org.eclipse.milo.opcua.stack.core.types.builtin.Variant; +import org.eclipse.milo.opcua.stack.core.types.structured.ReadValueId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,19 +69,33 @@ import java.util.Objects; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; public class OpcUaNameSpace extends ManagedNamespaceWithLifecycle { private static final Logger LOGGER = LoggerFactory.getLogger(OpcUaNameSpace.class); public static final String NAMESPACE_URI = "urn:apache:iotdb:opc-server"; - private final SubscriptionModel subscriptionModel; private final OpcUaServerBuilder builder; + // Do not use subscription model because the original subscription model has some bugs + private final ConcurrentMap> nodeSubscriptions = new ConcurrentHashMap<>(); + + // Debounce task cache: used to merge updates within a short period of time, avoiding unnecessary + // duplicate pushes + private final ConcurrentMap> debounceTasks = new ConcurrentHashMap<>(); + // Debounce interval: within 10ms, the same node is updated multiple times, and only the last one + // will be pushed (can be adjusted according to your site delay requirements, the minimum can be + // set to 1ms) + private final long debounceIntervalMs; + public OpcUaNameSpace(final OpcUaServer server, final OpcUaServerBuilder builder) { super(server, NAMESPACE_URI); this.builder = builder; + debounceIntervalMs = builder.getDebounceTimeMs(); - subscriptionModel = new SubscriptionModel(server, this); - getLifecycleManager().addLifecycle(subscriptionModel); getLifecycleManager() .addLifecycle( new Lifecycle() { @@ -245,7 +260,7 @@ private void transferTabletRowForClientServerModel( measurementNode = new UaVariableNode.UaVariableNodeBuilder(getNodeContext()) .setNodeId(nodeId) - .setAccessLevel(AccessLevel.READ_WRITE) + .setAccessLevel(AccessLevel.READ_ONLY) .setUserAccessLevel(AccessLevel.READ_ONLY) .setBrowseName(newQualifiedName(nodeName)) .setDisplayName(LocalizedText.english(nodeName)) @@ -279,7 +294,7 @@ private void transferTabletRowForClientServerModel( if (Objects.isNull(measurementNode.getValue()) || Objects.isNull(measurementNode.getValue().getSourceTime()) || measurementNode.getValue().getSourceTime().getUtcTime() < utcTimestamp) { - measurementNode.setValue(dataValue); + notifyNodeValueChange(nodeId, dataValue, measurementNode); } } else { valueNode = measurementNode; @@ -291,9 +306,11 @@ private void transferTabletRowForClientServerModel( if (Objects.isNull(valueNode.getValue()) || Objects.isNull(valueNode.getValue().getSourceTime()) || valueNode.getValue().getSourceTime().getUtcTime() < timestamp) { - valueNode.setValue( + notifyNodeValueChange( + valueNode.getNodeId(), new DataValue( - new Variant(value), currentQuality, new DateTime(timestamp), new DateTime())); + new Variant(value), currentQuality, new DateTime(timestamp), new DateTime()), + valueNode); } } } @@ -357,7 +374,10 @@ private void transferTabletForPubSubModel(final Tablet tablet) throws UaExceptio for (int rowIndex = 0; rowIndex < tablet.rowSize; ++rowIndex) { // Filter null value - if (tablet.bitMaps[columnIndex].isMarked(rowIndex)) { + if (tablet.bitMaps != null + && columnIndex < tablet.bitMaps.length + && tablet.bitMaps[columnIndex] != null + && tablet.bitMaps[columnIndex].isMarked(rowIndex)) { continue; } @@ -451,24 +471,131 @@ public static NodeId convertToOpcDataType(final TSDataType type) { } } + /** + * On point value changing, notify all subscribed clients proactively + * + * @param nodeId NodeId of the changing node + * @param newValue New value of the node (DataValue object containing value, status code, and + * timestamp) + * @param variableNode Corresponding UaVariableNode instance, used to update the local cached + * value of the node + */ + public void notifyNodeValueChange( + NodeId nodeId, DataValue newValue, UaVariableNode variableNode) { + // 1. Update the local cached value of the node + variableNode.setValue(newValue); + + // 2. If there are no subscribers, return directly without doing any extra operations + List subscribedItems = nodeSubscriptions.get(nodeId); + if (subscribedItems == null || subscribedItems.isEmpty()) { + return; + } + + // 2. Debounce+Async Push: Asynchronously push the expensive push operation, while merging + // high-frequency repeated updates + debounceTasks.compute( + nodeId, + (k, oldTask) -> { + // If there is already a pending push task, cancel it, we only need the latest value + if (oldTask != null && !oldTask.isDone()) { + oldTask.cancel(false); + } + + // Submit the push task to the Milo's scheduled thread pool, delay DEBOUNCE_INTERVAL_MS + // execution + return getServer() + .getScheduledExecutorService() + .schedule( + () -> { + try { + // Batch push changes to all subscribers, this time-consuming operation is put + // into the thread pool, not blocking your data update thread + for (DataItem item : subscribedItems) { + try { + item.setValue(newValue); + } catch (Exception e) { + // Single client push failure does not affect other clients + LOGGER.warn( + "Failed to push value change to client, nodeId={}", nodeId, e); + } + } + } finally { + // Task execution completed, clean up the debounce cache + debounceTasks.remove(nodeId); + } + }, + debounceIntervalMs, + TimeUnit.MILLISECONDS); + }); + } + @Override public void onDataItemsCreated(final List dataItems) { - subscriptionModel.onDataItemsCreated(dataItems); + for (DataItem item : dataItems) { + final ReadValueId readValueId = item.getReadValueId(); + // Only handle Value attribute subscription (align with the original SubscriptionModel logic, + // ignore other attribute subscriptions) + if (!AttributeId.Value.isEqual(readValueId.getAttributeId())) { + continue; + } + final NodeId nodeId = readValueId.getNodeId(); + + // 1. Add the new subscription item to the subscription mapping + nodeSubscriptions.compute( + nodeId, + (k, existingList) -> { + List list = + existingList != null ? existingList : new CopyOnWriteArrayList<>(); + list.add(item); + return list; + }); + + // 2. 【Key Optimization】Proactively push the current node's initial value when the new + // subscription item is created + // Eliminate Bad_WaitingForInitialData, no need to wait for any polling + try { + UaVariableNode node = (UaVariableNode) getNodeManager().getNode(nodeId).orElse(null); + if (node != null && node.getValue() != null) { + // Immediately push the current value to the new subscriber, the client will instantly be + // able to get the initial data + item.setValue(node.getValue()); + } + } catch (Exception e) { + LOGGER.warn("Failed to send initial value to new subscription, nodeId={}", nodeId, e); + } + } } @Override public void onDataItemsModified(final List dataItems) { - subscriptionModel.onDataItemsModified(dataItems); + // Push mode, client modifies subscription parameters (e.g. sampling interval) has no effect on + // our active push, no additional processing is needed } @Override public void onDataItemsDeleted(final List dataItems) { - subscriptionModel.onDataItemsDeleted(dataItems); + for (DataItem item : dataItems) { + final ReadValueId readValueId = item.getReadValueId(); + if (!AttributeId.Value.isEqual(readValueId.getAttributeId())) { + continue; + } + final NodeId nodeId = readValueId.getNodeId(); + + // When the client cancels the subscription, remove this subscription item from the mapping + nodeSubscriptions.computeIfPresent( + nodeId, + (k, existingList) -> { + existingList.remove(item); + // Automatically clean up the key when there are no subscribers, save memory + return existingList.isEmpty() ? null : existingList; + }); + } } @Override public void onMonitoringModeChanged(final List monitoredItems) { - subscriptionModel.onMonitoringModeChanged(monitoredItems); + // Push mode, monitoring mode change has no effect on active push, no additional processing is + // needed } /////////////////////////////// Conflict detection /////////////////////////////// @@ -478,8 +605,14 @@ public void checkEquals( final String password, final String securityDir, final boolean enableAnonymousAccess, - final Set securityPolicies) { + final Set securityPolicies, + final long debounceTimeMs) { builder.checkEquals( - user, password, Paths.get(securityDir), enableAnonymousAccess, securityPolicies); + user, + password, + Paths.get(securityDir), + enableAnonymousAccess, + securityPolicies, + debounceTimeMs); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaServerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaServerBuilder.java index f029031b6175b..281d6eae77edd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaServerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcua/server/OpcUaServerBuilder.java @@ -86,6 +86,7 @@ public class OpcUaServerBuilder implements Closeable { private boolean enableAnonymousAccess; private Set securityPolicies; private DefaultTrustListManager trustListManager; + private long debounceTimeMs; public OpcUaServerBuilder setTcpBindPort(final int tcpBindPort) { this.tcpBindPort = tcpBindPort; @@ -123,6 +124,15 @@ public OpcUaServerBuilder setSecurityPolicies(final Set security return this; } + public OpcUaServerBuilder setDebounceTimeMs(long debounceTimeMs) { + this.debounceTimeMs = debounceTimeMs; + return this; + } + + public long getDebounceTimeMs() { + return debounceTimeMs; + } + public OpcUaServer build() throws Exception { Files.createDirectories(securityDir); if (!Files.exists(securityDir)) { @@ -314,7 +324,8 @@ void checkEquals( final String password, final Path securityDir, final boolean enableAnonymousAccess, - final Set securityPolicies) { + final Set securityPolicies, + final long debounceTimeMs) { checkEquals("user", this.user, user); checkEquals("password", this.password, password); checkEquals( @@ -323,6 +334,7 @@ void checkEquals( FileSystems.getDefault().getPath(securityDir.toAbsolutePath().toString())); checkEquals("enableAnonymousAccess option", this.enableAnonymousAccess, enableAnonymousAccess); checkEquals("securityPolicies", this.securityPolicies, securityPolicies); + checkEquals("debounceTimeMs", this.debounceTimeMs, debounceTimeMs); } private void checkEquals(final String attrName, Object thisAttr, Object thatAttr) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/pipeconsensus/handler/PipeConsensusTabletBatchEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/pipeconsensus/handler/PipeConsensusTabletBatchEventHandler.java index 56d2c9ad0626f..32481f43bef1b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/pipeconsensus/handler/PipeConsensusTabletBatchEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/pipeconsensus/handler/PipeConsensusTabletBatchEventHandler.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.async.AsyncPipeConsensusServiceClient; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; +import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusBatchTransferReq; import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusBatchTransferResp; import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferResp; @@ -116,17 +117,20 @@ public void onComplete(final TPipeConsensusBatchTransferResp response) { @Override public void onError(final Exception exception) { - LOGGER.warn( - "PipeConsensus: Failed to transfer TabletInsertionEvent batch. Total failed events: {}, related pipe names: {}", - events.size(), + final Object pipeNames = events.stream() .map( event -> event instanceof EnrichedEvent ? ((EnrichedEvent) event).getPipeName() : "UNKNOWN") - .collect(Collectors.toSet()), - exception); + .collect(Collectors.toSet()); + PipeLogger.log( + LOGGER::warn, + exception, + "PipeConsensus: Failed to transfer TabletInsertionEvent batch. Total failed events: %s, related pipe names: %s", + events.size(), + pipeNames); connector.addFailureEventsToRetryQueue(events); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java index 9d027e711f3f4..91c083fe3c626 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/pipeconsensus/handler/PipeConsensusTabletInsertionEventHandler.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.async.AsyncPipeConsensusServiceClient; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; +import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferReq; import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferResp; import org.apache.iotdb.db.pipe.consensus.PipeConsensusSinkMetrics; @@ -106,14 +107,21 @@ public void onComplete(TPipeConsensusTransferResp response) { @Override public void onError(Exception exception) { - LOGGER.warn( - "Failed to transfer TabletInsertionEvent {} (committer key={}, commit id={}).", + final Object eventReportMessage = event instanceof EnrichedEvent ? ((EnrichedEvent) event).coreReportMessage() - : event.toString(), - event instanceof EnrichedEvent ? ((EnrichedEvent) event).getCommitterKey() : null, - event instanceof EnrichedEvent ? ((EnrichedEvent) event).getCommitId() : null, - exception); + : event.toString(); + final Object committerKey = + event instanceof EnrichedEvent ? ((EnrichedEvent) event).getCommitterKey() : null; + final Object commitId = + event instanceof EnrichedEvent ? ((EnrichedEvent) event).getCommitId() : null; + PipeLogger.log( + LOGGER::warn, + exception, + "Failed to transfer TabletInsertionEvent %s (committer key=%s, commit id=%s).", + eventReportMessage, + committerKey, + commitId); connector.addFailureEventToRetryQueue(event); metric.recordRetryCounter(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/pipeconsensus/handler/PipeConsensusTsFileInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/pipeconsensus/handler/PipeConsensusTsFileInsertionEventHandler.java index 22b55239e197d..3496c0cf85755 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/pipeconsensus/handler/PipeConsensusTsFileInsertionEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/pipeconsensus/handler/PipeConsensusTsFileInsertionEventHandler.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.async.AsyncPipeConsensusServiceClient; import org.apache.iotdb.commons.pipe.config.PipeConfig; +import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; import org.apache.iotdb.commons.pipe.sink.payload.pipeconsensus.response.PipeConsensusTransferFilePieceResp; import org.apache.iotdb.consensus.pipe.thrift.TCommitId; import org.apache.iotdb.consensus.pipe.thrift.TPipeConsensusTransferResp; @@ -273,12 +274,13 @@ public void onComplete(final TPipeConsensusTransferResp response) { @Override public void onError(final Exception exception) { - LOGGER.warn( - "Failed to transfer TsFileInsertionEvent {} (committer key {}, commit id {}).", + PipeLogger.log( + LOGGER::warn, + exception, + "Failed to transfer TsFileInsertionEvent %s (committer key %s, commit id %s).", tsFile, event.getCommitterKey(), - event.getCommitId(), - exception); + event.getCommitId()); try { if (reader != null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java index 07381d15e28b6..b61cb4543c74e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.client.ThriftClient; import org.apache.iotdb.commons.client.async.AsyncPipeDataTransferServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; @@ -74,6 +75,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; @@ -121,6 +123,8 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink { private final Map pendingHandlers = new ConcurrentHashMap<>(); + private final Set droppedPipeTaskKeys = ConcurrentHashMap.newKeySet(); + private boolean enableSendTsFileLimit; private volatile boolean isConnectionException; @@ -251,10 +255,11 @@ private void transferInBatchWithoutCheck( eventsHadBeenAddedToRetryQueue, sealedFile, null, - false)); + false, + null)); } } catch (final Exception e) { - LOGGER.warn("Failed to transfer tsfile batch ({}).", sealedFiles, e); + PipeLogger.log(LOGGER::warn, e, "Failed to transfer tsfile batch (%s).", sealedFiles); if (eventsHadBeenAddedToRetryQueue.compareAndSet(false, true)) { addFailureEventsToRetryQueue(events, e); } @@ -396,7 +401,8 @@ private boolean transferWithoutCheck(final TsFileInsertionEvent tsFileInsertionE pipeTsFileInsertionEvent.getTsFile(), pipeTsFileInsertionEvent.getModFile(), pipeTsFileInsertionEvent.isWithMod() - && clientManager.supportModsIfIsDataNodeReceiver()); + && clientManager.supportModsIfIsDataNodeReceiver(), + pipeTsFileInsertionEvent.getDatabaseName()); transfer(pipeTransferTsFileHandler); return true; @@ -427,23 +433,25 @@ private void transfer(final PipeTransferTsFileHandler pipeTransferTsFileHandler) }, transferTsFileClientManager.getExecutor()); - if (PipeConfig.getInstance().isTransferTsFileSync() || !isRealtimeFirst) { + if (PipeConfig.getInstance().isTransferTsFileSync()) { try { completableFuture.get(); } catch (final Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); - LOGGER.warn( - "Transfer tsfile event {} asynchronously was interrupted.", - pipeTransferTsFileHandler.getTsFile(), - e); + PipeLogger.log( + LOGGER::warn, + e, + "Transfer tsfile event %s asynchronously was interrupted.", + pipeTransferTsFileHandler.getTsFile()); } pipeTransferTsFileHandler.onError(e); - LOGGER.warn( - "Failed to transfer tsfile event {} asynchronously.", - pipeTransferTsFileHandler.getTsFile(), - e); + PipeLogger.log( + LOGGER::warn, + e, + "Failed to transfer tsfile event %s asynchronously.", + pipeTransferTsFileHandler.getTsFile()); } } } @@ -660,8 +668,15 @@ private void retryTransfer(final PipeTsFileInsertionEvent tsFileInsertionEvent) public void addFailureEventToRetryQueue(final Event event, final Exception e) { isConnectionException = e instanceof PipeConnectionException || ThriftClient.isConnectionBroken(e); - if (event instanceof EnrichedEvent && ((EnrichedEvent) event).isReleased()) { - return; + if (event instanceof EnrichedEvent) { + final EnrichedEvent enrichedEvent = (EnrichedEvent) event; + if (enrichedEvent.isReleased()) { + return; + } + if (isDroppedPipe(enrichedEvent)) { + enrichedEvent.clearReferenceCount(IoTDBDataRegionAsyncSink.class.getName()); + return; + } } if (isClosed.get()) { @@ -707,15 +722,22 @@ public boolean isEnableSendTsFileLimit() { //////////////////////////// Operations for close //////////////////////////// @Override - public synchronized void discardEventsOfPipe(final String pipeNameToDrop, final int regionId) { - if (isTabletBatchModeEnabled) { - tabletBatchBuilder.discardEventsOfPipe(pipeNameToDrop, regionId); + public synchronized void discardEventsOfPipe( + final String pipeNameToDrop, final long creationTimeToDrop, final int regionId) { + discardEventsOfPipe(new CommitterKey(pipeNameToDrop, creationTimeToDrop, regionId, -1)); + } + + @Override + public synchronized void discardEventsOfPipe(final CommitterKey committerKey) { + droppedPipeTaskKeys.add(committerKey); + + if (isTabletBatchModeEnabled && Objects.nonNull(tabletBatchBuilder)) { + tabletBatchBuilder.discardEventsOfPipe(committerKey); } retryEventQueue.removeIf( event -> { if (event instanceof EnrichedEvent - && pipeNameToDrop.equals(((EnrichedEvent) event).getPipeName()) - && regionId == ((EnrichedEvent) event).getRegionId()) { + && isDroppedPipe((EnrichedEvent) event, committerKey)) { ((EnrichedEvent) event).clearReferenceCount(IoTDBDataRegionAsyncSink.class.getName()); retryEventQueueEventCounter.decreaseEventCount(event); return true; @@ -726,8 +748,7 @@ public synchronized void discardEventsOfPipe(final String pipeNameToDrop, final retryTsFileQueue.removeIf( event -> { if (event instanceof EnrichedEvent - && pipeNameToDrop.equals(((EnrichedEvent) event).getPipeName()) - && regionId == ((EnrichedEvent) event).getRegionId()) { + && isDroppedPipe((EnrichedEvent) event, committerKey)) { ((EnrichedEvent) event).clearReferenceCount(IoTDBDataRegionAsyncSink.class.getName()); retryEventQueueEventCounter.decreaseEventCount(event); return true; @@ -771,6 +792,7 @@ public synchronized void close() { // clear reference count of events in retry queue after closing async client clearRetryEventsReferenceCount(); + droppedPipeTaskKeys.clear(); super.close(); } @@ -827,6 +849,17 @@ public void setTransferTsFileCounter(AtomicInteger transferTsFileCounter) { this.transferTsFileCounter = transferTsFileCounter; } + private boolean isDroppedPipe(final EnrichedEvent event) { + return droppedPipeTaskKeys.stream().anyMatch(key -> isDroppedPipe(event, key)); + } + + private static boolean isDroppedPipe(final EnrichedEvent event, final CommitterKey committerKey) { + return committerKey.getPipeName().equals(event.getPipeName()) + && committerKey.getCreationTime() == event.getCreationTime() + && committerKey.getRegionId() == event.getRegionId() + && (committerKey.getRestartTimes() < 0 || committerKey.equals(event.getCommitterKey())); + } + @Override public void setTabletBatchSizeHistogram(Histogram tabletBatchSizeHistogram) { if (tabletBatchBuilder != null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletBatchEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletBatchEventHandler.java index 110d3cb645027..d5ee15c6367ba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletBatchEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletBatchEventHandler.java @@ -69,7 +69,7 @@ public PipeTransferTabletBatchEventHandler( public void transfer(final AsyncPipeDataTransferServiceClient client) throws TException { for (final Map.Entry, Long> entry : pipeName2BytesAccumulated.entrySet()) { - connector.rateLimitIfNeeded( + sink.rateLimitIfNeeded( entry.getKey().getLeft(), entry.getKey().getRight(), client.getEndPoint(), @@ -92,13 +92,11 @@ protected boolean onCompleteInternal(final TPipeTransferResp response) { // Only handle the failed statuses to avoid string format performance overhead if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { - connector - .statusHandler() - .handle(status, response.getStatus().getMessage(), events.toString()); + sink.statusHandler().handle(status, response.getStatus().getMessage(), events.toString()); } for (final Pair redirectPair : LeaderCacheUtils.parseRecommendedRedirections(status)) { - connector.updateLeaderCache(redirectPair.getLeft(), redirectPair.getRight()); + sink.updateLeaderCache(redirectPair.getLeft(), redirectPair.getRight()); } events.forEach( @@ -123,7 +121,7 @@ protected void onErrorInternal(final Exception exception) { events.size(), events.stream().map(EnrichedEvent::getPipeName).collect(Collectors.toSet())); } finally { - connector.addFailureEventsToRetryQueue(events, exception); + sink.addFailureEventsToRetryQueue(events, exception); } } @@ -131,7 +129,7 @@ protected void onErrorInternal(final Exception exception) { protected void doTransfer( final AsyncPipeDataTransferServiceClient client, final TPipeTransferReq req) throws TException { - client.pipeTransfer(req, this); + transferWithOptionalRequestSlicing(client, req); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletInsertNodeEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletInsertNodeEventHandler.java index 70ba7f4cfc5b7..56d1ce41b029c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletInsertNodeEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletInsertNodeEventHandler.java @@ -41,12 +41,12 @@ public PipeTransferTabletInsertNodeEventHandler( protected void doTransfer( final AsyncPipeDataTransferServiceClient client, final TPipeTransferReq req) throws TException { - client.pipeTransfer(req, this); + transferWithOptionalRequestSlicing(client, req); } @Override protected void updateLeaderCache(final TSStatus status) { - connector.updateLeaderCache( + sink.updateLeaderCache( ((PipeInsertNodeTabletInsertionEvent) event).getDeviceId(), status.getRedirectNode()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java index 66a1f4a013b22..a8f1136b897d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletInsertionEventHandler.java @@ -54,7 +54,7 @@ protected PipeTransferTabletInsertionEventHandler( public void transfer(final AsyncPipeDataTransferServiceClient client) throws TException { if (event instanceof EnrichedEvent) { - connector.rateLimitIfNeeded( + sink.rateLimitIfNeeded( ((EnrichedEvent) event).getPipeName(), ((EnrichedEvent) event).getCreationTime(), client.getEndPoint(), @@ -77,8 +77,7 @@ protected boolean onCompleteInternal(final TPipeTransferResp response) { // Only handle the failed statuses to avoid string format performance overhead if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { - connector - .statusHandler() + sink.statusHandler() .handle(response.getStatus(), response.getStatus().getMessage(), event.toString()); } if (event instanceof EnrichedEvent) { @@ -109,7 +108,7 @@ protected void onErrorInternal(final Exception exception) { event instanceof EnrichedEvent ? ((EnrichedEvent) event).getCommitterKey() : null, event instanceof EnrichedEvent ? ((EnrichedEvent) event).getCommitIds() : null); } finally { - connector.addFailureEventToRetryQueue(event, exception); + sink.addFailureEventToRetryQueue(event, exception); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletRawEventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletRawEventHandler.java index ff1daa05c2859..eb4677de358e9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletRawEventHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTabletRawEventHandler.java @@ -32,20 +32,20 @@ public class PipeTransferTabletRawEventHandler extends PipeTransferTabletInserti public PipeTransferTabletRawEventHandler( final PipeRawTabletInsertionEvent event, final TPipeTransferReq req, - final IoTDBDataRegionAsyncSink connector) { - super(event, req, connector); + final IoTDBDataRegionAsyncSink sink) { + super(event, req, sink); } @Override protected void doTransfer( final AsyncPipeDataTransferServiceClient client, final TPipeTransferReq req) throws TException { - client.pipeTransfer(req, this); + transferWithOptionalRequestSlicing(client, req); } @Override protected void updateLeaderCache(final TSStatus status) { - connector.updateLeaderCache( + sink.updateLeaderCache( ((PipeRawTabletInsertionEvent) event).getDeviceId(), status.getRedirectNode()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTrackableHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTrackableHandler.java index 21f7c144bed22..40b05066a93f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTrackableHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTrackableHandler.java @@ -21,7 +21,12 @@ import org.apache.iotdb.commons.client.ThriftClient; import org.apache.iotdb.commons.client.async.AsyncPipeDataTransferServiceClient; +import org.apache.iotdb.commons.pipe.resource.log.PipeLogger; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferSliceReqBuilder; import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink; +import org.apache.iotdb.pipe.api.exception.PipeConnectionException; +import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; @@ -36,18 +41,18 @@ public abstract class PipeTransferTrackableHandler implements AsyncMethodCallback, AutoCloseable { private static final Logger LOGGER = LoggerFactory.getLogger(PipeTransferTsFileHandler.class); - protected final IoTDBDataRegionAsyncSink connector; + protected final IoTDBDataRegionAsyncSink sink; protected volatile AsyncPipeDataTransferServiceClient client; - public PipeTransferTrackableHandler(final IoTDBDataRegionAsyncSink connector) { - this.connector = connector; + public PipeTransferTrackableHandler(final IoTDBDataRegionAsyncSink sink) { + this.sink = sink; } @Override public void onComplete(final TPipeTransferResp response) { - if (connector.isClosed()) { + if (sink.isClosed()) { clearEventsReferenceCount(); - connector.eliminateHandler(this, true); + sink.eliminateHandler(this, true); return; } @@ -56,7 +61,7 @@ public void onComplete(final TPipeTransferResp response) { // completed // NOTE: We should not clear the reference count of events, as this would cause the // `org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic.IoTDBPipeDataSinkIT#testSinkTsFileFormat3` test to fail. - connector.eliminateHandler(this, false); + sink.eliminateHandler(this, false); } } @@ -67,14 +72,14 @@ public void onError(final Exception exception) { client.setPrintLogWhenEncounterException(false); } - if (connector.isClosed()) { + if (sink.isClosed()) { clearEventsReferenceCount(); - connector.eliminateHandler(this, true); + sink.eliminateHandler(this, true); return; } onErrorInternal(exception); - connector.eliminateHandler(this, false); + sink.eliminateHandler(this, false); } /** @@ -93,15 +98,16 @@ protected boolean tryTransfer( this.client = client; } // track handler before checking if connector is closed - connector.trackHandler(this); - if (connector.isClosed()) { + sink.trackHandler(this); + if (sink.isClosed()) { clearEventsReferenceCount(); - connector.eliminateHandler(this, true); + sink.eliminateHandler(this, true); client.setShouldReturnSelf(true); client.returnSelf( (e) -> { if (e instanceof IllegalStateException) { - LOGGER.info( + PipeLogger.log( + LOGGER::info, "Illegal state when return the client to object pool, maybe the pool is already cleared. Will ignore."); return true; } @@ -126,8 +132,131 @@ protected abstract void doTransfer( final AsyncPipeDataTransferServiceClient client, final TPipeTransferReq req) throws TException; + protected final void transferWithOptionalRequestSlicing( + final AsyncPipeDataTransferServiceClient client, final TPipeTransferReq req) + throws TException { + final int bodySizeLimit = PipeTransferSliceReqBuilder.getBodySizeLimit(); + if (!PipeTransferSliceReqBuilder.shouldSlice(req, bodySizeLimit)) { + client.pipeTransfer(req, this); + return; + } + + LOGGER.warn( + "The body size of the request is too large. The request will be sliced. Origin req: {}-{}. " + + "Request body size: {}, threshold: {}", + req.getVersion(), + req.getType(), + req.body.limit(), + bodySizeLimit); + + final int sliceCount = PipeTransferSliceReqBuilder.getSliceCount(req, bodySizeLimit); + final boolean shouldReturnSelf = client.shouldReturnSelf(); + try { + transferSlicedRequest( + client, + req, + shouldReturnSelf, + PipeTransferSliceReqBuilder.nextSliceOrderId(), + 0, + sliceCount, + bodySizeLimit); + } catch (final Exception e) { + fallbackToWholeRequest(client, req, shouldReturnSelf, e); + } + } + public abstract void clearEventsReferenceCount(); + private void transferSlicedRequest( + final AsyncPipeDataTransferServiceClient client, + final TPipeTransferReq originalReq, + final boolean shouldReturnSelf, + final int sliceOrderId, + final int sliceIndex, + final int sliceCount, + final int bodySizeLimit) + throws Exception { + client.setShouldReturnSelf(shouldReturnSelf && sliceIndex == sliceCount - 1); + client.pipeTransfer( + PipeTransferSliceReqBuilder.buildSliceReq( + originalReq, sliceOrderId, sliceIndex, sliceCount, bodySizeLimit), + new AsyncMethodCallback() { + @Override + public void onComplete(final TPipeTransferResp response) { + if (sink.isClosed() || sliceIndex == sliceCount - 1) { + PipeTransferTrackableHandler.this.onComplete(response); + return; + } + + if (response == null) { + fallbackToWholeRequest( + client, + originalReq, + shouldReturnSelf, + new PipeException("TPipeTransferResp is null when transferring slice.")); + return; + } + + if (response.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + fallbackToWholeRequest( + client, + originalReq, + shouldReturnSelf, + new PipeConnectionException( + String.format( + "Failed to transfer slice. Origin req: %s-%s, slice index: %d, slice count: %d. Reason: %s", + originalReq.getVersion(), + originalReq.getType(), + sliceIndex, + sliceCount, + response.getStatus()))); + return; + } + + try { + transferSlicedRequest( + client, + originalReq, + shouldReturnSelf, + sliceOrderId, + sliceIndex + 1, + sliceCount, + bodySizeLimit); + } catch (final Exception e) { + fallbackToWholeRequest(client, originalReq, shouldReturnSelf, e); + } + } + + @Override + public void onError(final Exception exception) { + if (sink.isClosed() || sliceIndex == sliceCount - 1) { + PipeTransferTrackableHandler.this.onError(exception); + return; + } + fallbackToWholeRequest(client, originalReq, shouldReturnSelf, exception); + } + }); + } + + private void fallbackToWholeRequest( + final AsyncPipeDataTransferServiceClient client, + final TPipeTransferReq originalReq, + final boolean shouldReturnSelf, + final Exception exception) { + LOGGER.warn( + "Failed to transfer slice. Origin req: {}-{}. Retry the whole transfer.", + originalReq.getVersion(), + originalReq.getType(), + exception); + + try { + client.setShouldReturnSelf(shouldReturnSelf); + client.pipeTransfer(originalReq, this); + } catch (final Exception e) { + PipeTransferTrackableHandler.this.onError(e); + } + } + public void closeClient() { if (Objects.isNull(client)) { return; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandler.java index a9426ed7b8b3b..d7515141daeb8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTsFileHandler.java @@ -78,6 +78,7 @@ public class PipeTransferTsFileHandler extends PipeTransferTrackableHandler { private File currentFile; private final boolean transferMod; + private final String dataBaseName; private final int readFileBufferSize; private PipeTsFileMemoryBlock memoryBlock; @@ -98,7 +99,8 @@ public PipeTransferTsFileHandler( final AtomicBoolean eventsHadBeenAddedToRetryQueue, final File tsFile, final File modFile, - final boolean transferMod) + final boolean transferMod, + final String dataBaseName) throws InterruptedException { super(connector); @@ -111,6 +113,7 @@ public PipeTransferTsFileHandler( this.tsFile = tsFile; this.modFile = modFile; this.transferMod = transferMod; + this.dataBaseName = dataBaseName; currentFile = transferMod ? modFile : tsFile; // NOTE: Waiting for resource enough for slicing here may cause deadlock! @@ -157,9 +160,10 @@ public void transfer( this.client = client; if (client == null) { - LOGGER.warn( - "Client has been returned to the pool. Current handler status is {}. Will not transfer {}.", - connector.isClosed() ? "CLOSED" : "NOT CLOSED", + PipeLogger.log( + LOGGER::warn, + "Client has been returned to the pool. Current handler status is %s. Will not transfer %s.", + sink.isClosed() ? "CLOSED" : "NOT CLOSED", tsFile); return; } @@ -168,7 +172,7 @@ public void transfer( client.setTimeoutDynamically(clientManager.getConnectionTimeout()); PipeResourceMetrics.getInstance().recordDiskIO(readFileBufferSize); - if (connector.isEnableSendTsFileLimit()) { + if (sink.isEnableSendTsFileLimit()) { TsFileSendRateLimiter.getInstance().acquire(readFileBufferSize); } final int readLength = reader.read(readBuffer); @@ -190,13 +194,21 @@ public void transfer( final TPipeTransferReq uncompressedReq = transferMod ? PipeTransferTsFileSealWithModReq.toTPipeTransferReq( - modFile.getName(), modFile.length(), tsFile.getName(), tsFile.length()) - : PipeTransferTsFileSealReq.toTPipeTransferReq(tsFile.getName(), tsFile.length()); - final TPipeTransferReq req = connector.compressIfNeeded(uncompressedReq); + modFile.getName(), + modFile.length(), + tsFile.getName(), + tsFile.length(), + dataBaseName) + : dataBaseName == null + ? PipeTransferTsFileSealReq.toTPipeTransferReq( + tsFile.getName(), tsFile.length()) + : PipeTransferTsFileSealWithModReq.toTPipeTransferReq( + tsFile.getName(), tsFile.length(), dataBaseName); + final TPipeTransferReq req = sink.compressIfNeeded(uncompressedReq); pipeName2WeightMap.forEach( (pipePair, weight) -> - connector.rateLimitIfNeeded( + sink.rateLimitIfNeeded( pipePair.getLeft(), pipePair.getRight(), client.getEndPoint(), @@ -219,11 +231,11 @@ public void transfer( currentFile.getName(), position, payload) : PipeTransferTsFilePieceReq.toTPipeTransferReq( currentFile.getName(), position, payload); - final TPipeTransferReq req = connector.compressIfNeeded(uncompressedReq); + final TPipeTransferReq req = sink.compressIfNeeded(uncompressedReq); pipeName2WeightMap.forEach( (pipePair, weight) -> - connector.rateLimitIfNeeded( + sink.rateLimitIfNeeded( pipePair.getLeft(), pipePair.getRight(), client.getEndPoint(), @@ -241,7 +253,7 @@ public void onComplete(final TPipeTransferResp response) { try { super.onComplete(response); } finally { - if (connector.isClosed()) { + if (sink.isClosed()) { returnClientIfNecessary(); } } @@ -255,8 +267,7 @@ protected boolean onCompleteInternal(final TPipeTransferResp response) { // Only handle the failed statuses to avoid string format performance overhead if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { - connector - .statusHandler() + sink.statusHandler() .handle( status, String.format( @@ -330,9 +341,7 @@ protected boolean onCompleteInternal(final TPipeTransferResp response) { // Only handle the failed statuses to avoid string format performance overhead if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() && status.getCode() != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { - connector - .statusHandler() - .handle(status, response.getStatus().getMessage(), tsFile.getName()); + sink.statusHandler().handle(status, response.getStatus().getMessage(), tsFile.getName()); } } @@ -404,7 +413,7 @@ protected void onErrorInternal(final Exception exception) { returnClientIfNecessary(); } finally { if (eventsHadBeenAddedToRetryQueue.compareAndSet(false, true)) { - connector.addFailureEventsToRetryQueue(events, exception); + sink.addFailureEventsToRetryQueue(events, exception); } } } @@ -415,7 +424,7 @@ private void returnClientIfNecessary() { return; } - if (connector.isClosed()) { + if (sink.isClosed()) { closeClient(); } @@ -423,7 +432,8 @@ private void returnClientIfNecessary() { client.returnSelf( (e) -> { if (e instanceof IllegalStateException) { - LOGGER.info( + PipeLogger.log( + LOGGER::info, "Illegal state when return the client to object pool, maybe the pool is already cleared. Will ignore."); return true; } @@ -437,14 +447,15 @@ protected void doTransfer( final AsyncPipeDataTransferServiceClient client, final TPipeTransferReq req) throws TException { if (client == null) { - LOGGER.warn( - "Client has been returned to the pool. Current handler status is {}. Will not transfer {}.", - connector.isClosed() ? "CLOSED" : "NOT CLOSED", + PipeLogger.log( + LOGGER::warn, + "Client has been returned to the pool. Current handler status is %s. Will not transfer %s.", + sink.isClosed() ? "CLOSED" : "NOT CLOSED", tsFile); return; } - client.pipeTransfer(req, this); + transferWithOptionalRequestSlicing(client, req); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java index a13f40b1b8308..1bb0c383ff87b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataRegionSyncSink.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.sink.client.IoTDBSyncClient; import org.apache.iotdb.commons.pipe.sink.limiter.TsFileSendRateLimiter; @@ -274,7 +275,7 @@ private void doTransfer(final PipeTabletEventTsFileBatch batchToTransfer) final Map, Double> pipe2WeightMap = batchToTransfer.deepCopyPipe2WeightMap(); for (final File tsFile : sealedFiles) { - doTransfer(pipe2WeightMap, tsFile, null); + doTransfer(pipe2WeightMap, tsFile, null, null); try { RetryUtils.retryOnException( () -> { @@ -427,7 +428,8 @@ private void doTransferWrapper(final PipeTsFileInsertionEvent pipeTsFileInsertio pipeTsFileInsertionEvent.getCreationTime()), 1.0), pipeTsFileInsertionEvent.getTsFile(), - pipeTsFileInsertionEvent.isWithMod() ? pipeTsFileInsertionEvent.getModFile() : null); + pipeTsFileInsertionEvent.isWithMod() ? pipeTsFileInsertionEvent.getModFile() : null, + pipeTsFileInsertionEvent.getDatabaseName()); } finally { pipeTsFileInsertionEvent.decreaseReferenceCount( IoTDBDataRegionSyncSink.class.getName(), false); @@ -437,7 +439,8 @@ private void doTransferWrapper(final PipeTsFileInsertionEvent pipeTsFileInsertio private void doTransfer( final Map, Double> pipeName2WeightMap, final File tsFile, - final File modFile) + final File modFile, + final String dataBaseName) throws PipeException, IOException { final Pair clientAndStatus = clientManager.getClient(); @@ -453,7 +456,11 @@ private void doTransfer( final TPipeTransferReq req = compressIfNeeded( PipeTransferTsFileSealWithModReq.toTPipeTransferReq( - modFile.getName(), modFile.length(), tsFile.getName(), tsFile.length())); + modFile.getName(), + modFile.length(), + tsFile.getName(), + tsFile.length(), + dataBaseName)); pipeName2WeightMap.forEach( (pipePair, weight) -> @@ -478,7 +485,11 @@ private void doTransfer( try { final TPipeTransferReq req = compressIfNeeded( - PipeTransferTsFileSealReq.toTPipeTransferReq(tsFile.getName(), tsFile.length())); + dataBaseName == null + ? PipeTransferTsFileSealReq.toTPipeTransferReq( + tsFile.getName(), tsFile.length()) + : PipeTransferTsFileSealWithModReq.toTPipeTransferReq( + tsFile.getName(), tsFile.length(), dataBaseName)); pipeName2WeightMap.forEach( (pipePair, weight) -> @@ -521,9 +532,15 @@ public TPipeTransferReq compressIfNeeded(final TPipeTransferReq req) throws IOEx } @Override - public synchronized void discardEventsOfPipe(final String pipeNameToDrop, final int regionId) { + public synchronized void discardEventsOfPipe( + final String pipeNameToDrop, final long creationTimeToDrop, final int regionId) { + discardEventsOfPipe(new CommitterKey(pipeNameToDrop, creationTimeToDrop, regionId, -1)); + } + + @Override + public synchronized void discardEventsOfPipe(final CommitterKey committerKey) { if (Objects.nonNull(tabletBatchBuilder)) { - tabletBatchBuilder.discardEventsOfPipe(pipeNameToDrop, regionId); + tabletBatchBuilder.discardEventsOfPipe(committerKey); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/websocket/WebSocketConnectorServer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/websocket/WebSocketConnectorServer.java index 6da644604955e..0ecd1ad6e340b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/websocket/WebSocketConnectorServer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/websocket/WebSocketConnectorServer.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.pipe.sink.protocol.websocket; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.pipe.api.event.Event; @@ -39,6 +40,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.PriorityBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; @@ -57,6 +59,8 @@ public class WebSocketConnectorServer extends WebSocketServer { private final ConcurrentHashMap> eventsWaitingForAck = new ConcurrentHashMap<>(); + private final Set droppedPipeTaskKeys = ConcurrentHashMap.newKeySet(); + private final BidiMap router = new DualTreeBidiMap(null, Comparator.comparing(Object::hashCode)) {}; @@ -97,13 +101,8 @@ public synchronized void unregister(WebSocketSink connector) { eventWrappers = new ArrayList<>(eventTransferQueue); eventTransferQueue.clear(); } - eventWrappers.forEach( - (eventWrapper) -> { - if (eventWrapper.event instanceof EnrichedEvent) { - ((EnrichedEvent) eventWrapper.event) - .decreaseReferenceCount(WebSocketConnectorServer.class.getName(), false); - } - }); + eventWrappers.forEach(eventWrapper -> discardEvent(eventWrapper.event)); + eventWrappers.clear(); synchronized (eventTransferQueue) { eventTransferQueue.notifyAll(); } @@ -113,13 +112,36 @@ public synchronized void unregister(WebSocketSink connector) { if (eventsWaitingForAck.containsKey(pipeName)) { eventsWaitingForAck .remove(pipeName) - .forEach( - (eventId, eventWrapper) -> { - if (eventWrapper.event instanceof EnrichedEvent) { - ((EnrichedEvent) eventWrapper.event) - .decreaseReferenceCount(WebSocketConnectorServer.class.getName(), false); - } - }); + .forEach((eventId, eventWrapper) -> discardEvent(eventWrapper.event)); + } + + droppedPipeTaskKeys.removeIf(key -> key.getPipeName().equals(pipeName)); + } + + public synchronized void discardEventsOfPipe( + final String pipeNameToDrop, final long creationTimeToDrop, final int regionId) { + discardEventsOfPipe(new CommitterKey(pipeNameToDrop, creationTimeToDrop, regionId, -1)); + } + + public synchronized void discardEventsOfPipe(final CommitterKey committerKey) { + droppedPipeTaskKeys.add(committerKey); + + final PriorityBlockingQueue eventTransferQueue = + eventsWaitingForTransfer.get(committerKey.getPipeName()); + if (eventTransferQueue != null) { + eventTransferQueue.removeIf( + eventWrapper -> discardIfMatches(eventWrapper.event, committerKey)); + synchronized (eventTransferQueue) { + eventTransferQueue.notifyAll(); + } + } + + final ConcurrentHashMap eventId2EventMap = + eventsWaitingForAck.get(committerKey.getPipeName()); + if (eventId2EventMap != null) { + eventId2EventMap + .entrySet() + .removeIf(entry -> discardIfMatches(entry.getValue().event, committerKey)); } } @@ -300,21 +322,24 @@ public void onError(WebSocket webSocket, Exception e) { } public void addEvent(Event event, WebSocketSink connector) { + if (isDroppedPipe(event)) { + discardEvent(event); + return; + } + + final String pipeName = connector.getPipeName(); final PriorityBlockingQueue queue = - eventsWaitingForTransfer.get(connector.getPipeName()); + eventsWaitingForTransfer.get(pipeName); if (queue == null) { LOGGER.warn("The pipe {} was dropped so the event {} will be dropped.", connector, event); - if (event instanceof EnrichedEvent) { - ((EnrichedEvent) event) - .decreaseReferenceCount(WebSocketConnectorServer.class.getName(), false); - } + discardEvent(event); return; } if (queue.size() >= 5) { synchronized (queue) { - while (queue.size() >= 5) { + while (queue.size() >= 5 && isQueueAvailable(pipeName, queue) && !isDroppedPipe(event)) { try { queue.wait(); } catch (InterruptedException e) { @@ -323,12 +348,22 @@ public void addEvent(Event event, WebSocketSink connector) { } } + if (!isQueueAvailable(pipeName, queue) || isDroppedPipe(event)) { + discardEvent(event); + return; + } + queue.put( new EventWaitingForTransfer(eventIdGenerator.incrementAndGet(), connector, event)); return; } } + if (!isQueueAvailable(pipeName, queue) || isDroppedPipe(event)) { + discardEvent(event); + return; + } + synchronized (queue) { queue.put(new EventWaitingForTransfer(eventIdGenerator.incrementAndGet(), connector, event)); } @@ -377,6 +412,11 @@ private void transfer(String pipeName, EventWaitingForTransfer element) { final WebSocketSink connector = element.connector; try { + if (isDroppedPipe(event)) { + discardEvent(event); + return; + } + ByteBuffer tabletBuffer; if (event instanceof PipeRawTabletInsertionEvent) { tabletBuffer = ((PipeRawTabletInsertionEvent) event).convertToTablet().serialize(); @@ -387,7 +427,11 @@ private void transfer(String pipeName, EventWaitingForTransfer element) { } if (tabletBuffer == null) { - connector.commit((EnrichedEvent) event); + if (isDroppedPipe(event)) { + discardEvent(event); + } else { + connector.commit((EnrichedEvent) event); + } return; } @@ -398,11 +442,17 @@ private void transfer(String pipeName, EventWaitingForTransfer element) { server.broadcast(payload, Collections.singletonList(router.get(pipeName))); + if (isDroppedPipe(event)) { + discardEvent(event); + return; + } + final ConcurrentHashMap eventId2EventMap = eventsWaitingForAck.get(pipeName); if (eventId2EventMap == null) { LOGGER.warn( "The pipe {} was dropped so the event ack {} will be ignored.", pipeName, eventId); + discardEvent(event); return; } eventId2EventMap.put(eventId, new EventWaitingForAck(connector, event)); @@ -410,13 +460,10 @@ private void transfer(String pipeName, EventWaitingForTransfer element) { synchronized (server) { final PriorityBlockingQueue queue = eventsWaitingForTransfer.get(pipeName); - if (queue == null) { + if (queue == null || isDroppedPipe(event)) { LOGGER.warn( "The pipe {} was dropped so the event {} will be dropped.", pipeName, eventId); - if (event instanceof EnrichedEvent) { - ((EnrichedEvent) event) - .decreaseReferenceCount(WebSocketConnectorServer.class.getName(), false); - } + discardEvent(event); return; } @@ -465,4 +512,43 @@ public EventWaitingForAck(WebSocketSink connector, Event event) { this.event = event; } } + + private boolean discardIfMatches(final Event event, final CommitterKey committerKey) { + if (!(event instanceof EnrichedEvent)) { + return false; + } + + final EnrichedEvent enrichedEvent = (EnrichedEvent) event; + if (!isEventFromPipe(enrichedEvent, committerKey)) { + return false; + } + + discardEvent(enrichedEvent); + return true; + } + + private boolean isDroppedPipe(final Event event) { + return event instanceof EnrichedEvent + && droppedPipeTaskKeys.stream() + .anyMatch(key -> isEventFromPipe((EnrichedEvent) event, key)); + } + + private static boolean isEventFromPipe( + final EnrichedEvent event, final CommitterKey committerKey) { + return committerKey.getPipeName().equals(event.getPipeName()) + && committerKey.getCreationTime() == event.getCreationTime() + && committerKey.getRegionId() == event.getRegionId() + && (committerKey.getRestartTimes() < 0 || committerKey.equals(event.getCommitterKey())); + } + + private boolean isQueueAvailable( + final String pipeName, final PriorityBlockingQueue queue) { + return eventsWaitingForTransfer.get(pipeName) == queue; + } + + private void discardEvent(final Event event) { + if (event instanceof EnrichedEvent) { + ((EnrichedEvent) event).clearReferenceCount(WebSocketSink.class.getName()); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/websocket/WebSocketSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/websocket/WebSocketSink.java index c89486bc2c81d..7841e0199b250 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/websocket/WebSocketSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/websocket/WebSocketSink.java @@ -19,8 +19,10 @@ package org.apache.iotdb.db.pipe.sink.protocol.websocket; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; +import org.apache.iotdb.commons.pipe.sink.protocol.PipeConnectorWithEventDiscard; import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; @@ -39,7 +41,7 @@ import java.util.Arrays; import java.util.Optional; -public class WebSocketSink implements PipeConnector { +public class WebSocketSink implements PipeConnector, PipeConnectorWithEventDiscard { private static final Logger LOGGER = LoggerFactory.getLogger(WebSocketSink.class); @@ -164,6 +166,21 @@ public void close() throws Exception { } } + @Override + public void discardEventsOfPipe( + final String pipeNameToDrop, final long creationTimeToDrop, final int regionId) { + if (server != null) { + server.discardEventsOfPipe(pipeNameToDrop, creationTimeToDrop, regionId); + } + } + + @Override + public void discardEventsOfPipe(final CommitterKey committerKey) { + if (server != null) { + server.discardEventsOfPipe(committerKey); + } + } + public void commit(EnrichedEvent enrichedEvent) { Optional.ofNullable(enrichedEvent) .ifPresent(event -> event.decreaseReferenceCount(WebSocketSink.class.getName(), true)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/writeback/WriteBackSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/writeback/WriteBackSink.java index 21976f63ab602..269d0059c6695 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/writeback/WriteBackSink.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/writeback/WriteBackSink.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.sink.protocol.writeback; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.pipe.PipeRuntimeSinkNonReportTimeConfigurableException; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; @@ -131,7 +132,8 @@ private void doTransfer( status = statement.isEmpty() ? RpcUtils.SUCCESS_STATUS : executeStatement(statement); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new PipeException( + throwWriteBackExceptionIfNecessary( + status, String.format( "Transfer PipeInsertNodeTabletInsertionEvent %s error, result status %s", pipeInsertNodeTabletInsertionEvent, status)); @@ -162,13 +164,23 @@ private void doTransfer(final PipeRawTabletInsertionEvent pipeRawTabletInsertion statement.isEmpty() ? RpcUtils.SUCCESS_STATUS : executeStatement(statement); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new PipeException( + throwWriteBackExceptionIfNecessary( + status, String.format( "Transfer PipeRawTabletInsertionEvent %s error, result status %s", pipeRawTabletInsertionEvent, status)); } } + private static void throwWriteBackExceptionIfNecessary( + final TSStatus status, final String exceptionMessage) { + if (status.getCode() == TSStatusCode.NO_PERMISSION.getStatusCode()) { + throw new PipeRuntimeSinkNonReportTimeConfigurableException(exceptionMessage, Long.MAX_VALUE); + } + + throw new PipeException(exceptionMessage); + } + private TSStatus executeStatement(final InsertBaseStatement statement) { return Coordinator.getInstance() .executeForTreeModel( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/PipeTabletEventSorter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/PipeTabletEventSorter.java index 17e7a7c13e263..50beb6405e96e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/PipeTabletEventSorter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/PipeTabletEventSorter.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.pipe.sink.util; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; + import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; @@ -129,6 +131,7 @@ private void deduplicateTimestamps() { // Col: [6, 1] private void sortAndMayDeduplicateValuesAndBitMaps() { int columnIndex = 0; + boolean bitMapsModified = false; for (int i = 0, size = tablet.getSchemas().size(); i < size; i++) { final IMeasurementSchema schema = tablet.getSchemas().get(i); if (schema != null) { @@ -145,10 +148,15 @@ private void sortAndMayDeduplicateValuesAndBitMaps() { if (tablet.bitMaps != null && tablet.bitMaps[columnIndex] != null) { tablet.bitMaps[columnIndex] = deDuplicatedBitMap; + bitMapsModified = true; } columnIndex++; } } + + if (bitMapsModified) { + tablet.bitMaps = PipeTabletUtils.compactBitMaps(tablet.bitMaps, deDuplicatedSize); + } } private Object reorderValueListAndBitMap( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java index fd960584c9f8c..54597cb1bcdac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java @@ -441,6 +441,8 @@ && mayTsFileResourceOverlappedWithPattern(resource))) ? Long.compare(o1.getFileStartTime(), o2.getFileStartTime()) : o1.getMaxProgressIndex().topologicalCompareTo(o2.getMaxProgressIndex())); pendingQueue = new ArrayDeque<>(originalResourceList); + PipeTerminateEvent.initializeHistoricalTransferSummary( + pipeName, creationTime, dataRegionId, filteredTsFileResources.size(), 0); LOGGER.info( "Pipe {}@{}: finish to extract historical TsFile, extracted sequence file count {}/{}, " @@ -537,6 +539,17 @@ public synchronized Event supply() { final TsFileResource resource = pendingQueue.poll(); if (resource == null) { + final PipeTerminateEvent.HistoricalTransferSummary historicalTransferSummary = + PipeTerminateEvent.snapshotHistoricalTransferSummary( + pipeName, creationTime, dataRegionId); + if (Objects.nonNull(historicalTransferSummary)) { + LOGGER.info( + "Pipe {}@{}: historical source has supplied all events, emitting terminate event. {}", + pipeName, + dataRegionId, + historicalTransferSummary.toReportMessage()); + } + final PipeTerminateEvent terminateEvent = new PipeTerminateEvent( pipeName, @@ -632,6 +645,9 @@ public int getPendingQueueSize() { @Override public synchronized void close() { + if (!isTerminateSignalSent) { + PipeTerminateEvent.clearHistoricalTransferSummary(pipeName, creationTime, dataRegionId); + } if (Objects.nonNull(pendingQueue)) { pendingQueue.forEach( resource -> { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java index 3c03ee732d05c..367c83274215f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java @@ -382,8 +382,11 @@ protected void extractProgressReportEvent(final PipeRealtimeEvent event) { } pendingQueue.pollLast(); } - if (pendingQueue.peekLast() instanceof ProgressReportEvent) { - final ProgressReportEvent oldEvent = (ProgressReportEvent) pendingQueue.peekLast(); + final Event last = pendingQueue.peekLast(); + if (last instanceof PipeRealtimeEvent + && ((PipeRealtimeEvent) last).getEvent() instanceof ProgressReportEvent) { + final ProgressReportEvent oldEvent = + (ProgressReportEvent) ((PipeRealtimeEvent) last).getEvent(); oldEvent.bindProgressIndex( oldEvent .getProgressIndex() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/DisruptorQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/DisruptorQueue.java index 2a2fa11074989..d3fc92a677a23 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/DisruptorQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/DisruptorQueue.java @@ -26,24 +26,22 @@ import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; +import org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor.Disruptor; +import org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor.EventHandler; +import org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor.RingBuffer; -import com.lmax.disruptor.BlockingWaitStrategy; -import com.lmax.disruptor.EventHandler; -import com.lmax.disruptor.RingBuffer; -import com.lmax.disruptor.dsl.Disruptor; -import com.lmax.disruptor.dsl.ProducerType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.function.Consumer; -import static org.apache.iotdb.commons.concurrent.ThreadName.PIPE_EXTRACTOR_DISRUPTOR; +import static org.apache.iotdb.commons.concurrent.ThreadName.PIPE_SOURCE_DISRUPTOR; public class DisruptorQueue { private static final Logger LOGGER = LoggerFactory.getLogger(DisruptorQueue.class); private static final IoTDBDaemonThreadFactory THREAD_FACTORY = - new IoTDBDaemonThreadFactory(PIPE_EXTRACTOR_DISRUPTOR.getName()); + new IoTDBDaemonThreadFactory(PIPE_SOURCE_DISRUPTOR.getName()); private final PipeMemoryBlock allocatedMemoryBlock; private final Disruptor disruptor; @@ -75,9 +73,8 @@ public DisruptorQueue( 32, Math.toIntExact( allocatedMemoryBlock.getMemoryUsageInBytes() / ringBufferEntrySizeInBytes)), - THREAD_FACTORY, - ProducerType.MULTI, - new BlockingWaitStrategy()); + THREAD_FACTORY); + disruptor.handleEventsWith( (container, sequence, endOfBatch) -> { final PipeRealtimeEvent realtimeEvent = container.getEvent(); @@ -90,16 +87,29 @@ public DisruptorQueue( } public void publish(final PipeRealtimeEvent event) { + publishOrDrop(event); + } + + public boolean publishOrDrop(final PipeRealtimeEvent event) { final EnrichedEvent innerEvent = event.getEvent(); if (innerEvent instanceof PipeHeartbeatEvent) { ((PipeHeartbeatEvent) innerEvent).recordDisruptorSize(ringBuffer); } - ringBuffer.publishEvent((container, sequence, o) -> container.setEvent(event), event); - mayPrintExceedingLog(); + final boolean published = + ringBuffer.publishEvent( + (container, sequence, o) -> container.setEvent(event), event, this::isClosed); + if (published) { + mayPrintExceedingLog(); + } + return published; } - public void shutdown() { + public void closeInput() { isClosed = true; + } + + public void shutdown() { + closeInput(); // use shutdown instead of halt to ensure all published events have been handled disruptor.shutdown(); allocatedMemoryBlock.close(); @@ -127,7 +137,7 @@ private void mayPrintExceedingLog() { private static class EventContainer { - private PipeRealtimeEvent event; + private volatile PipeRealtimeEvent event; private EventContainer() {} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/DisruptorQueueExceptionHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/DisruptorQueueExceptionHandler.java index 91ad0224fc538..5330f3486f50e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/DisruptorQueueExceptionHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/DisruptorQueueExceptionHandler.java @@ -19,7 +19,8 @@ package org.apache.iotdb.db.pipe.source.dataregion.realtime.assigner; -import com.lmax.disruptor.ExceptionHandler; +import org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor.ExceptionHandler; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java index e3067a13126eb..01dcbb8926509 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/assigner/PipeDataRegionAssigner.java @@ -56,7 +56,11 @@ public class PipeDataRegionAssigner implements Closeable { private final String dataRegionId; + private volatile int listenToTsFileSourceCount = 0; + private volatile int listenToInsertNodeSourceCount = 0; + private final PipeEventCounter eventCounter = new PipeDataRegionEventCounter(); + private int inFlightPublishCount = 0; public String getDataRegionId() { return dataRegionId; @@ -84,15 +88,29 @@ public void publishToAssign(final PipeRealtimeEvent event) { ((PipeHeartbeatEvent) innerEvent).onPublished(); } - // use synchronized here for completely preventing reference count leaks under extreme thread - // scheduling when closing synchronized (this) { - if (!disruptor.isClosed()) { - disruptor.publish(event); - } else { + if (disruptor.isClosed()) { onAssignedHook(event); + return; + } + inFlightPublishCount++; + } + + boolean isPublished = false; + try { + isPublished = disruptor.publishOrDrop(event); + } finally { + synchronized (this) { + inFlightPublishCount--; + if (inFlightPublishCount == 0) { + notifyAll(); + } } } + + if (!isPublished) { + onAssignedHook(event); + } } private void onAssignedHook(final PipeRealtimeEvent realtimeEvent) { @@ -194,15 +212,37 @@ private void assignToSource( }); } - public void startAssignTo(final PipeRealtimeDataRegionSource extractor) { - matcher.register(extractor); + public synchronized void startAssignTo(final PipeRealtimeDataRegionSource source) { + matcher.register(source); + if (source.isNeedListenToTsFile()) { + listenToTsFileSourceCount++; + } + if (source.isNeedListenToInsertNode()) { + listenToInsertNodeSourceCount++; + } + logSourceAssignmentChange("registered", source); + } + + public synchronized void stopAssignTo(final PipeRealtimeDataRegionSource source) { + matcher.deregister(source); + if (source.isNeedListenToTsFile()) { + listenToTsFileSourceCount--; + } + if (source.isNeedListenToInsertNode()) { + listenToInsertNodeSourceCount--; + } + logSourceAssignmentChange("deregistered", source); + } + + public boolean shouldListenToTsFile() { + return listenToTsFileSourceCount > 0; } - public void stopAssignTo(final PipeRealtimeDataRegionSource extractor) { - matcher.deregister(extractor); + public boolean shouldListenToInsertNode() { + return listenToInsertNodeSourceCount > 0; } - public boolean notMoreExtractorNeededToBeAssigned() { + public boolean notMoreSourceNeededToBeAssigned() { return matcher.getRegisterCount() == 0; } @@ -216,9 +256,25 @@ public boolean notMoreExtractorNeededToBeAssigned() { public synchronized void close() { PipeAssignerMetrics.getInstance().deregister(dataRegionId); + boolean interrupted = false; + disruptor.closeInput(); + while (inFlightPublishCount > 0) { + try { + wait(); + } catch (final InterruptedException e) { + interrupted = true; + LOGGER.warn( + "Interrupted while waiting for in-flight publishes to finish when closing assigner on data region {}.", + dataRegionId); + } + } + final long startTime = System.currentTimeMillis(); disruptor.shutdown(); matcher.clear(); + if (interrupted) { + Thread.currentThread().interrupt(); + } LOGGER.info( "Pipe: Assigner on data region {} shutdown internal disruptor within {} ms", dataRegionId, @@ -236,4 +292,19 @@ public int getTsFileInsertionEventCount() { public int getPipeHeartbeatEventCount() { return eventCounter.getPipeHeartbeatEventCount(); } + + private void logSourceAssignmentChange( + final String action, final PipeRealtimeDataRegionSource source) { + LOGGER.info( + "Pipe {}@{} {} realtime source on data region {} (listenToTsFile={}, listenToInsertNode={}, registeredSourceCount={}, tsFileSourceCount={}, insertNodeSourceCount={}).", + source.getPipeName(), + source.getCreationTime(), + action, + dataRegionId, + source.isNeedListenToTsFile(), + source.isNeedListenToInsertNode(), + matcher.getRegisterCount(), + listenToTsFileSourceCount, + listenToInsertNodeSourceCount); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/BatchEventProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/BatchEventProcessor.java new file mode 100644 index 0000000000000..6a3b2cc62fd93 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/BatchEventProcessor.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Batch event processor for consuming events + * + *

This implementation is based on LMAX Disruptor (https://github.com/LMAX-Exchange/disruptor) + * and simplified for IoTDB's Pipe module (removed complex lifecycle management). + * + *

Core algorithm preserved from LMAX Disruptor: + * + *

    + *
  • Batch processing loop + *
  • Sequence tracking + *
  • endOfBatch detection + *
+ * + * @param event type + */ +public final class BatchEventProcessor implements Runnable { + private static final Logger LOGGER = LoggerFactory.getLogger(BatchEventProcessor.class); + + private final RingBuffer ringBuffer; + private final SequenceBarrier sequenceBarrier; + private final EventHandler eventHandler; + private final Sequence sequence = new Sequence(); + private ExceptionHandler exceptionHandler = new DefaultExceptionHandler<>(); + private volatile boolean running = true; + + public BatchEventProcessor( + RingBuffer ringBuffer, SequenceBarrier barrier, EventHandler eventHandler) { + this.ringBuffer = ringBuffer; + this.sequenceBarrier = barrier; + this.eventHandler = eventHandler; + } + + public Sequence getSequence() { + return sequence; + } + + public void setExceptionHandler(ExceptionHandler exceptionHandler) { + this.exceptionHandler = exceptionHandler; + } + + public void halt() { + running = false; + } + + @Override + public void run() { + long nextSequence = sequence.get() + 1L; + + while (running) { + try { + // Wait for available sequence + final long availableSequence = sequenceBarrier.waitFor(nextSequence); + + // Batch process all available events + nextSequence = processAvailableEvents(nextSequence, availableSequence); + + } catch (final InterruptedException ex) { + if (!running) { + break; + } + // A transient interrupt should not permanently stop the consumer thread. Otherwise the + // gating sequence will stop advancing and producers may block forever on a full ring + // buffer, making the later close path appear stuck. + Thread.interrupted(); + LOGGER.warn("Processor interrupted unexpectedly, continue running"); + } catch (final Throwable ex) { + exceptionHandler.handleEventException(ex, nextSequence, ringBuffer.get(nextSequence)); + sequence.set(nextSequence); + nextSequence++; + } + } + + if (!running) { + drainRemainingPublishedEvents(nextSequence); + } + LOGGER.info("Processor stopped"); + } + + private long processAvailableEvents(long nextSequence, long availableSequence) throws Throwable { + while (nextSequence <= availableSequence) { + final T event = ringBuffer.get(nextSequence); + eventHandler.onEvent(event, nextSequence, nextSequence == availableSequence); + nextSequence++; + } + + sequence.set(availableSequence); + return nextSequence; + } + + private void drainRemainingPublishedEvents(long nextSequence) { + final long availableSequence = sequenceBarrier.getCursor(); + if (availableSequence < nextSequence) { + return; + } + + final long highestPublishedSequence = + sequenceBarrier.getHighestPublishedSequence(nextSequence, availableSequence); + while (nextSequence <= highestPublishedSequence) { + final T event = ringBuffer.get(nextSequence); + try { + eventHandler.onEvent(event, nextSequence, nextSequence == highestPublishedSequence); + } catch (final Throwable ex) { + exceptionHandler.handleEventException(ex, nextSequence, event); + } finally { + sequence.set(nextSequence); + } + nextSequence++; + } + } + + private static class DefaultExceptionHandler implements ExceptionHandler { + @Override + public void handleEventException(Throwable ex, long sequence, T event) { + LoggerFactory.getLogger(getClass()).error("Exception processing: {} {}", sequence, event, ex); + } + + @Override + public void handleOnStartException(Throwable ex) { + LoggerFactory.getLogger(getClass()).error("Exception during onStart()", ex); + } + + @Override + public void handleOnShutdownException(Throwable ex) { + LoggerFactory.getLogger(getClass()).error("Exception during onShutdown()", ex); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/Disruptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/Disruptor.java new file mode 100644 index 0000000000000..f3a647012856e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/Disruptor.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.ThreadFactory; + +/** + * Simplified Disruptor implementation for IoTDB Pipe + * + *

This implementation is based on LMAX Disruptor (https://github.com/LMAX-Exchange/disruptor) + * and simplified for IoTDB's specific use case in the Pipe module. + * + *

Key simplifications: + * + *

    + *
  • Single event handler support (no complex dependency graphs) + *
  • Simplified lifecycle management + *
  • Removed wait strategies (using simple sleep-based waiting) + *
+ * + * @param event type + */ +public class Disruptor { + private static final Logger LOGGER = LoggerFactory.getLogger(Disruptor.class); + + private final RingBuffer ringBuffer; + private final ThreadFactory threadFactory; + private BatchEventProcessor processor; + private Thread processorThread; + private ExceptionHandler exceptionHandler; + private volatile boolean started = false; + + /** + * Create a Disruptor instance + * + * @param eventFactory factory for creating pre-allocated events + * @param ringBufferSize buffer size (must be power of 2) + * @param threadFactory factory for creating consumer thread + */ + public Disruptor(EventFactory eventFactory, int ringBufferSize, ThreadFactory threadFactory) { + this.ringBuffer = RingBuffer.createMultiProducer(eventFactory, ringBufferSize); + this.threadFactory = threadFactory; + } + + /** + * Configure event handler for processing events + * + *

Creates a batch event processor that will run in its own thread + * + * @param handler event handler implementation + * @return this instance for method chaining + */ + public Disruptor handleEventsWith(final EventHandler handler) { + SequenceBarrier barrier = ringBuffer.newBarrier(); + processor = new BatchEventProcessor<>(ringBuffer, barrier, handler); + + if (exceptionHandler != null) { + processor.setExceptionHandler(exceptionHandler); + } + + ringBuffer.addGatingSequences(processor.getSequence()); + return this; + } + + /** + * Set exception handler for error handling + * + * @param exceptionHandler handler for processing exceptions + */ + public void setDefaultExceptionHandler(ExceptionHandler exceptionHandler) { + this.exceptionHandler = exceptionHandler; + if (processor != null) { + processor.setExceptionHandler(exceptionHandler); + } + } + + public RingBuffer start() { + if (started) { + throw new IllegalStateException("Disruptor already started"); + } + + if (processor == null) { + throw new IllegalStateException("No event handler configured"); + } + + processorThread = threadFactory.newThread(processor); + processorThread.start(); + started = true; + + LOGGER.info("Disruptor started with buffer size: {}", ringBuffer.getBufferSize()); + return ringBuffer; + } + + public void shutdown() { + if (!started) { + return; + } + + if (processor != null) { + processor.halt(); + } + + if (processorThread != null) { + try { + processorThread.interrupt(); + processorThread.join(5000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.warn("Interrupted waiting for processor to stop"); + } + if (processorThread.isAlive()) { + LOGGER.warn("Timed out waiting for processor to stop"); + } + } + + started = false; + LOGGER.info("Disruptor shutdown completed"); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/EventFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/EventFactory.java new file mode 100644 index 0000000000000..785033c68247b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/EventFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor; + +/** + * Event factory for pre-allocating events in RingBuffer + * + *

This interface is based on LMAX Disruptor (https://github.com/LMAX-Exchange/disruptor) and + * adapted for IoTDB's Pipe module. + * + * @param event type + */ +@FunctionalInterface +public interface EventFactory { + /** + * Create new event instance + * + * @return new event + */ + T newInstance(); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/EventHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/EventHandler.java new file mode 100644 index 0000000000000..1fc81a3762373 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/EventHandler.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor; + +/** + * Event handler for processing events from RingBuffer + * + *

This interface is based on LMAX Disruptor (https://github.com/LMAX-Exchange/disruptor) and + * adapted for IoTDB's Pipe module. + * + * @param event type + */ +@FunctionalInterface +public interface EventHandler { + /** + * Handle event + * + * @param event the event + * @param sequence sequence number + * @param endOfBatch whether this is the last event in current batch + * @throws Exception if processing fails + */ + void onEvent(T event, long sequence, boolean endOfBatch) throws Exception; +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/ExceptionHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/ExceptionHandler.java new file mode 100644 index 0000000000000..28396b51ffed1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/ExceptionHandler.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor; + +/** + * Exception handler for event processing errors + * + *

This interface is based on LMAX Disruptor (https://github.com/LMAX-Exchange/disruptor) and + * adapted for IoTDB's Pipe module. + * + * @param event type + */ +public interface ExceptionHandler { + /** + * Handle exception during event processing + * + * @param ex exception + * @param sequence sequence number + * @param event the event + */ + void handleEventException(Throwable ex, long sequence, T event); + + /** Handle exception during processor start */ + void handleOnStartException(Throwable ex); + + /** Handle exception during processor shutdown */ + void handleOnShutdownException(Throwable ex); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/MultiProducerSequencer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/MultiProducerSequencer.java new file mode 100644 index 0000000000000..79d205d670ce7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/MultiProducerSequencer.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor; + +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.concurrent.locks.LockSupport; +import java.util.function.BooleanSupplier; + +/** + * Multi-producer sequencer for coordinating concurrent publishers + * + *

This implementation is based on LMAX Disruptor (https://github.com/LMAX-Exchange/disruptor) + * and preserves the core lock-free multi-producer algorithm for IoTDB's Pipe module. + * + *

Key features preserved from LMAX Disruptor: + * + *

    + *
  • Lock-free CAS-based sequence claiming + *
  • Availability buffer for out-of-order publishing detection + *
  • Backpressure via gating sequences + *
  • Cache line padding to prevent false sharing + *
+ */ +public final class MultiProducerSequencer { + + /** Ring buffer size (must be power of 2) - immutable after construction */ + private final int bufferSize; + + /** + * Producer cursor tracking highest claimed sequence Updated via CAS in next() method Volatile + * reads/writes handled by Sequence class + */ + private final Sequence cursor = new Sequence(); + + /** + * Array of consumer sequences for backpressure control MUST be volatile for safe publication when + * modified by SequenceGroups Array reference is replaced atomically via + * AtomicReferenceFieldUpdater + */ + volatile Sequence[] gatingSequences; + + /** + * Cached minimum gating sequence to reduce contention Updated opportunistically in next() to + * avoid expensive array scan Does not need to be perfectly accurate (conservative is safe) + */ + private final Sequence gatingSequenceCache = new Sequence(); + + /** + * CRITICAL: Availability flags for tracking published sequences + * + *

Handles out-of-order publishing in multi-producer scenario: - Thread A claims seq 10, still + * writing - Thread B claims seq 11, finishes and publishes - Consumer MUST wait for seq 10 before + * reading seq 11 + * + *

Memory visibility guarantees: - Writers use lazySet() for store-store barrier (cheaper than + * volatile write) - Readers use get() for volatile read (ensures visibility across threads) + * + *

AtomicIntegerArray provides same semantics as Unsafe without reflection + */ + private final AtomicIntegerArray availableBuffer; + + /** Mask for fast modulo: sequence & indexMask == sequence % bufferSize */ + private final int indexMask; + + /** Shift for calculating wrap count: sequence >>> indexShift */ + private final int indexShift; + + public MultiProducerSequencer(int bufferSize, Sequence[] gatingSequences) { + if (bufferSize < 1) { + throw new IllegalArgumentException("bufferSize must not be less than 1"); + } + if (Integer.bitCount(bufferSize) != 1) { + throw new IllegalArgumentException("bufferSize must be a power of 2"); + } + + this.bufferSize = bufferSize; + this.gatingSequences = gatingSequences != null ? gatingSequences : new Sequence[0]; + this.availableBuffer = new AtomicIntegerArray(bufferSize); + this.indexMask = bufferSize - 1; + this.indexShift = log2(bufferSize); + + initialiseAvailableBuffer(); + } + + /** + * Claim next n sequences for publishing + * + *

Uses CAS loop to atomically claim sequence numbers. Implements backpressure by parking when + * buffer is full. + * + * @param n number of sequences to claim + * @return highest claimed sequence number + */ + public long next(int n) { + return next(n, () -> false); + } + + /** + * Claim next n sequences for publishing, or abort if the caller is closing. + * + * @param n number of sequences to claim + * @param abortCondition returns {@code true} if the claim should be abandoned + * @return highest claimed sequence number, or {@link Sequence#INITIAL_VALUE} if aborted + */ + public long next(final int n, final BooleanSupplier abortCondition) { + if (n < 1) { + throw new IllegalArgumentException("n must be > 0"); + } + + final BooleanSupplier effectiveAbortCondition = + abortCondition != null ? abortCondition : () -> false; + long current; + long next; + + do { + if (effectiveAbortCondition.getAsBoolean()) { + return Sequence.INITIAL_VALUE; + } + + current = cursor.get(); + next = current + n; + + final long wrapPoint = next - bufferSize; + final long cachedGatingSequence = gatingSequenceCache.get(); + + if (wrapPoint > cachedGatingSequence || cachedGatingSequence > current) { + long gatingSequence = Sequence.getMinimumSequence(gatingSequences, current); + + if (wrapPoint > gatingSequence) { + if (effectiveAbortCondition.getAsBoolean()) { + return Sequence.INITIAL_VALUE; + } + LockSupport.parkNanos(1); + continue; + } + + gatingSequenceCache.set(gatingSequence); + } else if (cursor.compareAndSet(current, next)) { + break; + } + } while (true); + + return next; + } + + /** Publish sequence */ + public void publish(final long sequence) { + setAvailable(sequence); + } + + /** Publish batch */ + public void publish(long lo, long hi) { + for (long l = lo; l <= hi; l++) { + setAvailable(l); + } + } + + /** + * CORE: Check if sequence is available for consumption Uses volatile read to ensure visibility of + * published sequences + */ + public boolean isAvailable(long sequence) { + int index = calculateIndex(sequence); + int flag = calculateAvailabilityFlag(sequence); + return availableBuffer.get(index) == flag; + } + + /** CORE: Get highest published - exact same algorithm */ + public long getHighestPublishedSequence(long lowerBound, long availableSequence) { + for (long sequence = lowerBound; sequence <= availableSequence; sequence++) { + if (!isAvailable(sequence)) { + return sequence - 1; + } + } + return availableSequence; + } + + public Sequence getCursor() { + return cursor; + } + + public int getBufferSize() { + return bufferSize; + } + + public long remainingCapacity() { + long consumed = Sequence.getMinimumSequence(gatingSequences, cursor.get()); + long produced = cursor.get(); + return bufferSize - (produced - consumed); + } + + /** + * Add gating sequences for consumer tracking + * + *

Atomically adds sequences to track consumer progress + * + * @param gatingSequences consumer sequences to add + */ + public void addGatingSequences(Sequence... gatingSequences) { + SequenceGroups.addSequences(this, this.cursor, gatingSequences); + } + + /** + * Create a sequence barrier for consumers + * + * @param sequencesToTrack upstream sequences to wait for + * @return new sequence barrier + */ + public SequenceBarrier newBarrier(Sequence... sequencesToTrack) { + return new SequenceBarrier(this, sequencesToTrack); + } + + /** Initialize available buffer */ + private void initialiseAvailableBuffer() { + for (int i = availableBuffer.length() - 1; i != 0; i--) { + setAvailableBufferValue(i, -1); + } + setAvailableBufferValue(0, -1); + } + + /** + * CORE: Mark sequence as available for consumption + * + *

Uses lazySet() which provides: - Store-store barrier (ensures all prior writes are visible) + * - Cheaper than full volatile write (no store-load barrier) - Sufficient for this use case + * (readers use volatile get) + */ + private void setAvailable(final long sequence) { + setAvailableBufferValue(calculateIndex(sequence), calculateAvailabilityFlag(sequence)); + } + + /** + * Set availability flag with release semantics lazySet() ensures previous event writes are + * visible before flag update + */ + private void setAvailableBufferValue(int index, int flag) { + availableBuffer.lazySet(index, flag); + } + + /** Calculate availability flag */ + private int calculateAvailabilityFlag(final long sequence) { + return (int) (sequence >>> indexShift); + } + + /** Calculate index */ + private int calculateIndex(final long sequence) { + return ((int) sequence) & indexMask; + } + + /** + * Calculate log2 for index shift calculation + * + * @param i input value (must be power of 2) + * @return log2 of input + */ + private static int log2(int i) { + int r = 0; + while ((i >>= 1) != 0) { + ++r; + } + return r; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/RingBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/RingBuffer.java new file mode 100644 index 0000000000000..25f7fe7d1b91f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/RingBuffer.java @@ -0,0 +1,315 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor; + +import java.util.function.BooleanSupplier; + +/** + * Left-hand side padding for cache line alignment + * + *

Prevents false sharing by ensuring RingBuffer fields don't share cache lines with preceding + * objects + */ +abstract class RingBufferPad { + protected long p1, p2, p3, p4, p5, p6, p7; +} + +/** + * Core fields for RingBuffer implementation + * + *

Contains the actual event storage array and sequencing state + */ +abstract class RingBufferFields extends RingBufferPad { + /** Pre-allocated event storage with padding to prevent false sharing */ + private final Object[] entries; + + /** Total number of events in the buffer (must be power of 2) */ + protected final int bufferSize; + + /** Mask for fast modulo operation (bufferSize - 1) */ + protected final int indexMask; + + /** Sequencer for managing producer/consumer coordination */ + protected final MultiProducerSequencer sequencer; + + /** + * Initialize ring buffer fields + * + * @param eventFactory factory for pre-allocating events + * @param sequencer multi-producer sequencer + */ + RingBufferFields(EventFactory eventFactory, MultiProducerSequencer sequencer) { + this.sequencer = sequencer; + this.bufferSize = sequencer.getBufferSize(); + + if (bufferSize < 1) { + throw new IllegalArgumentException("bufferSize must not be less than 1"); + } + if (Integer.bitCount(bufferSize) != 1) { + throw new IllegalArgumentException("bufferSize must be a power of 2"); + } + + this.indexMask = bufferSize - 1; + // Allocate array with padding on both sides to prevent false sharing + this.entries = new Object[bufferSize]; + fill(eventFactory); + } + + /** + * Pre-allocate all events in the buffer + * + * @param eventFactory factory for creating event instances + */ + private void fill(EventFactory eventFactory) { + for (int i = 0; i < bufferSize; i++) { + // Store events starting after front padding + entries[i] = eventFactory.newInstance(); + } + } + + /** + * Get event at sequence using direct memory access + * + * @param sequence sequence number + * @return event at the sequence position + */ + @SuppressWarnings("unchecked") + protected final E elementAt(long sequence) { + // Use Unsafe for lock-free array access with proper memory barriers + return (E) entries[(int) (sequence & indexMask)]; + } +} + +/** + * Lock-free ring buffer for storing pre-allocated event objects + * + *

This implementation is based on LMAX Disruptor (https://github.com/LMAX-Exchange/disruptor) + * and preserves the core ring buffer algorithm for IoTDB's Pipe module. + * + *

Supports multi-producer concurrent access with zero-garbage design. Events are pre-allocated + * and reused, avoiding GC pressure. Uses cache line padding to prevent false sharing. + * + * @param event type + */ +public final class RingBuffer extends RingBufferFields { + /** Initial cursor value for the ring buffer */ + public static final long INITIAL_CURSOR_VALUE = Sequence.INITIAL_VALUE; + + /** + * Right-hand side padding for cache line alignment + * + *

Prevents false sharing by ensuring RingBuffer fields don't share cache lines with following + * objects + */ + protected long p1, p2, p3, p4, p5, p6, p7; + + /** + * Construct a RingBuffer with given factory and sequencer + * + * @param eventFactory factory to create and pre-allocate events + * @param sequencer multi-producer sequencer for sequence management + */ + private RingBuffer(EventFactory eventFactory, MultiProducerSequencer sequencer) { + super(eventFactory, sequencer); + } + + /** + * Create a multi-producer RingBuffer + * + *

Supports concurrent publishing from multiple threads using lock-free CAS operations + * + * @param factory event factory for creating event instances + * @param bufferSize buffer size (must be power of 2) + * @param event type + * @return newly created ring buffer + */ + public static RingBuffer createMultiProducer(EventFactory factory, int bufferSize) { + MultiProducerSequencer sequencer = new MultiProducerSequencer(bufferSize, new Sequence[0]); + return new RingBuffer<>(factory, sequencer); + } + + /** + * Get the event at a specific sequence + * + * @param sequence sequence number to retrieve + * @return event at the given sequence + */ + public E get(long sequence) { + return elementAt(sequence); + } + + /** + * Claim the next sequence for publishing + * + *

Blocks if buffer is full until space becomes available + * + * @return claimed sequence number + */ + public long next() { + return sequencer.next(1); + } + + /** + * Claim next n sequences for batch publishing + * + * @param n number of sequences to claim + * @return highest claimed sequence number + */ + public long next(int n) { + return sequencer.next(n); + } + + /** + * Publish a single sequence + * + *

Makes the event at this sequence visible to consumers + * + * @param sequence sequence to publish + */ + public void publish(long sequence) { + sequencer.publish(sequence); + } + + /** + * Publish a batch of sequences + * + * @param lo lowest sequence in the batch (inclusive) + * @param hi highest sequence in the batch (inclusive) + */ + public void publish(long lo, long hi) { + sequencer.publish(lo, hi); + } + + /** + * Publish event using a translator function + * + *

Provides a higher-level API for publishing events with custom translation logic + * + * @param translator function to populate the event + * @param arg0 argument passed to translator + * @param argument type + */ + public void publishEvent(EventTranslator translator, A arg0) { + publishEvent(translator, arg0, () -> false); + } + + /** + * Publish event using a translator function, or abort if the caller is closing. + * + * @param translator function to populate the event + * @param arg0 argument passed to translator + * @param abortCondition returns {@code true} if the publish should be abandoned + * @param argument type + * @return {@code true} if the event is published, {@code false} if the publish is aborted + */ + public boolean publishEvent( + final EventTranslator translator, final A arg0, final BooleanSupplier abortCondition) { + final long sequence = sequencer.next(1, abortCondition); + if (sequence == Sequence.INITIAL_VALUE) { + return false; + } + translateAndPublish(translator, sequence, arg0); + return true; + } + + /** + * Translate event and publish atomically + * + * @param translator event translator function + * @param sequence claimed sequence number + * @param arg0 argument for translation + * @param argument type + */ + private void translateAndPublish(EventTranslator translator, long sequence, A arg0) { + try { + translator.translateTo(get(sequence), sequence, arg0); + } finally { + sequencer.publish(sequence); + } + } + + /** + * Add gating sequences for consumer tracking + * + *

Gating sequences represent consumer progress and prevent overwriting unprocessed events + * + * @param gatingSequences consumer sequences to track + */ + public void addGatingSequences(Sequence... gatingSequences) { + sequencer.addGatingSequences(gatingSequences); + } + + /** + * Create a sequence barrier for consumers + * + *

Barrier coordinates when events become available for processing + * + * @param sequencesToTrack upstream sequences to wait for + * @return new sequence barrier + */ + public SequenceBarrier newBarrier(Sequence... sequencesToTrack) { + return sequencer.newBarrier(sequencesToTrack); + } + + /** + * Get current producer cursor position + * + * @return current cursor value + */ + public long getCursor() { + return sequencer.getCursor().get(); + } + + /** + * Get the buffer size + * + * @return configured buffer size + */ + public int getBufferSize() { + return bufferSize; + } + + /** + * Get remaining capacity in the buffer + * + * @return number of available slots + */ + public long remainingCapacity() { + return sequencer.remainingCapacity(); + } + + /** + * Function interface for translating data into events + * + * @param event type + * @param argument type + */ + @FunctionalInterface + public interface EventTranslator { + /** + * Translate argument into event + * + * @param event pre-allocated event to populate + * @param sequence sequence number for this event + * @param arg source data + */ + void translateTo(E event, long sequence, A arg); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/Sequence.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/Sequence.java new file mode 100644 index 0000000000000..1f1d3445969f1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/Sequence.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor; + +import java.util.concurrent.atomic.AtomicLong; + +/** Left-hand side padding for cache line alignment */ +class LhsPadding { + protected long p1, p2, p3, p4, p5, p6, p7; +} + +/** Value class holding the actual sequence */ +class Value extends LhsPadding { + protected AtomicLong value = new AtomicLong(); +} + +/** Right-hand side padding for cache line alignment */ +class RhsPadding extends Value { + protected long p9, p10, p11, p12, p13, p14, p15; +} + +/** + * Lock-free sequence counter with cache line padding + * + *

This implementation is based on LMAX Disruptor (https://github.com/LMAX-Exchange/disruptor) + * and preserves the core sequence tracking mechanism for IoTDB's Pipe module. + * + *

Key design features: + * + *

+ */ +public class Sequence extends RhsPadding { + public static final long INITIAL_VALUE = -1L; + + /** Create sequence with initial value -1 */ + public Sequence() { + value.set(INITIAL_VALUE); + } + + /** Volatile read */ + public long get() { + return value.get(); + } + + /** + * Ordered write (store-store barrier only) + * + *

CRITICAL: Cheaper than volatile write, sufficient for most cases + */ + public void set(final long value) { + this.value.set(value); + } + + /** + * CAS operation - CORE for lock-free design + * + * @param expectedValue expected current value + * @param newValue new value + * @return true if successful + */ + public boolean compareAndSet(final long expectedValue, final long newValue) { + return value.compareAndSet(expectedValue, newValue); + } + + /** Atomically increment */ + public long incrementAndGet() { + return addAndGet(1L); + } + + /** Atomically add */ + public long addAndGet(final long increment) { + long currentValue; + long newValue; + + do { + currentValue = get(); + newValue = currentValue + increment; + } while (!compareAndSet(currentValue, newValue)); + + return newValue; + } + + @Override + public String toString() { + return Long.toString(get()); + } + + /** Get minimum sequence from array - CORE utility method */ + public static long getMinimumSequence(final Sequence[] sequences, long minimum) { + for (int i = 0, n = sequences.length; i < n; i++) { + long value = sequences[i].get(); + minimum = Math.min(minimum, value); + } + return minimum; + } + + public static long getMinimumSequence(final Sequence[] sequences) { + return getMinimumSequence(sequences, Long.MAX_VALUE); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/SequenceBarrier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/SequenceBarrier.java new file mode 100644 index 0000000000000..80f41162fc73a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/SequenceBarrier.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor; + +/** + * Sequence barrier for consumer coordination + * + *

This implementation is based on LMAX Disruptor (https://github.com/LMAX-Exchange/disruptor) + * and simplified for IoTDB's Pipe module (removed Alert mechanism - IoTDB doesn't need it). + * + *

Core features preserved from LMAX Disruptor: + * + *

    + *
  • waitFor() logic for waiting sequences + *
  • Scan available buffer for out-of-order publishing + *
+ */ +public class SequenceBarrier { + private final MultiProducerSequencer sequencer; + private final Sequence[] dependentSequences; + + public SequenceBarrier(MultiProducerSequencer sequencer, Sequence[] dependentSequences) { + this.sequencer = sequencer; + this.dependentSequences = dependentSequences != null ? dependentSequences : new Sequence[0]; + } + + /** + * CORE: Wait for sequence to become available (MUST keep logic) + * + * @param sequence sequence to wait for + * @return highest available sequence + * @throws InterruptedException if interrupted + */ + public long waitFor(long sequence) throws InterruptedException { + // Wait for cursor + long availableSequence; + while ((availableSequence = sequencer.getCursor().get()) < sequence) { + Thread.sleep(1); + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException(); + } + } + + // Wait for dependent sequences + if (dependentSequences.length > 0) { + while (Sequence.getMinimumSequence(dependentSequences) < sequence) { + Thread.sleep(1); + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException(); + } + } + } + + // CORE: Scan available buffer for highest continuously published sequence + return sequencer.getHighestPublishedSequence(sequence, availableSequence); + } + + public long getCursor() { + return sequencer.getCursor().get(); + } + + public long getHighestPublishedSequence(long lowerBound, long availableSequence) { + return sequencer.getHighestPublishedSequence(lowerBound, availableSequence); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/SequenceGroups.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/SequenceGroups.java new file mode 100644 index 0000000000000..af5039070f0e4 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/SequenceGroups.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor; + +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; + +/** + * Utility for atomic management of sequence arrays + * + *

This implementation is based on LMAX Disruptor (https://github.com/LMAX-Exchange/disruptor) + * and adapted for IoTDB's Pipe module. + * + *

Provides thread-safe operations for adding and removing sequences from gating sequence arrays + * used to track consumer progress. + */ +final class SequenceGroups { + + /** Field updater for atomic array replacement */ + private static final AtomicReferenceFieldUpdater + SEQUENCE_UPDATER = + AtomicReferenceFieldUpdater.newUpdater( + MultiProducerSequencer.class, Sequence[].class, "gatingSequences"); + + /** + * Atomically add sequences to the gating sequence array + * + *

Uses CAS loop to ensure thread-safe addition even under concurrent modification + * + * @param sequencer the multi-producer sequencer + * @param cursor the current cursor sequence + * @param sequencesToAdd sequences to add + */ + static void addSequences( + final MultiProducerSequencer sequencer, + final Sequence cursor, + final Sequence... sequencesToAdd) { + long cursorSequence; + Sequence[] updatedSequences; + Sequence[] currentSequences; + + do { + currentSequences = sequencer.gatingSequences; + updatedSequences = new Sequence[currentSequences.length + sequencesToAdd.length]; + System.arraycopy(currentSequences, 0, updatedSequences, 0, currentSequences.length); + + cursorSequence = cursor.get(); + + int index = currentSequences.length; + for (Sequence sequence : sequencesToAdd) { + sequence.set(cursorSequence); + updatedSequences[index++] = sequence; + } + } while (!SEQUENCE_UPDATER.compareAndSet(sequencer, currentSequences, updatedSequences)); + + cursorSequence = cursor.get(); + for (Sequence sequence : sequencesToAdd) { + sequence.set(cursorSequence); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java index d6cfa6f6abc6f..9345d25746295 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/listener/PipeInsertionDataNodeListener.java @@ -30,7 +30,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; /** * PipeInsertionEventListener is a singleton in each data node. @@ -41,53 +40,52 @@ * *

All events extracted by this listener will be first published to different * PipeEventDataRegionAssigners (identified by data region id), and then PipeEventDataRegionAssigner - * will filter events and assign them to different PipeRealtimeEventDataRegionExtractors. + * will filter events and assign them to different PipeRealtimeEventDataRegionSources. */ public class PipeInsertionDataNodeListener { private final ConcurrentMap dataRegionId2Assigner = new ConcurrentHashMap<>(); - private final AtomicInteger listenToTsFileExtractorCount = new AtomicInteger(0); - private final AtomicInteger listenToInsertNodeExtractorCount = new AtomicInteger(0); - //////////////////////////// start & stop //////////////////////////// public synchronized void startListenAndAssign( - String dataRegionId, PipeRealtimeDataRegionSource extractor) { - dataRegionId2Assigner - .computeIfAbsent(dataRegionId, o -> new PipeDataRegionAssigner(dataRegionId)) - .startAssignTo(extractor); - - if (extractor.isNeedListenToTsFile()) { - listenToTsFileExtractorCount.incrementAndGet(); - } - if (extractor.isNeedListenToInsertNode()) { - listenToInsertNodeExtractorCount.incrementAndGet(); - } + final String dataRegionId, final PipeRealtimeDataRegionSource source) { + // Keep registration inside compute so the assigner is fully started before it becomes visible + // to concurrent listeners. + dataRegionId2Assigner.compute( + dataRegionId, + (id, assigner) -> { + final PipeDataRegionAssigner actualAssigner = + assigner == null ? new PipeDataRegionAssigner(dataRegionId) : assigner; + actualAssigner.startAssignTo(source); + return actualAssigner; + }); } - public synchronized void stopListenAndAssign( - String dataRegionId, PipeRealtimeDataRegionSource extractor) { - final PipeDataRegionAssigner assigner = dataRegionId2Assigner.get(dataRegionId); - if (assigner == null) { - return; - } + public void stopListenAndAssign( + final String dataRegionId, final PipeRealtimeDataRegionSource source) { + PipeDataRegionAssigner assignerToClose = null; - assigner.stopAssignTo(extractor); + synchronized (this) { + final PipeDataRegionAssigner assigner = dataRegionId2Assigner.get(dataRegionId); + if (assigner == null) { + return; + } - if (extractor.isNeedListenToTsFile()) { - listenToTsFileExtractorCount.decrementAndGet(); - } - if (extractor.isNeedListenToInsertNode()) { - listenToInsertNodeExtractorCount.decrementAndGet(); + assigner.stopAssignTo(source); + + if (assigner.notMoreSourceNeededToBeAssigned()) { + // The removed assigner will is the same as the one referenced by the variable `assigner` + dataRegionId2Assigner.remove(dataRegionId); + // This will help to release the memory occupied by the assigner + assignerToClose = assigner; + } } - if (assigner.notMoreExtractorNeededToBeAssigned()) { - // The removed assigner will is the same as the one referenced by the variable `assigner` - dataRegionId2Assigner.remove(dataRegionId); - // This will help to release the memory occupied by the assigner - assigner.close(); + if (assignerToClose != null) { + // Closing the disruptor may block for a while, so keep it out of the global listener lock. + assignerToClose.close(); } } @@ -95,14 +93,10 @@ public synchronized void stopListenAndAssign( public void listenToTsFile( final String dataRegionId, final TsFileResource tsFileResource, final boolean isLoaded) { - // We don't judge whether listenToTsFileExtractorCount.get() == 0 here on purpose - // because extractors may use tsfile events when some exceptions occur in the - // insert nodes listening process. - final PipeDataRegionAssigner assigner = dataRegionId2Assigner.get(dataRegionId); - // only events from registered data region will be extracted - if (assigner == null) { + // only events from registered data region with tsfile listeners will be extracted + if (assigner == null || !assigner.shouldListenToTsFile()) { return; } @@ -111,15 +105,11 @@ public void listenToTsFile( } public void listenToInsertNode( - String dataRegionId, InsertNode insertNode, TsFileResource tsFileResource) { - if (listenToInsertNodeExtractorCount.get() == 0) { - return; - } - + final String dataRegionId, final InsertNode insertNode, final TsFileResource tsFileResource) { final PipeDataRegionAssigner assigner = dataRegionId2Assigner.get(dataRegionId); - // only events from registered data region will be extracted - if (assigner == null) { + // only events from registered data region with insert listeners will be extracted + if (assigner == null || !assigner.shouldListenToInsertNode()) { return; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/DataNodeClientPoolFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/DataNodeClientPoolFactory.java index b5f5df430129f..102b1d2cbad94 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/DataNodeClientPoolFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/DataNodeClientPoolFactory.java @@ -61,32 +61,4 @@ public GenericKeyedObjectPool createClientPool return clientPool; } } - - public static class ClusterDeletionConfigNodeClientPoolFactory - implements IClientPoolFactory { - - @Override - public GenericKeyedObjectPool createClientPool( - ClientManager manager) { - GenericKeyedObjectPool clientPool = - new GenericKeyedObjectPool<>( - new ConfigNodeClient.Factory( - manager, - new ThriftClientProperty.Builder() - .setConnectionTimeoutMs(conf.getConnectionTimeoutInMS() * 10) - .setRpcThriftCompressionEnabled(conf.isRpcThriftCompressionEnable()) - .setSelectorNumOfAsyncClientManager( - conf.getSelectorNumOfClientManager() / 10 > 0 - ? conf.getSelectorNumOfClientManager() / 10 - : 1) - .build()), - new ClientPoolProperty.Builder() - .setMaxClientNumForEachNode(conf.getMaxClientNumForEachNode()) - .build() - .getConfig()); - ClientManagerMetrics.getInstance() - .registerClientManager(this.getClass().getSimpleName(), clientPool); - return clientPool; - } - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/cn/DnToCnInternalServiceAsyncRequestManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/cn/DnToCnInternalServiceAsyncRequestManager.java index b6a218a732e2b..9ac6b70841136 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/cn/DnToCnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/cn/DnToCnInternalServiceAsyncRequestManager.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.client.request.AsyncRequestRPCHandler; import org.apache.iotdb.commons.client.request.ConfigNodeInternalServiceAsyncRequestManager; import org.apache.iotdb.commons.client.request.TestConnectionUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +36,10 @@ public class DnToCnInternalServiceAsyncRequestManager private static final Logger LOGGER = LoggerFactory.getLogger(DnToCnInternalServiceAsyncRequestManager.class); + public DnToCnInternalServiceAsyncRequestManager() { + super(IoTDBDescriptor.getInstance().getConfig().getSelectorNumOfClientManager()); + } + @Override protected void initActionMapBuilder() { actionMapBuilder.put( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DataNodeExternalServiceAsyncRequestManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DataNodeExternalServiceAsyncRequestManager.java index dd56e1366c03b..29c3f76790716 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DataNodeExternalServiceAsyncRequestManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DataNodeExternalServiceAsyncRequestManager.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.client.request.AsyncRequestContext; import org.apache.iotdb.commons.client.request.AsyncRequestManager; import org.apache.iotdb.commons.client.request.AsyncRequestRPCHandler; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,12 +39,17 @@ public class DataNodeExternalServiceAsyncRequestManager private static final Logger LOGGER = LoggerFactory.getLogger(DataNodeExternalServiceAsyncRequestManager.class); + private DataNodeExternalServiceAsyncRequestManager() { + super(IoTDBDescriptor.getInstance().getConfig().getSelectorNumOfClientManager()); + } + @Override - protected void initClientManager() { + protected void initClientManager(int selectorNumOfAsyncClientManager) { clientManager = new IClientManager.Factory() .createClientManager( - new ClientPoolFactory.AsyncDataNodeExternalServiceClientPoolFactory()); + new ClientPoolFactory.AsyncDataNodeExternalServiceClientPoolFactory( + selectorNumOfAsyncClientManager)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DataNodeMPPServiceAsyncRequestManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DataNodeMPPServiceAsyncRequestManager.java index ab08d83f2645b..e4d190e1571a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DataNodeMPPServiceAsyncRequestManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DataNodeMPPServiceAsyncRequestManager.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.client.request.AsyncRequestContext; import org.apache.iotdb.commons.client.request.AsyncRequestManager; import org.apache.iotdb.commons.client.request.AsyncRequestRPCHandler; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,14 +38,17 @@ public class DataNodeMPPServiceAsyncRequestManager private static final Logger LOGGER = LoggerFactory.getLogger(DataNodeMPPServiceAsyncRequestManager.class); - public DataNodeMPPServiceAsyncRequestManager() {} + public DataNodeMPPServiceAsyncRequestManager() { + super(IoTDBDescriptor.getInstance().getConfig().getSelectorNumOfClientManager()); + } @Override - protected void initClientManager() { + protected void initClientManager(int selectorNumOfAsyncClientManager) { clientManager = new IClientManager.Factory() .createClientManager( - new ClientPoolFactory.AsyncDataNodeMPPDataExchangeServiceClientPoolFactory()); + new ClientPoolFactory.AsyncDataNodeMPPDataExchangeServiceClientPoolFactory( + selectorNumOfAsyncClientManager)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DnToDnInternalServiceAsyncRequestManager.java index 88766458b649d..c2ca62cf15d48 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DnToDnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/dn/DnToDnInternalServiceAsyncRequestManager.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.client.request.AsyncRequestContext; import org.apache.iotdb.commons.client.request.AsyncRequestRPCHandler; import org.apache.iotdb.commons.client.request.DataNodeInternalServiceRequestManager; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,6 +33,10 @@ public class DnToDnInternalServiceAsyncRequestManager private static final Logger LOGGER = LoggerFactory.getLogger(DnToDnInternalServiceAsyncRequestManager.class); + private DnToDnInternalServiceAsyncRequestManager() { + super(IoTDBDescriptor.getInstance().getConfig().getSelectorNumOfClientManager()); + } + @Override protected void initActionMapBuilder() { actionMapBuilder.put( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/OperationType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/OperationType.java index 065a568353f7a..5838dac25bfba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/OperationType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/OperationType.java @@ -50,7 +50,8 @@ public enum OperationType { CREATE_SCHEMA_TEMPLATE("createSchemaTemplate"), CHECK_AUTHORITY("checkAuthority"), EXECUTE_NON_QUERY_PLAN("executeNonQueryPlan"), - QUERY_LATENCY("queryLatency"); + QUERY_LATENCY("queryLatency"), + DISPATCH_FRAGMENT_INSTANCE("dispatchFragmentInstance"); private final String name; OperationType(String name) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 2ae50d3605718..3d34def409f64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -25,6 +25,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.common.rpc.thrift.TShowConfigurationTemplateResp; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; @@ -1008,7 +1009,8 @@ public TSExecuteStatementResp executeFastLastDataQueryForOneDeviceV2( if (timeValuePair == null) { allCached = false; break; - } else if (timeValuePair.getValue() == null) { + } else if (timeValuePair == DeviceLastCache.EMPTY_TIME_VALUE_PAIR + || timeValuePair.getValue() == null) { // there is no data for this sensor if (!canUseNullEntry) { allCached = false; @@ -2745,24 +2747,59 @@ public TSStatus createTimeseriesUsingSchemaTemplate(TCreateTimeseriesUsingSchema @Override public TSStatus handshake(final TSyncIdentityInfo info) throws TException { - return PipeDataNodeAgent.receiver() - .legacy() - .handshake( - info, - SESSION_MANAGER.getCurrSession().getClientAddress(), - partitionFetcher, - schemaFetcher); + try { + final TSStatus status = checkLegacyPipeReceiverPermission(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return status; + } + return PipeDataNodeAgent.receiver() + .legacy() + .handshake( + info, + SESSION_MANAGER.getCurrSession().getClientAddress(), + partitionFetcher, + schemaFetcher); + } finally { + SESSION_MANAGER.updateIdleTime(); + } } @Override public TSStatus sendPipeData(final ByteBuffer buff) throws TException { - return PipeDataNodeAgent.receiver().legacy().transportPipeData(buff); + try { + final TSStatus status = checkLegacyPipeReceiverPermission(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return status; + } + return PipeDataNodeAgent.receiver().legacy().transportPipeData(buff); + } finally { + SESSION_MANAGER.updateIdleTime(); + } } @Override public TSStatus sendFile(final TSyncTransportMetaInfo metaInfo, final ByteBuffer buff) throws TException { - return PipeDataNodeAgent.receiver().legacy().transportFile(metaInfo, buff); + try { + final TSStatus status = checkLegacyPipeReceiverPermission(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return status; + } + return PipeDataNodeAgent.receiver().legacy().transportFile(metaInfo, buff); + } finally { + SESSION_MANAGER.updateIdleTime(); + } + } + + private TSStatus checkLegacyPipeReceiverPermission() { + final IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); + if (!SESSION_MANAGER.checkLogin(clientSession)) { + return getNotLoggedInStatus(); + } + return AuthorityChecker.getTSStatus( + AuthorityChecker.checkSystemPermission( + clientSession.getUsername(), PrivilegeType.USE_PIPE.ordinal()), + PrivilegeType.USE_PIPE); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index e265994a370ee..f5c84a2375feb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -926,7 +926,7 @@ public TCheckTimeSeriesExistenceResp checkTimeSeriesExistence(TCheckTimeSeriesEx for (PartialPath pattern : filteredPatternTree.getAllPathPatterns()) { ISchemaSource schemaSource = SchemaSourceFactory.getTimeSeriesSchemaCountSource( - pattern, false, null, null, SchemaConstant.ALL_MATCH_SCOPE); + pattern, false, null, null, SchemaConstant.ALL_MATCH_SCOPE, true); try (ISchemaReader schemaReader = schemaSource.getSchemaReader(schemaRegion)) { if (schemaReader.hasNext()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 7479e832a9016..13773c150906c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -100,6 +100,12 @@ public class MPPQueryContext { private boolean userQuery = false; + /** + * When true (e.g. SHOW QUERIES), operator and exchange memory may use fallback when pool is + * insufficient. Set from analysis via {@link #setNeedSetHighestPriority(boolean)}. + */ + private boolean needSetHighestPriority = false; + @TestOnly public MPPQueryContext(QueryId queryId) { this.queryId = queryId; @@ -406,6 +412,14 @@ public void setUserQuery(boolean userQuery) { this.userQuery = userQuery; } + public boolean needSetHighestPriority() { + return needSetHighestPriority; + } + + public void setNeedSetHighestPriority(boolean needSetHighestPriority) { + this.needSetHighestPriority = needSetHighestPriority; + } + public String getClientHostName() { if (session == null || session.getCliHostname() == null) { return "UNKNOWN"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java index 9225b97fc70ae..ecad26bac004c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java @@ -28,7 +28,11 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; +import java.util.Set; import static org.apache.iotdb.db.queryengine.execution.operator.schema.source.TimeSeriesSchemaSource.mapToString; @@ -42,11 +46,70 @@ public class TimeseriesContext { private final String deadband; private final String deadbandParameters; + private final String database; + private final int activeCountMultiplier; + private final boolean logicalView; + private final Map activeLogicalViewContextMap; public TimeseriesContext(IMeasurementSchemaInfo schemaInfo) { + this(schemaInfo, 1, Collections.emptyMap()); + } + + public TimeseriesContext( + IMeasurementSchemaInfo schemaInfo, + int activeCountMultiplier, + Set activeLogicalViewCountSet) { + this(schemaInfo, activeCountMultiplier, createLogicalViewContextMap(activeLogicalViewCountSet)); + } + + public TimeseriesContext( + IMeasurementSchemaInfo schemaInfo, + int activeCountMultiplier, + Map activeLogicalViewContextMap) { this.dataType = schemaInfo.getSchema().getType().toString(); - this.encoding = schemaInfo.getSchema().getEncodingType().toString(); - this.compression = schemaInfo.getSchema().getCompressor().toString(); + this.logicalView = schemaInfo.isLogicalView(); + if (logicalView) { + this.encoding = null; + this.compression = null; + } else { + this.encoding = schemaInfo.getSchema().getEncodingType().toString(); + this.compression = schemaInfo.getSchema().getCompressor().toString(); + } + this.alias = schemaInfo.getAlias(); + this.tags = mapToString(schemaInfo.getTagMap()); + this.attributes = mapToString(schemaInfo.getAttributeMap()); + Pair deadbandInfo = + MetaUtils.parseDeadbandInfo(schemaInfo.getSchema().getProps()); + this.deadband = deadbandInfo.left; + this.deadbandParameters = deadbandInfo.right; + this.database = null; + this.activeCountMultiplier = activeCountMultiplier; + this.activeLogicalViewContextMap = new HashMap<>(activeLogicalViewContextMap); + } + + public TimeseriesContext( + IMeasurementSchemaInfo schemaInfo, + String dataType, + int activeCountMultiplier, + Map activeLogicalViewContextMap) { + this(schemaInfo, dataType, null, activeCountMultiplier, activeLogicalViewContextMap); + } + + public TimeseriesContext( + IMeasurementSchemaInfo schemaInfo, + String dataType, + String database, + int activeCountMultiplier, + Map activeLogicalViewContextMap) { + this.dataType = dataType; + this.logicalView = schemaInfo.isLogicalView(); + if (logicalView) { + this.encoding = null; + this.compression = null; + } else { + this.encoding = schemaInfo.getSchema().getEncodingType().toString(); + this.compression = schemaInfo.getSchema().getCompressor().toString(); + } this.alias = schemaInfo.getAlias(); this.tags = mapToString(schemaInfo.getTagMap()); this.attributes = mapToString(schemaInfo.getAttributeMap()); @@ -54,6 +117,9 @@ public TimeseriesContext(IMeasurementSchemaInfo schemaInfo) { MetaUtils.parseDeadbandInfo(schemaInfo.getSchema().getProps()); this.deadband = deadbandInfo.left; this.deadbandParameters = deadbandInfo.right; + this.database = database; + this.activeCountMultiplier = activeCountMultiplier; + this.activeLogicalViewContextMap = new HashMap<>(activeLogicalViewContextMap); } public String getDataType() { @@ -88,6 +154,26 @@ public String getDeadband() { return deadband; } + public String getDatabase() { + return database; + } + + public int getActiveCountMultiplier() { + return activeCountMultiplier; + } + + public Set getActiveLogicalViewCountSet() { + return activeLogicalViewContextMap.keySet(); + } + + public Map getActiveLogicalViewContextMap() { + return activeLogicalViewContextMap; + } + + public boolean isLogicalView() { + return logicalView; + } + public TimeseriesContext( String dataType, String alias, @@ -97,6 +183,113 @@ public TimeseriesContext( String attributes, String deadband, String deadbandParameters) { + this( + dataType, + alias, + encoding, + compression, + tags, + attributes, + deadband, + deadbandParameters, + 1, + false, + null, + Collections.emptyMap()); + } + + public TimeseriesContext( + String dataType, + String alias, + String encoding, + String compression, + String tags, + String attributes, + String deadband, + String deadbandParameters, + int activeCountMultiplier, + Set activeLogicalViewCountSet) { + this( + dataType, + alias, + encoding, + compression, + tags, + attributes, + deadband, + deadbandParameters, + activeCountMultiplier, + false, + null, + createLogicalViewContextMap(activeLogicalViewCountSet)); + } + + public TimeseriesContext( + String dataType, + String alias, + String encoding, + String compression, + String tags, + String attributes, + String deadband, + String deadbandParameters, + int activeCountMultiplier, + Map activeLogicalViewContextMap) { + this( + dataType, + alias, + encoding, + compression, + tags, + attributes, + deadband, + deadbandParameters, + activeCountMultiplier, + false, + null, + activeLogicalViewContextMap); + } + + public TimeseriesContext( + String dataType, + String alias, + String encoding, + String compression, + String tags, + String attributes, + String deadband, + String deadbandParameters, + int activeCountMultiplier, + boolean logicalView, + Map activeLogicalViewContextMap) { + this( + dataType, + alias, + encoding, + compression, + tags, + attributes, + deadband, + deadbandParameters, + activeCountMultiplier, + logicalView, + null, + activeLogicalViewContextMap); + } + + public TimeseriesContext( + String dataType, + String alias, + String encoding, + String compression, + String tags, + String attributes, + String deadband, + String deadbandParameters, + int activeCountMultiplier, + boolean logicalView, + String database, + Map activeLogicalViewContextMap) { this.dataType = dataType; this.alias = alias; this.encoding = encoding; @@ -105,6 +298,44 @@ public TimeseriesContext( this.attributes = attributes; this.deadband = deadband; this.deadbandParameters = deadbandParameters; + this.database = database; + this.activeCountMultiplier = activeCountMultiplier; + this.logicalView = logicalView; + this.activeLogicalViewContextMap = new HashMap<>(activeLogicalViewContextMap); + } + + private static Map createLogicalViewContextMap( + Set activeLogicalViewCountSet) { + if (activeLogicalViewCountSet.isEmpty()) { + return Collections.emptyMap(); + } + Map activeLogicalViewContextMap = new HashMap<>(); + for (String logicalView : activeLogicalViewCountSet) { + activeLogicalViewContextMap.put( + logicalView, + new TimeseriesContext( + null, null, null, null, null, null, null, null, 1, true, Collections.emptyMap())); + } + return activeLogicalViewContextMap; + } + + public TimeseriesContext mergeActiveCount(TimeseriesContext that) { + Map mergedActiveLogicalViewContextMap = + new HashMap<>(activeLogicalViewContextMap); + mergedActiveLogicalViewContextMap.putAll(that.activeLogicalViewContextMap); + return new TimeseriesContext( + dataType, + alias, + encoding, + compression, + tags, + attributes, + deadband, + deadbandParameters, + activeCountMultiplier + that.activeCountMultiplier, + logicalView, + database, + mergedActiveLogicalViewContextMap); } public void serializeAttributes(ByteBuffer byteBuffer) { @@ -116,6 +347,14 @@ public void serializeAttributes(ByteBuffer byteBuffer) { ReadWriteIOUtils.write(attributes, byteBuffer); ReadWriteIOUtils.write(deadband, byteBuffer); ReadWriteIOUtils.write(deadbandParameters, byteBuffer); + ReadWriteIOUtils.write(database, byteBuffer); + ReadWriteIOUtils.write(activeCountMultiplier, byteBuffer); + ReadWriteIOUtils.write(logicalView, byteBuffer); + ReadWriteIOUtils.write(activeLogicalViewContextMap.size(), byteBuffer); + for (Map.Entry entry : activeLogicalViewContextMap.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), byteBuffer); + entry.getValue().serializeAttributes(byteBuffer); + } } public void serializeAttributes(DataOutputStream stream) throws IOException { @@ -127,6 +366,14 @@ public void serializeAttributes(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(attributes, stream); ReadWriteIOUtils.write(deadband, stream); ReadWriteIOUtils.write(deadbandParameters, stream); + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(activeCountMultiplier, stream); + ReadWriteIOUtils.write(logicalView, stream); + ReadWriteIOUtils.write(activeLogicalViewContextMap.size(), stream); + for (Map.Entry entry : activeLogicalViewContextMap.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), stream); + entry.getValue().serializeAttributes(stream); + } } public static TimeseriesContext deserialize(ByteBuffer buffer) { @@ -138,8 +385,28 @@ public static TimeseriesContext deserialize(ByteBuffer buffer) { String attributes = ReadWriteIOUtils.readString(buffer); String deadband = ReadWriteIOUtils.readString(buffer); String deadbandParameters = ReadWriteIOUtils.readString(buffer); + String database = ReadWriteIOUtils.readString(buffer); + int activeCountMultiplier = ReadWriteIOUtils.readInt(buffer); + boolean logicalView = ReadWriteIOUtils.readBool(buffer); + int activeLogicalViewContextMapSize = ReadWriteIOUtils.readInt(buffer); + Map activeLogicalViewContextMap = new HashMap<>(); + for (int i = 0; i < activeLogicalViewContextMapSize; i++) { + activeLogicalViewContextMap.put( + ReadWriteIOUtils.readString(buffer), TimeseriesContext.deserialize(buffer)); + } return new TimeseriesContext( - dataType, alias, encoding, compression, tags, attributes, deadband, deadbandParameters); + dataType, + alias, + encoding, + compression, + tags, + attributes, + deadband, + deadbandParameters, + activeCountMultiplier, + logicalView, + database, + activeLogicalViewContextMap); } @Override @@ -154,18 +421,33 @@ public boolean equals(Object obj) { boolean res = Objects.equals(dataType, that.dataType) && Objects.equals(alias, that.alias) - && encoding.equals(that.encoding) + && Objects.equals(encoding, that.encoding) && Objects.equals(compression, that.compression) && Objects.equals(tags, that.tags) && Objects.equals(attributes, that.attributes) && Objects.equals(deadband, that.deadband) - && Objects.equals(deadbandParameters, that.deadbandParameters); + && Objects.equals(deadbandParameters, that.deadbandParameters) + && Objects.equals(database, that.database) + && activeCountMultiplier == that.activeCountMultiplier + && logicalView == that.logicalView + && Objects.equals(activeLogicalViewContextMap, that.activeLogicalViewContextMap); return res; } @Override public int hashCode() { return Objects.hash( - dataType, alias, encoding, compression, tags, attributes, deadband, deadbandParameters); + dataType, + alias, + encoding, + compression, + tags, + attributes, + deadband, + deadbandParameters, + database, + activeCountMultiplier, + logicalView, + activeLogicalViewContextMap); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriver.java index c9c41f5b993ab..6caad1a175d27 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriver.java @@ -31,6 +31,7 @@ import java.util.List; +import static com.google.common.base.Throwables.throwIfUnchecked; import static org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet.QUERY_RESOURCE_INIT; import static org.apache.iotdb.db.storageengine.dataregion.VirtualDataRegion.UNFINISHED_QUERY_DATA_SOURCE; @@ -68,7 +69,9 @@ protected boolean init(SettableFuture blockedFuture) { "Failed to do the initialization for driver {} ", driverContext.getDriverTaskID(), t); driverContext.failed(t); blockedFuture.setException(t); - return false; + throwIfUnchecked(t); + // should never happen + throw new AssertionError(t); } } return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java index ac0b411109089..61dc05d4bfd34 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.queryengine.exception.exchange.GetTsBlockFromClosedOrAbortedChannelException; import org.apache.iotdb.db.queryengine.execution.driver.DriverContext; import org.apache.iotdb.db.queryengine.execution.exchange.sink.DownStreamChannelIndex; @@ -656,7 +657,11 @@ private synchronized ISinkChannel createLocalSinkChannel( } queue = new SharedTsBlockQueue( - localFragmentInstanceId, localPlanNodeId, localMemoryManager, executorService); + localFragmentInstanceId, + localPlanNodeId, + localMemoryManager, + executorService, + instanceContext.isHighestPriority()); } return new LocalSinkChannel( @@ -680,7 +685,8 @@ public ISinkChannel createLocalSinkChannelForPipeline( driverContext.getDriverTaskID().getFragmentInstanceId().toThrift(), planNodeId, localMemoryManager, - executorService); + executorService, + driverContext.getFragmentInstanceContext().isHighestPriority()); queue.allowAddingTsBlock(); return new LocalSinkChannel( queue, @@ -718,6 +724,7 @@ private ISinkChannel createSinkChannel( tsBlockSerdeFactory.get(), new ISinkChannelListenerImpl( localFragmentInstanceId, instanceContext, instanceContext::failed, cnt), + instanceContext.isHighestPriority(), mppDataExchangeServiceClientManager); } @@ -802,6 +809,7 @@ public ISourceHandle createLocalSourceHandleForPipeline( context.getDriverTaskID().toString()); } + @TestOnly public synchronized ISourceHandle createLocalSourceHandleForFragment( TFragmentInstanceId localFragmentInstanceId, String localPlanNodeId, @@ -809,6 +817,24 @@ public synchronized ISourceHandle createLocalSourceHandleForFragment( TFragmentInstanceId remoteFragmentInstanceId, int index, IMPPDataExchangeManagerCallback onFailureCallback) { + return createLocalSourceHandleForFragment( + localFragmentInstanceId, + localPlanNodeId, + remotePlanNodeId, + remoteFragmentInstanceId, + index, + onFailureCallback, + false); + } + + public synchronized ISourceHandle createLocalSourceHandleForFragment( + TFragmentInstanceId localFragmentInstanceId, + String localPlanNodeId, + String remotePlanNodeId, + TFragmentInstanceId remoteFragmentInstanceId, + int index, + IMPPDataExchangeManagerCallback onFailureCallback, + boolean isHighestPriority) { if (sourceHandles.containsKey(localFragmentInstanceId) && sourceHandles.get(localFragmentInstanceId).containsKey(localPlanNodeId)) { throw new IllegalStateException( @@ -840,7 +866,11 @@ public synchronized ISourceHandle createLocalSourceHandleForFragment( } queue = new SharedTsBlockQueue( - remoteFragmentInstanceId, remotePlanNodeId, localMemoryManager, executorService); + remoteFragmentInstanceId, + remotePlanNodeId, + localMemoryManager, + executorService, + isHighestPriority); } LocalSourceHandle localSourceHandle = new LocalSourceHandle( @@ -854,6 +884,7 @@ public synchronized ISourceHandle createLocalSourceHandleForFragment( return localSourceHandle; } + @TestOnly @Override public ISourceHandle createSourceHandle( TFragmentInstanceId localFragmentInstanceId, @@ -862,6 +893,24 @@ public ISourceHandle createSourceHandle( TEndPoint remoteEndpoint, TFragmentInstanceId remoteFragmentInstanceId, IMPPDataExchangeManagerCallback onFailureCallback) { + return createSourceHandle( + localFragmentInstanceId, + localPlanNodeId, + indexOfUpstreamSinkHandle, + remoteEndpoint, + remoteFragmentInstanceId, + onFailureCallback, + false); + } + + public ISourceHandle createSourceHandle( + TFragmentInstanceId localFragmentInstanceId, + String localPlanNodeId, + int indexOfUpstreamSinkHandle, + TEndPoint remoteEndpoint, + TFragmentInstanceId remoteFragmentInstanceId, + IMPPDataExchangeManagerCallback onFailureCallback, + boolean isHighestPriority) { Map sourceHandleMap = sourceHandles.get(localFragmentInstanceId); if (sourceHandleMap != null && sourceHandleMap.containsKey(localPlanNodeId)) { throw new IllegalStateException( @@ -891,6 +940,7 @@ public ISourceHandle createSourceHandle( executorService, tsBlockSerdeFactory.get(), new SourceHandleListenerImpl(onFailureCallback), + isHighestPriority, mppDataExchangeServiceClientManager); sourceHandles .computeIfAbsent(localFragmentInstanceId, key -> new ConcurrentHashMap<>()) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java index 555cf9efe5ad9..f4c21e2fdfa91 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java @@ -19,11 +19,13 @@ package org.apache.iotdb.db.queryengine.execution.exchange; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.execution.exchange.sink.LocalSinkChannel; import org.apache.iotdb.db.queryengine.execution.exchange.source.LocalSourceHandle; import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager; +import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool.MemoryReservationResult; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; import com.google.common.util.concurrent.ListenableFuture; @@ -62,7 +64,7 @@ public class SharedTsBlockQueue { private long bufferRetainedSizeInBytes = 0L; - private final Queue queue = new LinkedList<>(); + private final Queue> queue = new LinkedList<>(); private SettableFuture blocked = SettableFuture.create(); @@ -82,17 +84,28 @@ public class SharedTsBlockQueue { private long maxBytesCanReserve = IoTDBDescriptor.getInstance().getConfig().getMaxBytesPerFragmentInstance(); + private final boolean isHighestPriority; private volatile Throwable abortedCause = null; // used for SharedTsBlockQueue listener private final ExecutorService executorService; + @TestOnly public SharedTsBlockQueue( TFragmentInstanceId fragmentInstanceId, String planNodeId, LocalMemoryManager localMemoryManager, ExecutorService executorService) { + this(fragmentInstanceId, planNodeId, localMemoryManager, executorService, false); + } + + public SharedTsBlockQueue( + TFragmentInstanceId fragmentInstanceId, + String planNodeId, + LocalMemoryManager localMemoryManager, + ExecutorService executorService, + boolean isHighestPriority) { this.localFragmentInstanceId = Validate.notNull(fragmentInstanceId, "fragment instance ID cannot be null"); this.fullFragmentInstanceId = @@ -101,6 +114,7 @@ public SharedTsBlockQueue( this.localMemoryManager = Validate.notNull(localMemoryManager, "local memory manager cannot be null"); this.executorService = Validate.notNull(executorService, "ExecutorService can not be null."); + this.isHighestPriority = isHighestPriority; } public boolean hasNoMoreTsBlocks() { @@ -195,15 +209,18 @@ public TsBlock remove() { } throw new IllegalStateException("queue has been destroyed"); } - TsBlock tsBlock = queue.remove(); - localMemoryManager - .getQueryPool() - .free( - localFragmentInstanceId.getQueryId(), - fullFragmentInstanceId, - localPlanNodeId, - tsBlock.getSizeInBytes()); - bufferRetainedSizeInBytes -= tsBlock.getSizeInBytes(); + Pair tsBlockWithReservedBytes = queue.remove(); + long reservedBytes = tsBlockWithReservedBytes.right; + if (reservedBytes > 0) { + localMemoryManager + .getQueryPool() + .free( + localFragmentInstanceId.getQueryId(), + fullFragmentInstanceId, + localPlanNodeId, + reservedBytes); + bufferRetainedSizeInBytes -= reservedBytes; + } // Every time LocalSourceHandle consumes a TsBlock, it needs to send the event // to // corresponding LocalSinkChannel. @@ -213,7 +230,7 @@ public TsBlock remove() { if (blocked.isDone() && queue.isEmpty() && !noMoreTsBlocks) { blocked = SettableFuture.create(); } - return tsBlock; + return tsBlockWithReservedBytes.left; } /** @@ -236,20 +253,22 @@ public ListenableFuture add(TsBlock tsBlock) { localFragmentInstanceId.queryId, fullFragmentInstanceId, localPlanNodeId); alreadyRegistered = true; } - Pair, Boolean> pair = + MemoryReservationResult reserveResult = localMemoryManager .getQueryPool() - .reserve( + .reserveWithPriority( localFragmentInstanceId.getQueryId(), fullFragmentInstanceId, localPlanNodeId, tsBlock.getSizeInBytes(), - maxBytesCanReserve); - blockedOnMemory = pair.left; - bufferRetainedSizeInBytes += tsBlock.getSizeInBytes(); + maxBytesCanReserve, + isHighestPriority); + blockedOnMemory = reserveResult.getFuture(); + long reservedBytes = reserveResult.getReservedBytes(); + bufferRetainedSizeInBytes += reservedBytes; // reserve memory failed, we should wait until there is enough memory - if (!Boolean.TRUE.equals(pair.right)) { + if (!reserveResult.isReserveSuccess()) { SettableFuture channelBlocked = SettableFuture.create(); blockedOnMemory.addListener( () -> { @@ -264,7 +283,7 @@ public ListenableFuture add(TsBlock tsBlock) { channelBlocked.set(null); return; } - queue.add(tsBlock); + queue.add(new Pair<>(tsBlock, reservedBytes)); if (!blocked.isDone()) { blocked.set(null); } @@ -281,7 +300,7 @@ public ListenableFuture add(TsBlock tsBlock) { executorService); return channelBlocked; } else { // reserve memory succeeded, add the TsBlock directly - queue.add(tsBlock); + queue.add(new Pair<>(tsBlock, reservedBytes)); if (!blocked.isDone()) { blocked.set(null); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java index ca6fdadc993e6..47ce6128fb76a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.exception.exchange.GetTsBlockFromClosedOrAbortedChannelException; import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeManager.SinkListener; import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager; +import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool.MemoryReservationResult; import org.apache.iotdb.db.queryengine.metric.DataExchangeCostMetricSet; import org.apache.iotdb.db.queryengine.metric.DataExchangeCountMetricSet; import org.apache.iotdb.db.utils.SetThreadName; @@ -119,6 +120,8 @@ public class SinkChannel implements ISinkChannel { private long maxBytesCanReserve = IoTDBDescriptor.getInstance().getConfig().getMaxBytesPerFragmentInstance(); + private final boolean isHighestPriority; + private static final DataExchangeCostMetricSet DATA_EXCHANGE_COST_METRIC_SET = DataExchangeCostMetricSet.getInstance(); private static final DataExchangeCountMetricSet DATA_EXCHANGE_COUNT_METRIC_SET = @@ -128,6 +131,34 @@ public class SinkChannel implements ISinkChannel { RamUsageEstimator.shallowSizeOfInstance(SinkChannel.class) + RamUsageEstimator.shallowSizeOfInstance(TFragmentInstanceId.class) * 2; + @TestOnly + @SuppressWarnings("squid:S107") + public SinkChannel( + TEndPoint remoteEndpoint, + TFragmentInstanceId remoteFragmentInstanceId, + String remotePlanNodeId, + String localPlanNodeId, + TFragmentInstanceId localFragmentInstanceId, + LocalMemoryManager localMemoryManager, + ExecutorService executorService, + TsBlockSerde serde, + SinkListener sinkListener, + IClientManager + mppDataExchangeServiceClientManager) { + this( + remoteEndpoint, + remoteFragmentInstanceId, + remotePlanNodeId, + localPlanNodeId, + localFragmentInstanceId, + localMemoryManager, + executorService, + serde, + sinkListener, + false, + mppDataExchangeServiceClientManager); + } + @SuppressWarnings("squid:S107") public SinkChannel( TEndPoint remoteEndpoint, @@ -139,6 +170,7 @@ public SinkChannel( ExecutorService executorService, TsBlockSerde serde, SinkListener sinkListener, + boolean isHighestPriority, IClientManager mppDataExchangeServiceClientManager) { this.remoteEndpoint = Validate.notNull(remoteEndpoint, "remoteEndPoint can not be null."); @@ -155,6 +187,7 @@ public SinkChannel( this.executorService = Validate.notNull(executorService, "executorService can not be null."); this.serde = Validate.notNull(serde, "serde can not be null."); this.sinkListener = Validate.notNull(sinkListener, "sinkListener can not be null."); + this.isHighestPriority = isHighestPriority; this.mppDataExchangeServiceClientManager = mppDataExchangeServiceClientManager; this.retryIntervalInMs = DEFAULT_RETRY_INTERVAL_IN_MS; this.threadName = @@ -204,21 +237,22 @@ public synchronized void send(TsBlock tsBlock) { long sizeInBytes = tsBlock.getSizeInBytes(); int startSequenceId; startSequenceId = nextSequenceId; - blocked = + MemoryReservationResult reserveResult = localMemoryManager .getQueryPool() - .reserve( + .reserveWithPriority( localFragmentInstanceId.getQueryId(), fullFragmentInstanceId, localPlanNodeId, sizeInBytes, - maxBytesCanReserve) - .left; - bufferRetainedSizeInBytes += sizeInBytes; + maxBytesCanReserve, + isHighestPriority); + blocked = reserveResult.getFuture(); + bufferRetainedSizeInBytes += reserveResult.getReservedBytes(); sequenceIdToTsBlock.put(nextSequenceId, new Pair<>(tsBlock, currentTsBlockSize)); nextSequenceId += 1; - currentTsBlockSize = sizeInBytes; + currentTsBlockSize = reserveResult.getReservedBytes(); submitSendNewDataBlockEventTask(startSequenceId, ImmutableList.of(sizeInBytes)); } finally { @@ -433,19 +467,21 @@ public synchronized void open() { return; } // SinkChannel is opened when ShuffleSinkHandle choose it as the next channel - this.blocked = + MemoryReservationResult reserveResult = localMemoryManager .getQueryPool() - .reserve( + .reserveWithPriority( localFragmentInstanceId.getQueryId(), fullFragmentInstanceId, localPlanNodeId, DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES, - maxBytesCanReserve) // actually we only know maxBytesCanReserve after - // the handle is created, so we use DEFAULT here. It is ok to use DEFAULT here because - // at first this SinkChannel has not reserved memory. - .left; - this.bufferRetainedSizeInBytes = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES; + maxBytesCanReserve, + isHighestPriority); // actually we only know maxBytesCanReserve after + // the handle is created, so we use DEFAULT here. It is ok to use DEFAULT here because + // at first this SinkChannel has not reserved memory. + this.blocked = reserveResult.getFuture(); + this.bufferRetainedSizeInBytes = reserveResult.getReservedBytes(); + this.currentTsBlockSize = reserveResult.getReservedBytes(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java index 2d1a06fcd3217..0e0b94094b566 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/SourceHandle.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeManager.SourceHandleListener; import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager; +import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool.MemoryReservationResult; import org.apache.iotdb.db.queryengine.metric.DataExchangeCostMetricSet; import org.apache.iotdb.db.queryengine.metric.DataExchangeCountMetricSet; import org.apache.iotdb.db.utils.SetThreadName; @@ -115,6 +116,8 @@ public class SourceHandle implements ISourceHandle { */ private boolean canGetTsBlockFromRemote = false; + private final boolean isHighestPriority; + private static final DataExchangeCostMetricSet DATA_EXCHANGE_COST_METRIC_SET = DataExchangeCostMetricSet.getInstance(); private static final DataExchangeCountMetricSet DATA_EXCHANGE_COUNT_METRIC_SET = @@ -124,6 +127,7 @@ public class SourceHandle implements ISourceHandle { RamUsageEstimator.shallowSizeOfInstance(SourceHandle.class) + RamUsageEstimator.shallowSizeOfInstance(TFragmentInstanceId.class) * 2; + @TestOnly @SuppressWarnings("squid:S107") public SourceHandle( TEndPoint remoteEndpoint, @@ -137,6 +141,34 @@ public SourceHandle( SourceHandleListener sourceHandleListener, IClientManager mppDataExchangeServiceClientManager) { + this( + remoteEndpoint, + remoteFragmentInstanceId, + localFragmentInstanceId, + localPlanNodeId, + indexOfUpstreamSinkHandle, + localMemoryManager, + executorService, + serde, + sourceHandleListener, + false, + mppDataExchangeServiceClientManager); + } + + @SuppressWarnings("squid:S107") + public SourceHandle( + TEndPoint remoteEndpoint, + TFragmentInstanceId remoteFragmentInstanceId, + TFragmentInstanceId localFragmentInstanceId, + String localPlanNodeId, + int indexOfUpstreamSinkHandle, + LocalMemoryManager localMemoryManager, + ExecutorService executorService, + TsBlockSerde serde, + SourceHandleListener sourceHandleListener, + boolean isHighestPriority, + IClientManager + mppDataExchangeServiceClientManager) { this.remoteEndpoint = Validate.notNull(remoteEndpoint, "remoteEndpoint can not be null."); this.remoteFragmentInstanceId = Validate.notNull(remoteFragmentInstanceId, "remoteFragmentInstanceId can not be null."); @@ -152,6 +184,7 @@ public SourceHandle( this.serde = Validate.notNull(serde, "serde can not be null."); this.sourceHandleListener = Validate.notNull(sourceHandleListener, "sourceHandleListener can not be null."); + this.isHighestPriority = isHighestPriority; this.bufferRetainedSizeInBytes = 0L; this.mppDataExchangeServiceClientManager = mppDataExchangeServiceClientManager; this.retryIntervalInMs = DEFAULT_RETRY_INTERVAL_IN_MS; @@ -192,19 +225,24 @@ public synchronized ByteBuffer getSerializedTsBlock() { if (tsBlock == null) { return null; } - long retainedSize = sequenceIdToDataBlockSize.remove(currSequenceId); + Long retainedSize = sequenceIdToDataBlockSize.remove(currSequenceId); + if (retainedSize == null) { + throw new IllegalStateException("Reserved data block size is null."); + } if (LOGGER.isDebugEnabled()) { LOGGER.debug("[GetTsBlockFromBuffer] sequenceId:{}, size:{}", currSequenceId, retainedSize); } currSequenceId += 1; - bufferRetainedSizeInBytes -= retainedSize; - localMemoryManager - .getQueryPool() - .free( - localFragmentInstanceId.getQueryId(), - fullFragmentInstanceId, - localPlanNodeId, - retainedSize); + if (retainedSize > 0) { + bufferRetainedSizeInBytes -= retainedSize; + localMemoryManager + .getQueryPool() + .free( + localFragmentInstanceId.getQueryId(), + fullFragmentInstanceId, + localPlanNodeId, + retainedSize); + } if (sequenceIdToTsBlock.isEmpty() && !isFinished()) { if (LOGGER.isDebugEnabled()) { @@ -241,18 +279,24 @@ private synchronized void trySubmitGetDataBlocksTask() { if (bytesToReserve == null) { throw new IllegalStateException("Data block size is null."); } - pair = + MemoryReservationResult reserveResult = localMemoryManager .getQueryPool() - .reserve( + .reserveWithPriority( localFragmentInstanceId.getQueryId(), fullFragmentInstanceId, localPlanNodeId, bytesToReserve, - maxBytesCanReserve); - bufferRetainedSizeInBytes += bytesToReserve; + maxBytesCanReserve, + isHighestPriority); + pair = new Pair<>(reserveResult.getFuture(), reserveResult.isReserveSuccess()); + // actually reserve size is not equals raw size, update the actually reserve size to the map + if (reserveResult.getReservedBytes() != bytesToReserve) { + sequenceIdToDataBlockSize.put(endSequenceId, reserveResult.getReservedBytes()); + } + bufferRetainedSizeInBytes += reserveResult.getReservedBytes(); endSequenceId += 1; - reservedBytes += bytesToReserve; + reservedBytes += reserveResult.getReservedBytes(); if (!Boolean.TRUE.equals(pair.right)) { blockedSize = bytesToReserve; break; @@ -619,14 +663,16 @@ private void fail(Throwable t) { if (aborted || closed) { return; } - bufferRetainedSizeInBytes -= reservedBytes; - localMemoryManager - .getQueryPool() - .free( - localFragmentInstanceId.getQueryId(), - fullFragmentInstanceId, - localPlanNodeId, - reservedBytes); + if (reservedBytes > 0) { + bufferRetainedSizeInBytes -= reservedBytes; + localMemoryManager + .getQueryPool() + .free( + localFragmentInstanceId.getQueryId(), + fullFragmentInstanceId, + localPlanNodeId, + reservedBytes); + } sourceHandleListener.onFailure(SourceHandle.this, t); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index f655df432bff7..a224e5b2c3586 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.exception.MemoryNotEnoughException; import org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet; import org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet; import org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet; @@ -152,6 +153,7 @@ public class FragmentInstanceContext extends QueryContext { private long unclosedUnseqFileNum = 0; private long closedSeqFileNum = 0; private long closedUnseqFileNum = 0; + private boolean highestPriority = false; public static FragmentInstanceContext createFragmentInstanceContext( FragmentInstanceId id, FragmentInstanceStateMachine stateMachine, SessionInfo sessionInfo) { @@ -869,7 +871,7 @@ public void releaseResourceWhenAllDriversAreClosed() { */ private void releaseTVListOwnedByQuery() { for (TVList tvList : tvListSet) { - long tvListRamSize = tvList.calculateRamSize(); + long tvListRamSize = tvList.calculateRamSize().getRamSize(); tvList.lockQueryList(); Set queryContextSet = tvList.getQueryContextSet(); try { @@ -898,9 +900,22 @@ private void releaseTVListOwnedByQuery() { memoryReservationManager.releaseMemoryVirtually(tvList.getReservedMemoryBytes()); FragmentInstanceContext queryContext = (FragmentInstanceContext) queryContextSet.iterator().next(); - queryContext - .getMemoryReservationContext() - .reserveMemoryVirtually(releasedBytes.left, releasedBytes.right); + try { + queryContext + .getMemoryReservationContext() + .reserveMemoryVirtually(releasedBytes.left, releasedBytes.right); + } catch (MemoryNotEnoughException ex) { + LOGGER.warn( + "MemoryNotEnoughException when transferring TVList ownership from query {} to another query {}.", + this.getId(), + queryContext.getId()); + } catch (RuntimeException ex) { + LOGGER.warn( + "Unexpected Exception when transferring TVList ownership from query {} to another query {}.", + this.getId(), + queryContext.getId(), + ex); + } if (LOGGER.isDebugEnabled()) { LOGGER.debug( @@ -1128,6 +1143,18 @@ public boolean ignoreNotExistsDevice() { return ignoreNotExistsDevice; } + /** + * Same flag as {@link + * org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis#needSetHighestPriority()}. + */ + public boolean isHighestPriority() { + return highestPriority; + } + + public void setHighestPriority(boolean highestPriority) { + this.highestPriority = highestPriority; + } + public boolean isSingleSourcePath() { return singleSourcePath; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java index e8d0fd8243201..9dacdf44d887d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java @@ -158,6 +158,7 @@ public FragmentInstanceInfo execDataQueryFragmentInstance( dataRegion, instance.getGlobalTimePredicate(), dataNodeQueryContextMap)); + context.setHighestPriority(instance.isHighestPriority()); try { List driverFactories = @@ -259,6 +260,7 @@ public FragmentInstanceInfo execSchemaQueryFragmentInstance( fragmentInstanceId -> createFragmentInstanceContext( fragmentInstanceId, stateMachine, instance.getSessionInfo())); + context.setHighestPriority(instance.isHighestPriority()); try { List driverFactories = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java index 8d62e20776679..d1b235ff7b100 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java @@ -217,7 +217,7 @@ public void setQueryStatistics(QueryStatistics queryStatistics) { this.queryStatistics = queryStatistics; } - public void addTVListToSet(Map tvListMap) { - tvListSet.addAll(tvListMap.keySet()); + public void addTVListToSet(Set set) { + tvListSet.addAll(set); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java index a9ab6ed5d8120..9a00d5a87de0d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPool.java @@ -23,10 +23,8 @@ import org.apache.iotdb.db.exception.runtime.MemoryLeakException; import com.google.common.util.concurrent.AbstractFuture; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import org.apache.commons.lang3.Validate; -import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,6 +39,8 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; + /** A thread-safe memory pool. */ public class MemoryPool { @@ -111,6 +111,31 @@ public boolean set(@Nullable V value) { } } + public static class MemoryReservationResult { + private final ListenableFuture future; + private final boolean reserveSuccess; + private final long reservedBytes; + + public MemoryReservationResult( + ListenableFuture future, boolean reserveSuccess, long reservedBytes) { + this.future = future; + this.reserveSuccess = reserveSuccess; + this.reservedBytes = reservedBytes; + } + + public ListenableFuture getFuture() { + return future; + } + + public boolean isReserveSuccess() { + return reserveSuccess; + } + + public long getReservedBytes() { + return reservedBytes; + } + } + private final String id; private final long maxBytes; private final long maxBytesPerFragmentInstance; @@ -220,18 +245,20 @@ public void deRegisterFragmentInstanceFromQueryMemoryMap( } /** - * Reserve memory with bytesToReserve. + * Reserve memory with bytesToReserve respect priority. * - * @return if reserve succeed, pair.right will be true, otherwise false + * @return if reserve succeed, reservedBytes may be zero or equals with bytesToReserve; if reserve + * failed, reservedBytes must be equals with bytesToReserve * @throws IllegalArgumentException throw exception if current query requests more memory than can * be allocated. */ - public Pair, Boolean> reserve( + public MemoryReservationResult reserveWithPriority( String queryId, String fragmentInstanceId, String planNodeId, long bytesToReserve, - long maxBytesCanReserve) { + long maxBytesCanReserve, + boolean isHighestPriority) { Validate.notNull(queryId, "queryId can not be null."); Validate.notNull(fragmentInstanceId, "fragmentInstanceId can not be null."); Validate.notNull(planNodeId, "planNodeId can not be null."); @@ -250,19 +277,21 @@ public Pair, Boolean> reserve( "Query is aborted since it requests more memory than can be allocated."); } - ListenableFuture result; if (tryReserve(queryId, fragmentInstanceId, planNodeId, bytesToReserve, maxBytesCanReserve)) { - result = Futures.immediateFuture(null); - return new Pair<>(result, Boolean.TRUE); + return new MemoryReservationResult(immediateVoidFuture(), true, bytesToReserve); } else { + rollbackReserve(queryId, fragmentInstanceId, planNodeId, bytesToReserve); + if (isHighestPriority) { + // SHOW QUERIES: treat as success with zero bytes reserved from pool when insufficient. + return new MemoryReservationResult(immediateVoidFuture(), true, 0L); + } LOGGER.debug( "Blocked reserve request: {} bytes memory for planNodeId{}", bytesToReserve, planNodeId); - rollbackReserve(queryId, fragmentInstanceId, planNodeId, bytesToReserve); - result = + ListenableFuture result = MemoryReservationFuture.create( queryId, fragmentInstanceId, planNodeId, bytesToReserve, maxBytesCanReserve); memoryReservationFutures.add((MemoryReservationFuture) result); - return new Pair<>(result, Boolean.FALSE); + return new MemoryReservationResult(result, false, bytesToReserve); } } @@ -293,7 +322,8 @@ public boolean tryReserveForTest( /** * Cancel the specified memory reservation. If the reservation has finished, do nothing. * - * @param future The future returned from {@link #reserve(String, String, String, long, long)} + * @param future The future returned from {@link #reserveWithPriority(String, String, String, + * long, long, boolean)} * @return If the future has not complete, return the number of bytes being reserved. Otherwise, * return 0. */ diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/CountGroupByLevelScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/CountGroupByLevelScanOperator.java index 084935f68b02a..8d46f25327c45 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/CountGroupByLevelScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/CountGroupByLevelScanOperator.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.schema.source.ISchemaSource; import org.apache.iotdb.db.queryengine.execution.operator.source.SourceOperator; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ISchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; @@ -95,6 +96,10 @@ public OperatorContext getOperatorContext() { return operatorContext; } + private ISchemaRegion getSchemaRegion() { + return ((SchemaDriverContext) operatorContext.getDriverContext()).getSchemaRegion(); + } + @Override public ListenableFuture isBlocked() { if (isBlocked == null) { @@ -109,6 +114,11 @@ public ListenableFuture isBlocked() { */ private ListenableFuture tryGetNext() { if (schemaReader == null) { + if (schemaSource.shouldSkipSchemaRegion(getSchemaRegion())) { + next = null; + isFinished = true; + return NOT_BLOCKED; + } schemaReader = createTimeSeriesReader(); } while (true) { @@ -172,15 +182,14 @@ public TsBlock next() throws Exception { @Override public boolean hasNext() throws Exception { isBlocked().get(); // wait for the next TsBlock - if (!schemaReader.isSuccess()) { + if (schemaReader != null && !schemaReader.isSuccess()) { throw new SchemaExecutionException(schemaReader.getFailure()); } return next != null; } public ISchemaReader createTimeSeriesReader() { - return schemaSource.getSchemaReader( - ((SchemaDriverContext) operatorContext.getDriverContext()).getSchemaRegion()); + return schemaSource.getSchemaReader(getSchemaRegion()); } private TsBlock constructTsBlockAndClearMap(Map countMap) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperator.java index 91c7ebda26b01..9e7e935dd330c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperator.java @@ -93,6 +93,10 @@ public ListenableFuture isBlocked() { */ private ListenableFuture tryGetNext() { ISchemaRegion schemaRegion = getSchemaRegion(); + if (schemaSource.shouldSkipSchemaRegion(schemaRegion)) { + next = constructTsBlock(0); + return NOT_BLOCKED; + } if (schemaSource.hasSchemaStatistic(schemaRegion)) { next = constructTsBlock(schemaSource.getSchemaStatistic(schemaRegion)); return NOT_BLOCKED; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/ISchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/ISchemaSource.java index 2a80e1071e535..573c7e35f062a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/ISchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/ISchemaSource.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ISchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.read.common.block.TsBlockBuilder; import java.util.List; @@ -52,4 +53,16 @@ public interface ISchemaSource { boolean hasSchemaStatistic(ISchemaRegion schemaRegion); long getSchemaStatistic(ISchemaRegion schemaRegion); + + default boolean shouldSkipSchemaRegion(ISchemaRegion schemaRegion) { + return false; + } + + default boolean checkRegionDatabaseIncluded(ISchemaRegion schemaRegion) { + return true; + } + + default long getMaxMemory(ISchemaRegion schemaRegion) { + return TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java index 7fe1084257957..ee8ea51baafb0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/SchemaSourceFactory.java @@ -42,9 +42,18 @@ public static ISchemaSource getTimeSeriesSchemaCountSourc boolean isPrefixMatch, SchemaFilter schemaFilter, Map templateMap, - PathPatternTree scope) { + PathPatternTree scope, + boolean includeSystemDatabase) { return new TimeSeriesSchemaSource( - pathPattern, isPrefixMatch, 0, 0, schemaFilter, templateMap, false, scope); + pathPattern, + isPrefixMatch, + 0, + 0, + schemaFilter, + templateMap, + false, + includeSystemDatabase, + scope); } // show time series @@ -57,7 +66,7 @@ public static ISchemaSource getTimeSeriesSchemaScanSource Map templateMap, PathPatternTree scope) { return new TimeSeriesSchemaSource( - pathPattern, isPrefixMatch, limit, offset, schemaFilter, templateMap, true, scope); + pathPattern, isPrefixMatch, limit, offset, schemaFilter, templateMap, true, true, scope); } // count device diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java index 47b73ccb0e5cd..6ecd9dc282d5e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java @@ -54,6 +54,7 @@ public class TimeSeriesSchemaSource implements ISchemaSource templateMap; private final boolean needViewDetail; + private final boolean includeSystemDatabase; TimeSeriesSchemaSource( PartialPath pathPattern, @@ -63,6 +64,7 @@ public class TimeSeriesSchemaSource implements ISchemaSource templateMap, boolean needViewDetail, + boolean includeSystemDatabase, PathPatternTree scope) { this.pathPattern = pathPattern; this.isPrefixMatch = isPrefixMatch; @@ -71,6 +73,7 @@ public class TimeSeriesSchemaSource implements ISchemaSource map) { if (map == null || map.isEmpty()) { return null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java index a9cbea64ddbe3..a42386dfd4ef3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java @@ -36,15 +36,22 @@ import org.apache.tsfile.utils.RamUsageEstimator; import java.io.IOException; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; public class ActiveTimeSeriesRegionScanOperator extends AbstractRegionScanDataSourceOperator { // Timeseries which need to be checked. private final Map> timeSeriesToSchemasInfo; - private static final Binary VIEW_TYPE = new Binary("BASE".getBytes()); + private final Set countedLogicalViews; + private static final Binary BASE_VIEW_TYPE = + new Binary("BASE".getBytes(TSFileConfig.STRING_CHARSET)); + private static final Binary LOGICAL_VIEW_TYPE = + new Binary("VIEW".getBytes(TSFileConfig.STRING_CHARSET)); private final Binary dataBaseName; + private final String dataBaseNameString; private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ActiveTimeSeriesRegionScanOperator.class) + RamUsageEstimator.shallowSizeOfInstance(Map.class) @@ -61,15 +68,15 @@ public ActiveTimeSeriesRegionScanOperator( this.operatorContext = operatorContext; this.sourceId = sourceId; this.timeSeriesToSchemasInfo = timeSeriesToSchemasInfo; + this.countedLogicalViews = new HashSet<>(); this.regionScanUtil = new RegionScanForActiveTimeSeriesUtil(timeFilter, ttlCache); - this.dataBaseName = - new Binary( - operatorContext - .getDriverContext() - .getFragmentInstanceContext() - .getDataRegion() - .getDatabaseName() - .getBytes(TSFileConfig.STRING_CHARSET)); + this.dataBaseNameString = + operatorContext + .getDriverContext() + .getFragmentInstanceContext() + .getDataRegion() + .getDatabaseName(); + this.dataBaseName = new Binary(this.dataBaseNameString.getBytes(TSFileConfig.STRING_CHARSET)); } @Override @@ -93,16 +100,22 @@ private void checkAndAppend(String info, ColumnBuilder columnBuilders) { @Override protected void updateActiveData() { - TimeColumnBuilder timeColumnBuilder = resultTsBlockBuilder.getTimeColumnBuilder(); - ColumnBuilder[] columnBuilders = resultTsBlockBuilder.getValueColumnBuilders(); - Map> activeTimeSeries = ((RegionScanForActiveTimeSeriesUtil) regionScanUtil).getActiveTimeSeries(); if (outputCount) { for (Map.Entry> entry : activeTimeSeries.entrySet()) { List timeSeriesList = entry.getValue(); - count += timeSeriesList.size(); + Map timeSeriesInfo = timeSeriesToSchemasInfo.get(entry.getKey()); + for (String timeSeries : timeSeriesList) { + TimeseriesContext schemaInfo = timeSeriesInfo.get(timeSeries); + count += schemaInfo.getActiveCountMultiplier(); + for (String logicalView : schemaInfo.getActiveLogicalViewCountSet()) { + if (countedLogicalViews.add(logicalView)) { + count++; + } + } + } removeTimeseriesListFromDevice(entry.getKey(), timeSeriesList); } return; @@ -115,26 +128,49 @@ protected void updateActiveData() { Map timeSeriesInfo = timeSeriesToSchemasInfo.get(deviceID); for (String timeSeries : timeSeriesList) { TimeseriesContext schemaInfo = timeSeriesInfo.get(timeSeries); - timeColumnBuilder.writeLong(-1); - columnBuilders[0].writeBinary( - new Binary(contactDeviceAndMeasurement(deviceStr, timeSeries))); - - checkAndAppend(schemaInfo.getAlias(), columnBuilders[1]); // Measurement - columnBuilders[2].writeBinary(dataBaseName); // Database - checkAndAppend(schemaInfo.getDataType(), columnBuilders[3]); // DataType - checkAndAppend(schemaInfo.getEncoding(), columnBuilders[4]); // Encoding - checkAndAppend(schemaInfo.getCompression(), columnBuilders[5]); // Compression - checkAndAppend(schemaInfo.getTags(), columnBuilders[6]); // Tags - checkAndAppend(schemaInfo.getAttributes(), columnBuilders[7]); // Attributes - checkAndAppend(schemaInfo.getDeadband(), columnBuilders[8]); // Description - checkAndAppend(schemaInfo.getDeadbandParameters(), columnBuilders[9]); // DeadbandParameters - columnBuilders[10].writeBinary(VIEW_TYPE); // ViewType - resultTsBlockBuilder.declarePosition(); + if (schemaInfo.getActiveCountMultiplier() > 0) { + appendTimeseries( + contactDeviceAndMeasurement(deviceStr, timeSeries), schemaInfo, BASE_VIEW_TYPE); + } + for (Map.Entry logicalViewEntry : + schemaInfo.getActiveLogicalViewContextMap().entrySet()) { + if (countedLogicalViews.add(logicalViewEntry.getKey())) { + appendTimeseries( + logicalViewEntry.getKey().getBytes(TSFileConfig.STRING_CHARSET), + logicalViewEntry.getValue(), + LOGICAL_VIEW_TYPE); + } + } } removeTimeseriesListFromDevice(deviceID, timeSeriesList); } } + private void appendTimeseries( + byte[] timeseriesPath, TimeseriesContext schemaInfo, Binary viewType) { + TimeColumnBuilder timeColumnBuilder = resultTsBlockBuilder.getTimeColumnBuilder(); + ColumnBuilder[] columnBuilders = resultTsBlockBuilder.getValueColumnBuilders(); + + timeColumnBuilder.writeLong(-1); + columnBuilders[0].writeBinary(new Binary(timeseriesPath)); + + checkAndAppend(schemaInfo.getAlias(), columnBuilders[1]); // Measurement + if (schemaInfo.getDatabase() == null || dataBaseNameString.equals(schemaInfo.getDatabase())) { + columnBuilders[2].writeBinary(dataBaseName); // Database + } else { + checkAndAppend(schemaInfo.getDatabase(), columnBuilders[2]); // Database + } + checkAndAppend(schemaInfo.getDataType(), columnBuilders[3]); // DataType + checkAndAppend(schemaInfo.getEncoding(), columnBuilders[4]); // Encoding + checkAndAppend(schemaInfo.getCompression(), columnBuilders[5]); // Compression + checkAndAppend(schemaInfo.getTags(), columnBuilders[6]); // Tags + checkAndAppend(schemaInfo.getAttributes(), columnBuilders[7]); // Attributes + checkAndAppend(schemaInfo.getDeadband(), columnBuilders[8]); // Description + checkAndAppend(schemaInfo.getDeadbandParameters(), columnBuilders[9]); // DeadbandParameters + columnBuilders[10].writeBinary(viewType); // ViewType + resultTsBlockBuilder.declarePosition(); + } + private void removeTimeseriesListFromDevice(IDeviceID deviceID, List timeSeriesList) { Map timeSeriesInfo = timeSeriesToSchemasInfo.get(deviceID); for (String timeSeries : timeSeriesList) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java index a629c151fa901..239ebb11329ae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java @@ -162,6 +162,7 @@ public void stop() { t -> { try { t.close(); + t.interrupt(); } catch (IOException e) { // Only a field is set, there's no chance to throw an IOException } @@ -335,16 +336,18 @@ private void clearDriverTask(DriverTask task) { return; case READY: task.setStatus(DriverTaskStatus.ABORTED); - readyQueue.remove(task.getDriverTaskId()); + if (readyQueue.remove(task.getDriverTaskId()) == null) { + readyQueue.decreaseReservedSize(task); + } break; case BLOCKED: task.setStatus(DriverTaskStatus.ABORTED); blockedTasks.remove(task); - readyQueue.decreaseReservedSize(); + readyQueue.decreaseReservedSize(task); break; case RUNNING: task.setStatus(DriverTaskStatus.ABORTED); - readyQueue.decreaseReservedSize(); + readyQueue.decreaseReservedSize(task); break; case FINISHED: break; @@ -413,6 +416,10 @@ public long getReadyQueueTaskCount() { return readyQueue.size(); } + public long getReadyQueueReservedTaskCount() { + return readyQueue.getReservedSize(); + } + public long getBlockQueueTaskCount() { return blockedTasks.size(); } @@ -488,6 +495,7 @@ public boolean readyToRunning(DriverTask task) { task.lock(); try { if (task.getStatus() != DriverTaskStatus.READY) { + readyQueue.decreaseReservedSize(task); return false; } @@ -544,7 +552,7 @@ public void runningToFinished(DriverTask task, ExecutionContext context) { } task.updateSchedulePriority(context); task.setStatus(DriverTaskStatus.FINISHED); - readyQueue.decreaseReservedSize(); + readyQueue.decreaseReservedSize(task); } finally { task.unlock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/queue/IndexedBlockingReserveQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/queue/IndexedBlockingReserveQueue.java index 1891fb6355677..b0f0edbd0e1aa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/queue/IndexedBlockingReserveQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/queue/IndexedBlockingReserveQueue.java @@ -47,6 +47,7 @@ public synchronized E poll() throws InterruptedException { E output = pollFirst(); size--; reservedSize++; + markReserved(output); return output; } @@ -66,7 +67,7 @@ public synchronized void repush(E element) { throw new NullPointerException("pushed element is null"); } pushToQueue(element); - reservedSize--; + decreaseReservedSizeIfNecessary(element); size++; this.notifyAll(); } @@ -74,7 +75,36 @@ public synchronized void repush(E element) { /** * For task that is not in readyQueue when it's cleared, it won't be added into the queue again. */ - public synchronized void decreaseReservedSize() { - this.reservedSize--; + public synchronized boolean decreaseReservedSize(E element) { + if (element == null) { + throw new NullPointerException("pushed element is null"); + } + return decreaseReservedSizeIfNecessary(element); + } + + public final synchronized int getReservedSize() { + return reservedSize; + } + + @Override + public synchronized void clear() { + super.clear(); + this.reservedSize = 0; + } + + protected void markReserved(E element) { + // Do nothing by default. + } + + protected boolean releaseReserved(E element) { + return true; + } + + private boolean decreaseReservedSizeIfNecessary(E element) { + if (!releaseReserved(element) || reservedSize <= 0) { + return false; + } + reservedSize--; + return true; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/queue/multilevelqueue/MultilevelPriorityQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/queue/multilevelqueue/MultilevelPriorityQueue.java index c5f3cb3e6df1a..06c7275473b4e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/queue/multilevelqueue/MultilevelPriorityQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/queue/multilevelqueue/MultilevelPriorityQueue.java @@ -191,6 +191,16 @@ protected DriverTask get(DriverTask driverTask) { "MultilevelPriorityQueue does not support access element by get."); } + @Override + protected void markReserved(DriverTask task) { + task.markReservedInReadyQueue(); + } + + @Override + protected boolean releaseReserved(DriverTask task) { + return task.releaseReservedInReadyQueue(); + } + @Override protected void clearAllElements() { highestPriorityLevelQueue.clear(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/task/DriverTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/task/DriverTask.java index f320c6cad6c2f..85bada856caa2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/task/DriverTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/task/DriverTask.java @@ -59,6 +59,7 @@ public class DriverTask implements IDIndexedAccessible { private final DriverTaskHandle driverTaskHandle; private long lastEnterReadyQueueTime; private long lastEnterBlockQueueTime; + private boolean reservedInReadyQueue; private long estimatedMemorySize; @@ -210,6 +211,18 @@ public void setLastEnterBlockQueueTime(long lastEnterBlockQueueTime) { this.lastEnterBlockQueueTime = lastEnterBlockQueueTime; } + public void markReservedInReadyQueue() { + reservedInReadyQueue = true; + } + + public boolean releaseReservedInReadyQueue() { + if (!reservedInReadyQueue) { + return false; + } + reservedInReadyQueue = false; + return true; + } + /** a comparator of ddl, the less the ddl is, the low order it has. */ public static class TimeoutComparator implements Comparator { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/DriverSchedulerMetricSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/DriverSchedulerMetricSet.java index 0b9e0c04f050d..94aaccfd9b7d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/DriverSchedulerMetricSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/DriverSchedulerMetricSet.java @@ -39,6 +39,7 @@ private DriverSchedulerMetricSet() { public static final String READY_QUEUED_TIME = "ready_queued_time"; public static final String BLOCK_QUEUED_TIME = "block_queued_time"; public static final String READY_QUEUE_TASK_COUNT = "ready_queue_task_count"; + public static final String READY_QUEUE_RESERVED_TASK_COUNT = "ready_queue_reserved_task_count"; public static final String BLOCK_QUEUE_TASK_COUNT = "block_queue_task_count"; private static final String TIMEOUT_QUEUE_SIZE = "timeout_queue_task_count"; private static final String QUERY_MAP_SIZE = "query_map_size"; @@ -67,6 +68,13 @@ public void bindTo(AbstractMetricService metricService) { DriverScheduler::getReadyQueueTaskCount, Tag.NAME.toString(), READY_QUEUE_TASK_COUNT); + metricService.createAutoGauge( + Metric.DRIVER_SCHEDULER.toString(), + MetricLevel.IMPORTANT, + DriverScheduler.getInstance(), + DriverScheduler::getReadyQueueReservedTaskCount, + Tag.NAME.toString(), + READY_QUEUE_RESERVED_TASK_COUNT); metricService.createAutoGauge( Metric.DRIVER_SCHEDULER.toString(), MetricLevel.IMPORTANT, @@ -109,6 +117,11 @@ public void unbindFrom(AbstractMetricService metricService) { Metric.DRIVER_SCHEDULER.toString(), Tag.NAME.toString(), READY_QUEUE_TASK_COUNT); + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.DRIVER_SCHEDULER.toString(), + Tag.NAME.toString(), + READY_QUEUE_RESERVED_TASK_COUNT); metricService.remove( MetricType.AUTO_GAUGE, Metric.DRIVER_SCHEDULER.toString(), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java index 8afdfce7d5c65..2f173861f22e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java @@ -92,10 +92,10 @@ public class Coordinator { ASYNC_INTERNAL_SERVICE_CLIENT_MANAGER = new IClientManager.Factory() .createClientManager( - new ClientPoolFactory.AsyncDataNodeInternalServiceClientPoolFactory()); + new ClientPoolFactory.AsyncDataNodeInternalServiceClientPoolFactory( + CONFIG.getSelectorNumOfClientManager())); private final ExecutorService executor; - private final ExecutorService writeOperationExecutor; private final ScheduledExecutorService scheduledExecutor; private final ExecutorService dispatchExecutor; @@ -109,7 +109,6 @@ public class Coordinator { private Coordinator() { this.queryExecutionMap = new ConcurrentHashMap<>(); this.executor = getQueryExecutor(); - this.writeOperationExecutor = getWriteExecutor(); this.scheduledExecutor = getScheduledExecutor(); int dispatchThreadNum = Math.max(20, Runtime.getRuntime().availableProcessors() * 2); this.dispatchExecutor = @@ -220,8 +219,6 @@ private IQueryExecution createQueryExecutionForTreeModel( TreeModelPlanner treeModelPlanner = new TreeModelPlanner( statement, - executor, - writeOperationExecutor, scheduledExecutor, partitionFetcher, schemaFetcher, @@ -248,12 +245,6 @@ private ExecutorService getQueryExecutor() { coordinatorReadExecutorSize, ThreadName.MPP_COORDINATOR_EXECUTOR_POOL.getName()); } - private ExecutorService getWriteExecutor() { - int coordinatorWriteExecutorSize = CONFIG.getCoordinatorWriteExecutorSize(); - return IoTDBThreadPoolFactory.newFixedThreadPool( - coordinatorWriteExecutorSize, ThreadName.MPP_COORDINATOR_WRITE_EXECUTOR.getName()); - } - private ScheduledExecutorService getScheduledExecutor() { return IoTDBThreadPoolFactory.newScheduledThreadPool( COORDINATOR_SCHEDULED_EXECUTOR_SIZE, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java index 01d1020934dda..63b2d379ddd93 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java @@ -503,7 +503,7 @@ public boolean isQuery() { public boolean needSetHighestPriority() { // if is this Statement is ShowQueryStatement, set its instances to the highest priority, so // that the sub-tasks of the ShowQueries instances could be executed first. - return StatementType.SHOW_QUERIES.equals(statement.getType()); + return statement != null && StatementType.SHOW_QUERIES.equals(statement.getType()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index a28ca79c9567b..a7f955346700d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -36,6 +36,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; @@ -53,6 +54,7 @@ import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.NodeRef; import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; @@ -83,6 +85,7 @@ import org.apache.iotdb.db.queryengine.plan.expression.leaf.ConstantOperand; import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand; import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression; +import org.apache.iotdb.db.queryengine.plan.expression.visitor.CompleteMeasurementSchemaVisitor; import org.apache.iotdb.db.queryengine.plan.expression.visitor.ExistUnknownTypeInExpression; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.MeasurementGroup; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.DeviceViewIntoPathDescriptor; @@ -157,6 +160,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; +import org.apache.iotdb.db.schemaengine.schemaregion.view.visitor.TransformToExpressionVisitor; import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.iotdb.rpc.RpcUtils; @@ -3112,7 +3116,9 @@ private boolean analyzeTimeseriesRegionScan( PathPatternTree patternTree, Analysis analysis, MPPQueryContext context, - PathPatternTree authorityScope) + PathPatternTree authorityScope, + boolean canSeeSystemDB, + boolean includeLogicalView) throws IllegalPathException { analyzeGlobalTimeConditionInShowMetaData(timeCondition, analysis); context.generateGlobalTimeFilter(analysis); @@ -3124,7 +3130,15 @@ private boolean analyzeTimeseriesRegionScan( analysis.setFinishQueryAfterAnalyze(true); return false; } - removeLogicViewMeasurement(schemaTree); + List deviceSchemaInfoList; + if (includeLogicalView) { + deviceSchemaInfoList = schemaTree.getMatchedDevices(ALL_MATCH_PATTERN); + updateSchemaTreeByViews(analysis, schemaTree, context); + } else { + removeLogicViewMeasurement(schemaTree); + deviceSchemaInfoList = schemaTree.getMatchedDevices(ALL_MATCH_PATTERN); + } + Map>> deviceToTimeseriesContext = new HashMap<>(); /** @@ -3132,38 +3146,59 @@ private boolean analyzeTimeseriesRegionScan( * as a normal node, not a device+templateId. This means that all nodes are what we need.). We * can use ALL_MATCH_PATTERN to get result. */ - List deviceSchemaInfoList = schemaTree.getMatchedDevices(ALL_MATCH_PATTERN); Set deviceSet = new HashSet<>(); for (DeviceSchemaInfo deviceSchemaInfo : deviceSchemaInfoList) { boolean isAligned = deviceSchemaInfo.isAligned(); PartialPath devicePath = deviceSchemaInfo.getDevicePath(); - deviceSet.add(devicePath.getFullPath()); + if (shouldSkipInternalDatabaseForActiveCount(devicePath, schemaTree, canSeeSystemDB)) { + continue; + } if (isAligned) { List measurementList = new ArrayList<>(); List schemaList = new ArrayList<>(); List timeseriesContextList = new ArrayList<>(); for (IMeasurementSchemaInfo measurementSchemaInfo : deviceSchemaInfo.getMeasurementSchemaInfoList()) { + if (includeLogicalView && measurementSchemaInfo.isLogicalView()) { + addLogicalViewSourcesForActiveCount( + devicePath, + measurementSchemaInfo, + schemaTree, + deviceToTimeseriesContext, + deviceSet); + continue; + } schemaList.add(measurementSchemaInfo.getSchema()); measurementList.add(measurementSchemaInfo.getName()); timeseriesContextList.add(new TimeseriesContext(measurementSchemaInfo)); } - AlignedPath alignedPath = - new AlignedPath(devicePath.getNodes(), measurementList, schemaList); - deviceToTimeseriesContext - .computeIfAbsent(devicePath, k -> new HashMap<>()) - .put(alignedPath, timeseriesContextList); + if (!measurementList.isEmpty()) { + deviceSet.add(devicePath.getFullPath()); + AlignedPath alignedPath = + new AlignedPath(devicePath.getNodes(), measurementList, schemaList); + deviceToTimeseriesContext + .computeIfAbsent(devicePath, k -> new HashMap<>()) + .put(alignedPath, timeseriesContextList); + } } else { for (IMeasurementSchemaInfo measurementSchemaInfo : deviceSchemaInfo.getMeasurementSchemaInfoList()) { - MeasurementPath measurementPath = - new MeasurementPath( - devicePath.concatNode(measurementSchemaInfo.getName()).getNodes()); - deviceToTimeseriesContext - .computeIfAbsent(devicePath, k -> new HashMap<>()) - .put( - measurementPath, - Collections.singletonList(new TimeseriesContext(measurementSchemaInfo))); + if (includeLogicalView && measurementSchemaInfo.isLogicalView()) { + addLogicalViewSourcesForActiveCount( + devicePath, + measurementSchemaInfo, + schemaTree, + deviceToTimeseriesContext, + deviceSet); + } else { + addPhysicalTimeseriesForActiveCount( + devicePath, + measurementSchemaInfo, + false, + new TimeseriesContext(measurementSchemaInfo), + deviceToTimeseriesContext, + deviceSet); + } } } } @@ -3175,6 +3210,109 @@ private boolean analyzeTimeseriesRegionScan( return true; } + private boolean shouldSkipInternalDatabaseForActiveCount( + PartialPath devicePath, ISchemaTree schemaTree, boolean canSeeSystemDB) { + String database = schemaTree.getBelongedDatabase(devicePath); + if (SchemaConstant.SYSTEM_DATABASE.equals(database)) { + return !canSeeSystemDB; + } + return false; + } + + private void addLogicalViewSourcesForActiveCount( + PartialPath viewDevicePath, + IMeasurementSchemaInfo viewSchemaInfo, + ISchemaTree schemaTree, + Map>> deviceToTimeseriesContext, + Set deviceSet) { + LogicalViewSchema logicalViewSchema = viewSchemaInfo.getSchemaAsLogicalViewSchema(); + if (logicalViewSchema == null) { + return; + } + + String viewPath = viewDevicePath.concatNode(viewSchemaInfo.getName()).getFullPath(); + String viewDataType = getLogicalViewDataType(logicalViewSchema, schemaTree); + String viewDatabase = schemaTree.getBelongedDatabase(viewDevicePath); + for (PartialPath sourcePath : getSourcePaths(logicalViewSchema.getExpression())) { + if (sourcePath.getNodeLength() <= 1) { + continue; + } + PartialPath sourceDevicePath = + new PartialPath(Arrays.copyOf(sourcePath.getNodes(), sourcePath.getNodeLength() - 1)); + DeviceSchemaInfo sourceDeviceSchemaInfo = + schemaTree.searchDeviceSchemaInfo( + sourceDevicePath, Collections.singletonList(sourcePath.getMeasurement())); + if (sourceDeviceSchemaInfo == null + || sourceDeviceSchemaInfo.getMeasurementSchemaInfoList().isEmpty()) { + continue; + } + + IMeasurementSchemaInfo sourceSchemaInfo = + sourceDeviceSchemaInfo.getMeasurementSchemaInfoList().get(0); + if (sourceSchemaInfo == null || sourceSchemaInfo.isLogicalView()) { + continue; + } + + Map activeLogicalViewContextMap = + Collections.singletonMap( + viewPath, + new TimeseriesContext( + viewSchemaInfo, viewDataType, viewDatabase, 1, Collections.emptyMap())); + addPhysicalTimeseriesForActiveCount( + sourceDevicePath, + sourceSchemaInfo, + sourceDeviceSchemaInfo.isAligned(), + new TimeseriesContext(sourceSchemaInfo, 0, activeLogicalViewContextMap), + deviceToTimeseriesContext, + deviceSet); + } + } + + private String getLogicalViewDataType( + LogicalViewSchema logicalViewSchema, ISchemaTree schemaTree) { + if (logicalViewSchema.getType() != TSDataType.UNKNOWN) { + return logicalViewSchema.getType().toString(); + } + try { + Expression expression = + new TransformToExpressionVisitor().process(logicalViewSchema.getExpression(), null); + expression = new CompleteMeasurementSchemaVisitor().process(expression, schemaTree); + Map, TSDataType> expressionTypes = new HashMap<>(); + analyzeExpression(expressionTypes, expression); + TSDataType dataType = expressionTypes.get(NodeRef.of(expression)); + return dataType == null ? TSDataType.UNKNOWN.toString() : dataType.toString(); + } catch (Exception e) { + return TSDataType.UNKNOWN.toString(); + } + } + + private void addPhysicalTimeseriesForActiveCount( + PartialPath devicePath, + IMeasurementSchemaInfo measurementSchemaInfo, + boolean isAligned, + TimeseriesContext timeseriesContext, + Map>> deviceToTimeseriesContext, + Set deviceSet) { + deviceSet.add(devicePath.getFullPath()); + PartialPath timeseriesPath = + isAligned + ? new AlignedPath( + devicePath.getNodes(), + Collections.singletonList(measurementSchemaInfo.getName()), + Collections.singletonList(measurementSchemaInfo.getSchema())) + : new MeasurementPath( + devicePath.concatNode(measurementSchemaInfo.getName()).getNodes()); + Map> timeseriesContextMap = + deviceToTimeseriesContext.computeIfAbsent(devicePath, k -> new HashMap<>()); + List existingContextList = timeseriesContextMap.get(timeseriesPath); + if (existingContextList == null) { + timeseriesContextMap.put( + timeseriesPath, new ArrayList<>(Collections.singletonList(timeseriesContext))); + } else { + existingContextList.set(0, existingContextList.get(0).mergeActiveCount(timeseriesContext)); + } + } + @Override public Analysis visitShowTimeSeries( ShowTimeSeriesStatement showTimeSeriesStatement, MPPQueryContext context) { @@ -3193,7 +3331,9 @@ public Analysis visitShowTimeSeries( patternTree, analysis, context, - showTimeSeriesStatement.getAuthorityScope()); + showTimeSeriesStatement.getAuthorityScope(), + true, + true); if (!hasSchema) { analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowTimeSeriesHeader()); return analysis; @@ -3437,7 +3577,9 @@ public Analysis visitCountTimeSeries( patternTree, analysis, context, - countTimeSeriesStatement.getAuthorityScope()); + countTimeSeriesStatement.getAuthorityScope(), + countTimeSeriesStatement.isCanSeeSystemDB(), + true); if (!hasSchema) { analysis.setRespDatasetHeader(DatasetHeaderFactory.getCountTimeSeriesHeader()); return analysis; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java index ff19dd79cf58e..6b28dbda34705 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java @@ -59,7 +59,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -259,26 +258,28 @@ public DataPartition getOrCreateDataPartition( dataPartitionQueryParams, config.isAutoCreateSchemaEnabled(), userName); DataPartition dataPartition = partitionCache.getDataPartition(splitDataPartitionQueryParams); - if (null == dataPartition) { - try (ConfigNodeClient client = - configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - TDataPartitionReq req = constructDataPartitionReq(splitDataPartitionQueryParams); - TDataPartitionTableResp dataPartitionTableResp = client.getOrCreateDataPartitionTable(req); + if (null != dataPartition) { + return dataPartition; + } - if (dataPartitionTableResp.getStatus().getCode() - == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - dataPartition = parseDataPartitionResp(dataPartitionTableResp); - partitionCache.updateDataPartitionCache(dataPartitionTableResp.getDataPartitionTable()); - } else { - throw new RuntimeException( - new IoTDBException( - dataPartitionTableResp.getStatus().getMessage(), - dataPartitionTableResp.getStatus().getCode())); - } - } catch (ClientManagerException | TException e) { - throw new StatementAnalyzeException( - "An error occurred when executing getOrCreateDataPartition():" + e.getMessage()); + try (ConfigNodeClient client = + configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + TDataPartitionReq req = constructDataPartitionReq(splitDataPartitionQueryParams); + TDataPartitionTableResp dataPartitionTableResp = client.getOrCreateDataPartitionTable(req); + + if (dataPartitionTableResp.getStatus().getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + dataPartition = parseDataPartitionResp(dataPartitionTableResp); + partitionCache.updateDataPartitionCache(dataPartitionTableResp.getDataPartitionTable()); + } else { + throw new RuntimeException( + new IoTDBException( + dataPartitionTableResp.getStatus().getMessage(), + dataPartitionTableResp.getStatus().getCode())); } + } catch (ClientManagerException | TException e) { + throw new StatementAnalyzeException( + "An error occurred when executing getOrCreateDataPartition():" + e.getMessage()); } return dataPartition; } @@ -421,16 +422,31 @@ private SchemaPartition parseSchemaPartitionTableResp( String database = entry1.getKey(); final Map result1 = regionReplicaMap.computeIfAbsent(database, k -> new HashMap<>()); + Map seriesPartitionTable = entry1.getValue(); + + if (seriesPartitionTable.size() == 1) { + // Fast collection in case of query for single device + Map.Entry seriesPartitionEntry = + seriesPartitionTable.entrySet().iterator().next(); + List regionReplicaSets = + partitionCache.getRegionReplicaSet( + Collections.singletonList(seriesPartitionEntry.getValue())); + result1.put(seriesPartitionEntry.getKey(), regionReplicaSets.get(0)); + continue; + } - Map orderedMap = - new LinkedHashMap<>(entry1.getValue()); - List orderedGroupIds = new ArrayList<>(orderedMap.values()); + List distinctRegionGroupIds = + new ArrayList<>(new HashSet<>(seriesPartitionTable.values())); List regionReplicaSets = - partitionCache.getRegionReplicaSet(orderedGroupIds); + partitionCache.getRegionReplicaSet(distinctRegionGroupIds); + Map groupIdToReplicaSet = new HashMap<>(); + for (int index = 0; index < distinctRegionGroupIds.size(); index++) { + groupIdToReplicaSet.put(distinctRegionGroupIds.get(index), regionReplicaSets.get(index)); + } - int index = 0; - for (Map.Entry entry2 : orderedMap.entrySet()) { - result1.put(entry2.getKey(), regionReplicaSets.get(index++)); + for (Map.Entry entry2 : + seriesPartitionTable.entrySet()) { + result1.put(entry2.getKey(), groupIdToReplicaSet.get(entry2.getValue())); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java index 3337e2f1420e9..ecd4c297e5729 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceCacheEntry.java @@ -30,7 +30,6 @@ import java.util.Map; import java.util.Objects; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; @@ -94,15 +93,14 @@ IDeviceSchema getDeviceSchema() { } int invalidateSchema() { - final AtomicInteger size = new AtomicInteger(0); - deviceSchema.updateAndGet( - schema -> { - if (Objects.nonNull(schema)) { - size.set(schema.estimateSize()); - } - return null; - }); - return size.get(); + IDeviceSchema schema; + do { + schema = deviceSchema.get(); + if (Objects.isNull(schema)) { + return 0; + } + } while (!deviceSchema.compareAndSet(schema, null)); + return schema.estimateSize(); } /////////////////////////////// Last Cache /////////////////////////////// @@ -150,15 +148,14 @@ boolean updateInputMap(final @Nonnull Map updateMap) { } int invalidateLastCache() { - final AtomicInteger size = new AtomicInteger(0); - lastCache.updateAndGet( - cacheEntry -> { - if (Objects.nonNull(cacheEntry)) { - size.set(cacheEntry.estimateSize()); - } - return null; - }); - return size.get(); + DeviceLastCache cacheEntry; + do { + cacheEntry = lastCache.get(); + if (Objects.isNull(cacheEntry)) { + return 0; + } + } while (!lastCache.compareAndSet(cacheEntry, null)); + return cacheEntry.estimateSize(); } /////////////////////////////// Management /////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java index ecbb2d3aea25a..9bcb020b72fc8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceSchemaCache.java @@ -165,7 +165,7 @@ void invalidateLastCache(final PartialPath devicePath, final String measurement) // the un-related paths being cleared, like "root.*.b.c.**" affects // "root.*.d.c.**", thereby lower the query performance. dualKeyCache.update( - cachedDeviceID -> cachedDeviceID.matchFullPath(devicePath), updateFunction); + cachedDeviceID -> devicePath.matchFullPath(cachedDeviceID), updateFunction); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index 5a346fdf8e09d..b6d4fb0043b79 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -372,7 +372,7 @@ private boolean handleSingleMiniFile(final int i) throws FileNotFoundException { final TSStatus status = loadTsFileDataTypeConverter .convertForTreeModel( - new LoadTsFileStatement(tsFiles.get(i).getPath()) + LoadTsFileStatement.createUnchecked(tsFiles.get(i).getPath()) .setDeleteAfterLoad(isDeleteAfterLoad) .setConvertOnTypeMismatch(isConvertOnTypeMismatch)) .orElse(null); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java index 8de6165525e29..5af143bfca83f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java @@ -125,6 +125,7 @@ public QueryExecution(IPlanner planner, MPPQueryContext context, ExecutorService this.context = context; this.planner = planner; this.analysis = analyze(context); + context.setNeedSetHighestPriority(analysis.needSetHighestPriority()); this.stateMachine = new QueryStateMachine(context.getQueryId(), executor); // We add the abort logic inside the QueryExecution. @@ -587,7 +588,8 @@ private void initResultHandle() { context.getResultNodeContext().getUpStreamPlanNodeId().getId(), context.getResultNodeContext().getUpStreamFragmentInstanceId().toThrift(), 0, // Upstream of result ExchangeNode will only have one child. - stateMachine::transitionToFailed) + stateMachine::transitionToFailed, + context.needSetHighestPriority()) : MPPDataExchangeService.getInstance() .getMPPDataExchangeManager() .createSourceHandle( @@ -596,7 +598,8 @@ private void initResultHandle() { 0, upstreamEndPoint, context.getResultNodeContext().getUpStreamFragmentInstanceId().toThrift(), - stateMachine::transitionToFailed); + stateMachine::transitionToFailed, + context.needSetHighestPriority()); } @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 023f7e3379368..99eee84964e97 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -140,10 +140,10 @@ import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; +import org.apache.iotdb.db.pipe.source.dataregion.DataRegionListeningFilter; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; -import org.apache.iotdb.db.protocol.client.DataNodeClientPoolFactory; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; import org.apache.iotdb.db.queryengine.plan.Coordinator; @@ -306,13 +306,6 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor { private static final IClientManager CONFIG_NODE_CLIENT_MANAGER = ConfigNodeClientManager.getInstance(); - /** FIXME Consolidate this clientManager with the upper one. */ - private static final IClientManager - CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER = - new IClientManager.Factory() - .createClientManager( - new DataNodeClientPoolFactory.ClusterDeletionConfigNodeClientPoolFactory()); - private static final class ClusterConfigTaskExecutorHolder { private static final ClusterConfigTaskExecutor INSTANCE = new ClusterConfigTaskExecutor(); @@ -1598,7 +1591,7 @@ public SettableFuture deactivateSchemaTemplate( req.setPathPatternTree( serializePatternListToByteBuffer(deactivateTemplateStatement.getPathPatternList())); try (final ConfigNodeClient client = - CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { TSStatus tsStatus; do { try { @@ -1699,7 +1692,7 @@ public SettableFuture alterSchemaTemplate( alterSchemaTemplateStatement.getOperationType(), alterSchemaTemplateStatement.getTemplateAlterInfo())); try (final ConfigNodeClient client = - CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { TSStatus tsStatus; do { try { @@ -1754,7 +1747,7 @@ public SettableFuture unsetSchemaTemplate( req.setTemplateName(unsetSchemaTemplateStatement.getTemplateName()); req.setPath(unsetSchemaTemplateStatement.getPath().getFullPath()); try (final ConfigNodeClient client = - CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { TSStatus tsStatus; do { try { @@ -1834,10 +1827,10 @@ public SettableFuture createPipe( new PipeParameters(createPipeStatement.getSourceAttributes()); final PipeParameters sinkPipeParameters = new PipeParameters(createPipeStatement.getSinkAttributes()); - if (PipeConfig.getInstance().getPipeAutoSplitFullEnabled() - && PipeDataNodeAgent.task().isFullSync(sourcePipeParameters)) { - try (final ConfigNodeClient configNodeClient = - CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + try (final ConfigNodeClient configNodeClient = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + if (PipeConfig.getInstance().getPipeAutoSplitFullEnabled() + && PipeDataNodeAgent.task().isFullSync(sourcePipeParameters)) { // 1. Send request to create the real-time data synchronization pipeline final TCreatePipeReq realtimeReq = new TCreatePipeReq() @@ -1882,11 +1875,19 @@ public SettableFuture createPipe( Boolean.toString(false), PipeSourceConstant.EXTRACTOR_HISTORY_ENABLE_KEY, Boolean.toString(true), - // We force the historical pipe to transfer data only + PipeSourceConstant.EXTRACTOR_MODE_KEY, + PipeSourceConstant.EXTRACTOR_MODE_SNAPSHOT_VALUE, + // We force the historical pipe to transfer data (and maybe + // deletion) only // Thus we can transfer schema only once // And may drop the historical pipe on successfully transferred PipeSourceConstant.SOURCE_INCLUSION_KEY, - PipeSourceConstant.EXTRACTOR_INCLUSION_DEFAULT_VALUE, + DataRegionListeningFilter + .parseInsertionDeletionListeningOptionPair( + sourcePipeParameters) + .getRight() + ? "data" + : PipeSourceConstant.EXTRACTOR_INCLUSION_DEFAULT_VALUE, PipeSourceConstant.SOURCE_EXCLUSION_KEY, PipeSourceConstant.EXTRACTOR_EXCLUSION_DEFAULT_VALUE))) .getAttribute()) @@ -1909,31 +1910,24 @@ public SettableFuture createPipe( // 3. Set success status only if both pipelines are created successfully future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); - } catch (final Exception e) { - // Catch any other exceptions (e.g., network issues) - future.setException(e); - } - return future; - } - - try (final ConfigNodeClient configNodeClient = - CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - TCreatePipeReq req = - new TCreatePipeReq() - .setPipeName(pipeName) - .setIfNotExistsCondition(createPipeStatement.hasIfNotExistsCondition()) - .setExtractorAttributes(createPipeStatement.getSourceAttributes()) - .setProcessorAttributes(createPipeStatement.getProcessorAttributes()) - .setConnectorAttributes(createPipeStatement.getSinkAttributes()); - TSStatus tsStatus = configNodeClient.createPipe(req); - if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { - LOGGER.warn( - "Failed to create pipe {} in config node, status is {}.", - createPipeStatement.getPipeName(), - tsStatus); - future.setException(new IoTDBException(tsStatus)); } else { - future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + final TCreatePipeReq req = + new TCreatePipeReq() + .setPipeName(pipeName) + .setIfNotExistsCondition(createPipeStatement.hasIfNotExistsCondition()) + .setExtractorAttributes(createPipeStatement.getSourceAttributes()) + .setProcessorAttributes(createPipeStatement.getProcessorAttributes()) + .setConnectorAttributes(createPipeStatement.getSinkAttributes()); + final TSStatus tsStatus = configNodeClient.createPipe(req); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { + LOGGER.warn( + "Failed to create pipe {} in config node, status is {}.", + createPipeStatement.getPipeName(), + tsStatus); + future.setException(new IoTDBException(tsStatus)); + } else { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } } } catch (final Exception e) { future.setException(e); @@ -2406,7 +2400,7 @@ public SettableFuture deleteTimeSeries( queryId, serializePatternListToByteBuffer(deleteTimeSeriesStatement.getPathPatternList())); try (ConfigNodeClient client = - CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { TSStatus tsStatus; do { try { @@ -2452,7 +2446,7 @@ public SettableFuture deleteLogicalView( queryId, serializePatternListToByteBuffer(deleteLogicalViewStatement.getPathPatternList())); try (ConfigNodeClient client = - CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { TSStatus tsStatus; do { try { @@ -2541,7 +2535,7 @@ public SettableFuture renameLogicalView( new TDeleteLogicalViewReq( queryId, serializePatternListToByteBuffer(Collections.singletonList(oldName))); try (ConfigNodeClient client = - CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { TSStatus tsStatus; do { try { @@ -2613,7 +2607,7 @@ public SettableFuture alterLogicalView( new TAlterLogicalViewReq( context.getQueryId().getId(), ByteBuffer.wrap(stream.toByteArray())); try (final ConfigNodeClient client = - CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { TSStatus tsStatus; do { try { @@ -2675,7 +2669,7 @@ public TSStatus alterLogicalViewByPipe( .setIsGeneratedByPipe(shouldMarkAsPipeRequest); TSStatus tsStatus; try (ConfigNodeClient client = - CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { do { try { tsStatus = client.alterLogicalView(req); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java index 401e5393aac1e..32d7b31225775 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java @@ -1164,7 +1164,8 @@ public LogicalPlanBuilder planTimeSeriesCountSource( boolean prefixPath, SchemaFilter schemaFilter, Map templateMap, - PathPatternTree scope) { + PathPatternTree scope, + boolean includeSystemDatabase) { this.root = new TimeSeriesCountNode( context.getQueryId().genPlanNodeId(), @@ -1172,7 +1173,8 @@ public LogicalPlanBuilder planTimeSeriesCountSource( prefixPath, schemaFilter, templateMap, - scope); + scope, + includeSystemDatabase); return this; } @@ -1182,7 +1184,8 @@ public LogicalPlanBuilder planLevelTimeSeriesCountSource( int level, SchemaFilter schemaFilter, Map templateMap, - PathPatternTree scope) { + PathPatternTree scope, + boolean includeSystemDatabase) { this.root = new LevelTimeSeriesCountNode( context.getQueryId().genPlanNodeId(), @@ -1191,7 +1194,8 @@ public LogicalPlanBuilder planLevelTimeSeriesCountSource( level, schemaFilter, templateMap, - scope); + scope, + includeSystemDatabase); return this; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index be0b28a0282ac..5a004d7903869 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -585,8 +585,8 @@ public PlanNode visitShowDevices( planBuilder = planBuilder .planDeviceRegionScan(analysis.getDevicePathToContextMap(), false) - .planLimit(showDevicesStatement.getLimit()) - .planOffset(showDevicesStatement.getOffset()); + .planOffset(showDevicesStatement.getOffset()) + .planLimit(showDevicesStatement.getLimit()); return planBuilder.getRoot(); } @@ -660,7 +660,8 @@ public PlanNode visitCountTimeSeries( countTimeSeriesStatement.isPrefixPath(), countTimeSeriesStatement.getSchemaFilter(), analysis.getRelatedTemplateInfo(), - countTimeSeriesStatement.getAuthorityScope()) + countTimeSeriesStatement.getAuthorityScope(), + countTimeSeriesStatement.isCanSeeSystemDB()) .planCountMerge() .getRoot(); } @@ -676,7 +677,8 @@ public PlanNode visitCountLevelTimeSeries( countLevelTimeSeriesStatement.getLevel(), countLevelTimeSeriesStatement.getSchemaFilter(), analysis.getRelatedTemplateInfo(), - countLevelTimeSeriesStatement.getAuthorityScope()) + countLevelTimeSeriesStatement.getAuthorityScope(), + countLevelTimeSeriesStatement.isCanSeeSystemDB()) .planCountMerge() .getRoot(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index e0aeacb7f56b0..2ec962c82744c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -993,7 +993,8 @@ public Operator visitTimeSeriesCount( node.isPrefixPath(), node.getSchemaFilter(), node.getTemplateMap(), - node.getScope())); + node.getScope(), + node.isIncludeSystemDatabase())); } @Override @@ -1015,7 +1016,8 @@ public Operator visitLevelTimeSeriesCount( node.isPrefixPath(), node.getSchemaFilter(), node.getTemplateMap(), - node.getScope())); + node.getScope(), + node.isIncludeSystemDatabase())); } @Override @@ -2624,14 +2626,16 @@ public Operator visitExchange(ExchangeNode node, LocalExecutionPlanContext conte node.getUpstreamPlanNodeId().getId(), remoteInstanceId.toThrift(), node.getIndexOfUpstreamSinkHandle(), - context.getInstanceContext()::failed) + context.getInstanceContext()::failed, + context.getInstanceContext().isHighestPriority()) : MPP_DATA_EXCHANGE_MANAGER.createSourceHandle( localInstanceId.toThrift(), node.getPlanNodeId().getId(), node.getIndexOfUpstreamSinkHandle(), upstreamEndPoint, remoteInstanceId.toThrift(), - context.getInstanceContext()::failed); + context.getInstanceContext()::failed, + context.getInstanceContext().isHighestPriority()); if (!isSameNode) { context.addExchangeSumNum(1); } @@ -3779,9 +3783,17 @@ private static Map getTimeseriesSchemaInfoMap( Map timeseriesSchemaInfoMap = new HashMap<>(); for (Map.Entry> entry : entryMap.getValue().entrySet()) { PartialPath path = entry.getKey(); - context.addPath(path); if (path instanceof MeasurementPath) { - timeseriesSchemaInfoMap.put(path.getMeasurement(), entry.getValue().get(0)); + String measurement = path.getMeasurement(); + TimeseriesContext timeseriesContext = entry.getValue().get(0); + TimeseriesContext existingContext = timeseriesSchemaInfoMap.get(measurement); + if (existingContext == null) { + timeseriesSchemaInfoMap.put(measurement, timeseriesContext); + context.addPath(path); + } else { + timeseriesSchemaInfoMap.put( + measurement, existingContext.mergeActiveCount(timeseriesContext)); + } } else if (path instanceof AlignedPath) { AlignedPath alignedPath = (AlignedPath) path; List measurementList = alignedPath.getMeasurementList(); @@ -3789,8 +3801,25 @@ private static Map getTimeseriesSchemaInfoMap( throw new IllegalArgumentException( "The size of measurementList and timeseriesSchemaInfoList should be equal in aligned path."); } - for (int i = 0; i < measurementList.size(); i++) { - timeseriesSchemaInfoMap.put(measurementList.get(i), entry.getValue().get(i)); + int size = measurementList.size(); + List schemaList = new ArrayList<>(size); + List newMeasurementList = new ArrayList<>(size); + List alignedSchemaList = alignedPath.getSchemaList(); + for (int i = 0; i < size; i++) { + String measurement = measurementList.get(i); + TimeseriesContext timeseriesContext = entry.getValue().get(i); + TimeseriesContext existingContext = timeseriesSchemaInfoMap.get(measurement); + if (existingContext == null) { + timeseriesSchemaInfoMap.put(measurement, timeseriesContext); + newMeasurementList.add(measurement); + schemaList.add(alignedSchemaList.get(i)); + } else { + timeseriesSchemaInfoMap.put( + measurement, existingContext.mergeActiveCount(timeseriesContext)); + } + } + if (!newMeasurementList.isEmpty()) { + context.addPath(new AlignedPath(path.getNodes(), newMeasurementList, schemaList)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TreeModelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TreeModelPlanner.java index 1da701a8e3529..2812d81cf27fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TreeModelPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TreeModelPlanner.java @@ -48,15 +48,12 @@ import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; public class TreeModelPlanner implements IPlanner { private final Statement statement; - private final ExecutorService executor; - private final ExecutorService writeOperationExecutor; private final ScheduledExecutorService scheduledExecutor; private final IPartitionFetcher partitionFetcher; @@ -71,8 +68,6 @@ public class TreeModelPlanner implements IPlanner { public TreeModelPlanner( Statement statement, - ExecutorService executor, - ExecutorService writeOperationExecutor, ScheduledExecutorService scheduledExecutor, IPartitionFetcher partitionFetcher, ISchemaFetcher schemaFetcher, @@ -80,8 +75,6 @@ public TreeModelPlanner( IClientManager asyncInternalServiceClientManager) { this.statement = statement; - this.executor = executor; - this.writeOperationExecutor = writeOperationExecutor; this.scheduledExecutor = scheduledExecutor; this.partitionFetcher = partitionFetcher; this.schemaFetcher = schemaFetcher; @@ -134,8 +127,6 @@ public IScheduler doSchedule( stateMachine, distributedPlan, context.getQueryType(), - executor, - writeOperationExecutor, scheduledExecutor, syncInternalServiceClientManager, asyncInternalServiceClientManager); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/DistributionPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/DistributionPlanner.java index c2ca7011039c9..1f1bb0c3080fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/DistributionPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/DistributionPlanner.java @@ -145,13 +145,23 @@ private void adjustUpStreamHelper( ExchangeNode exchangeNode = (ExchangeNode) child; TRegionReplicaSet regionOfChild = context.getNodeDistribution(exchangeNode.getChild().getPlanNodeId()).getRegion(); - MultiChildrenSinkNode newChild = - memo.computeIfAbsent( - regionOfChild, - tRegionReplicaSet -> - needShuffleSinkNode - ? new ShuffleSinkNode(context.queryContext.getQueryId().genPlanNodeId()) - : new IdentitySinkNode(context.queryContext.getQueryId().genPlanNodeId())); + MultiChildrenSinkNode newChild; + if (exchangeNode.isForcedExchange()) { + // Keep forced exchange branch isolated: do not merge into shared sink memo. + newChild = + needShuffleSinkNode + ? new ShuffleSinkNode(context.queryContext.getQueryId().genPlanNodeId()) + : new IdentitySinkNode(context.queryContext.getQueryId().genPlanNodeId()); + } else { + newChild = + memo.computeIfAbsent( + regionOfChild, + tRegionReplicaSet -> + needShuffleSinkNode + ? new ShuffleSinkNode(context.queryContext.getQueryId().genPlanNodeId()) + : new IdentitySinkNode( + context.queryContext.getQueryId().genPlanNodeId())); + } newChild.addChild(exchangeNode.getChild()); newChild.addDownStreamChannelLocation( new DownStreamChannelLocation(exchangeNode.getPlanNodeId().toString())); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java index f390d90c842ed..eb1490a6fc24d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java @@ -69,17 +69,17 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SeriesScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SourceNode; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static com.google.common.collect.ImmutableList.toImmutableList; - public class ExchangeNodeAdder extends PlanVisitor { private final Analysis analysis; + private boolean containsInnerTimeJoinInCurrentSubtree = false; public ExchangeNodeAdder(Analysis analysis) { this.analysis = analysis; @@ -92,10 +92,7 @@ public PlanNode visitPlan(PlanNode node, NodeGroupContext context) { return node; } // Visit all the children of current node - List children = - node.getChildren().stream() - .map(child -> child.accept(this, context)) - .collect(toImmutableList()); + List children = visitChildrenAndRecordInnerTimeJoin(node.getChildren(), context); // Calculate the node distribution info according to its children @@ -215,7 +212,13 @@ private PlanNode processNoChildSourceNode(SourceNode node, NodeGroupContext cont @Override public PlanNode visitDeviceView(DeviceViewNode node, NodeGroupContext context) { - return processMultiChildNode(node, context); + List visitedChildren = + visitChildrenAndRecordInnerTimeJoin(node.getChildren(), context); + // Force Exchange for multi-child DeviceView if any child subtree contains InnerTimeJoin. + if (node.getChildren().size() > 1 && containsInnerTimeJoinInCurrentSubtree) { + return processMultiChildNodeWithForcedExchange(node, context, visitedChildren); + } + return processMultiChildNode(node, context, visitedChildren); } @Override @@ -236,7 +239,13 @@ public PlanNode visitSingleDeviceView(SingleDeviceViewNode node, NodeGroupContex @Override public PlanNode visitMergeSort(MergeSortNode node, NodeGroupContext context) { - return processMultiChildNode(node, context); + List visitedChildren = + visitChildrenAndRecordInnerTimeJoin(node.getChildren(), context); + // Force Exchange if any child subtree contains InnerTimeJoin. + if (containsInnerTimeJoinInCurrentSubtree) { + return processMultiChildNodeWithForcedExchange(node, context, visitedChildren); + } + return processMultiChildNode(node, context, visitedChildren); } @Override @@ -428,11 +437,14 @@ private PlanNode processMultiChildNode(MultiChildProcessNode node, NodeGroupCont return processMultiChildNodeByLocation(node, context); } - MultiChildProcessNode newNode = (MultiChildProcessNode) node.clone(); List visitedChildren = - node.getChildren().stream() - .map(child -> visit(child, context)) - .collect(Collectors.toList()); + visitChildrenAndRecordInnerTimeJoin(node.getChildren(), context); + return processMultiChildNode(node, context, visitedChildren); + } + + private PlanNode processMultiChildNode( + MultiChildProcessNode node, NodeGroupContext context, List visitedChildren) { + MultiChildProcessNode newNode = (MultiChildProcessNode) node.clone(); // DataRegion in which node locates TRegionReplicaSet dataRegion; @@ -546,13 +558,47 @@ private PlanNode processTopKNode( } private ExchangeNode genExchangeNode(NodeGroupContext context, PlanNode child) { + return genExchangeNode(context, child, false); + } + + private ExchangeNode genExchangeNode( + NodeGroupContext context, PlanNode child, boolean forcedExchange) { ExchangeNode exchangeNode = new ExchangeNode(context.queryContext.getQueryId().genPlanNodeId()); exchangeNode.setChild(child); exchangeNode.setOutputColumnNames(child.getOutputColumnNames()); + exchangeNode.setForcedExchange(forcedExchange); context.hasExchangeNode = true; return exchangeNode; } + private PlanNode processMultiChildNodeWithForcedExchange( + MultiChildProcessNode node, NodeGroupContext context, List visitedChildren) { + MultiChildProcessNode newNode = (MultiChildProcessNode) node.clone(); + for (PlanNode child : visitedChildren) { + newNode.addChild(genExchangeNode(context, child, true)); + } + context.putNodeDistribution( + newNode.getPlanNodeId(), + new NodeDistribution( + NodeDistributionType.SAME_WITH_SOME_CHILD, context.getMostlyUsedDataRegion())); + return newNode; + } + + private List visitChildrenAndRecordInnerTimeJoin( + List children, NodeGroupContext context) { + List result = new ArrayList<>(children.size()); + boolean originalTimeJoin = containsInnerTimeJoinInCurrentSubtree; + boolean hasInnerTimeJoinInChildren = false; + for (PlanNode child : children) { + containsInnerTimeJoinInCurrentSubtree = false; + PlanNode visitedChild = visit(child, context); + hasInnerTimeJoinInChildren |= containsInnerTimeJoinInCurrentSubtree; + result.add(visitedChild); + } + containsInnerTimeJoinInCurrentSubtree = originalTimeJoin || hasInnerTimeJoinInChildren; + return result; + } + @Override public PlanNode visitSlidingWindowAggregation( SlidingWindowAggregationNode node, NodeGroupContext context) { @@ -584,9 +630,9 @@ private TRegionReplicaSet calculateDataRegionByChildren( if (region == null && context.getNodeDistribution(child.getPlanNodeId()).getType() == NodeDistributionType.SAME_WITH_ALL_CHILDREN) { - return calculateSchemaRegionByChildren(child.getChildren(), context); + region = calculateSchemaRegionByChildren(child.getChildren(), context); } - return region; + return region == null ? DataPartition.NOT_ASSIGNED : region; }, Collectors.counting())); @@ -644,6 +690,13 @@ private boolean nodeDistributionIsSame(List children, NodeGroupContext } public PlanNode visit(PlanNode node, NodeGroupContext context) { - return node.accept(this, context); + boolean originalTimeJoin = containsInnerTimeJoinInCurrentSubtree; + containsInnerTimeJoinInCurrentSubtree = false; + PlanNode visitedNode = node.accept(this, context); + containsInnerTimeJoinInCurrentSubtree = + originalTimeJoin + || containsInnerTimeJoinInCurrentSubtree + || node instanceof InnerTimeJoinNode; + return visitedNode; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java index 380a351f65ed3..60bb2807668bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java @@ -823,7 +823,10 @@ private List processRegionScan(RegionScanNode node, DistributionPlanCo return planNodeList; } - boolean outputCountInScanNode = node.isOutputCount() && !context.isOneSeriesInMultiRegion(); + boolean outputCountInScanNode = + node.isOutputCount() + && !context.isOneSeriesInMultiRegion() + && !hasActiveLogicalViewContext(node); ActiveRegionScanMergeNode regionMergeNode = new ActiveRegionScanMergeNode( context.queryContext.getQueryId().genPlanNodeId(), @@ -837,6 +840,11 @@ private List processRegionScan(RegionScanNode node, DistributionPlanCo return Collections.singletonList(regionMergeNode); } + private boolean hasActiveLogicalViewContext(RegionScanNode node) { + return node instanceof TimeseriesRegionScanNode + && ((TimeseriesRegionScanNode) node).hasActiveLogicalViewContext(); + } + @Override public List visitDeviceRegionScan( DeviceRegionScanNode node, DistributionPlanContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java index d212f4ca72526..b96485199ad4e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java @@ -223,6 +223,7 @@ public static FragmentInstance deserializeFrom(ByteBuffer buffer) { fragmentInstance.hostDataNode = hasHostDataNode ? ThriftCommonsSerDeUtils.deserializeTDataNodeLocation(buffer) : null; fragmentInstance.isExplainAnalyze = ReadWriteIOUtils.readBool(buffer); + fragmentInstance.setHighestPriority(ReadWriteIOUtils.readBool(buffer)); return fragmentInstance; } @@ -247,6 +248,7 @@ public ByteBuffer serializeToByteBuffer() { ThriftCommonsSerDeUtils.serializeTDataNodeLocation(hostDataNode, outputStream); } ReadWriteIOUtils.write(isExplainAnalyze, outputStream); + ReadWriteIOUtils.write(isHighestPriority, outputStream); return ByteBuffer.wrap(publicBAOS.getBuf(), 0, publicBAOS.size()); } catch (IOException e) { LOGGER.error("Unexpected error occurs when serializing this FragmentInstance.", e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 64b4655be2863..a168228deb30f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -79,27 +79,35 @@ public boolean isTsFileEmpty() { public boolean needDecodeTsFile( Function>, List> partitionFetcher) { - List> slotList = new ArrayList<>(); - resource - .getDevices() - .forEach( - o -> { - // iterating the index, must present - slotList.add( - new Pair<>( - o, TimePartitionUtils.getTimePartitionSlot(resource.getStartTime(o).get()))); - slotList.add( - new Pair<>( - o, TimePartitionUtils.getTimePartitionSlot(resource.getEndTime(o).get()))); - }); + if (needDecodeTsFile) { + return true; + } + + List> slotList = + new ArrayList<>(resource.getDevices().size() << 1); + for (final IDeviceID device : resource.getDevices()) { + // iterating the index, must present + final TTimePartitionSlot startSlot = + TimePartitionUtils.getTimePartitionSlot(resource.getStartTime(device).get()); + final TTimePartitionSlot endSlot = + TimePartitionUtils.getTimePartitionSlot(resource.getEndTime(device).get()); + slotList.add(new Pair<>(device, startSlot)); + if (!startSlot.equals(endSlot)) { + slotList.add(new Pair<>(device, endSlot)); + } + } if (slotList.isEmpty()) { throw new IllegalStateException( String.format("Devices in TsFile %s is empty, this should not happen here.", tsFile)); - } else if (slotList.stream() - .anyMatch(slotPair -> !slotPair.getRight().equals(slotList.get(0).right))) { - needDecodeTsFile = true; } else { + final TTimePartitionSlot firstSlot = slotList.get(0).right; + for (int i = 1, size = slotList.size(); i < size; i++) { + if (!slotList.get(i).right.equals(firstSlot)) { + needDecodeTsFile = true; + return true; + } + } needDecodeTsFile = !isDispatchedToLocal(new HashSet<>(partitionFetcher.apply(slotList))); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/LevelTimeSeriesCountNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/LevelTimeSeriesCountNode.java index 2178e06c9e759..c4c0589330088 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/LevelTimeSeriesCountNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/LevelTimeSeriesCountNode.java @@ -47,6 +47,7 @@ public class LevelTimeSeriesCountNode extends SchemaQueryScanNode { private final int level; private final SchemaFilter schemaFilter; private final Map templateMap; + private final boolean includeSystemDatabase; public LevelTimeSeriesCountNode( PlanNodeId id, @@ -55,11 +56,13 @@ public LevelTimeSeriesCountNode( int level, SchemaFilter schemaFilter, @NotNull Map templateMap, - @NotNull PathPatternTree scope) { + @NotNull PathPatternTree scope, + boolean includeSystemDatabase) { super(id, partialPath, isPrefixPath, scope); this.level = level; this.schemaFilter = schemaFilter; this.templateMap = templateMap; + this.includeSystemDatabase = includeSystemDatabase; } public SchemaFilter getSchemaFilter() { @@ -74,6 +77,10 @@ public Map getTemplateMap() { return templateMap; } + public boolean isIncludeSystemDatabase() { + return includeSystemDatabase; + } + @Override public PlanNodeType getType() { return PlanNodeType.LEVEL_TIME_SERIES_COUNT; @@ -82,7 +89,14 @@ public PlanNodeType getType() { @Override public PlanNode clone() { return new LevelTimeSeriesCountNode( - getPlanNodeId(), path, isPrefixPath, level, schemaFilter, templateMap, scope); + getPlanNodeId(), + path, + isPrefixPath, + level, + schemaFilter, + templateMap, + scope, + includeSystemDatabase); } @Override @@ -100,6 +114,7 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { ReadWriteIOUtils.write(isPrefixPath, byteBuffer); ReadWriteIOUtils.write(level, byteBuffer); SchemaFilter.serialize(schemaFilter, byteBuffer); + ReadWriteIOUtils.write(includeSystemDatabase, byteBuffer); ReadWriteIOUtils.write(templateMap.size(), byteBuffer); for (Template template : templateMap.values()) { template.serialize(byteBuffer); @@ -114,6 +129,7 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(isPrefixPath, stream); ReadWriteIOUtils.write(level, stream); SchemaFilter.serialize(schemaFilter, stream); + ReadWriteIOUtils.write(includeSystemDatabase, stream); ReadWriteIOUtils.write(templateMap.size(), stream); for (Template template : templateMap.values()) { template.serialize(stream); @@ -132,6 +148,7 @@ public static PlanNode deserialize(ByteBuffer buffer) { boolean isPrefixPath = ReadWriteIOUtils.readBool(buffer); int level = ReadWriteIOUtils.readInt(buffer); SchemaFilter schemaFilter = SchemaFilter.deserialize(buffer); + boolean includeSystemDatabase = ReadWriteIOUtils.readBool(buffer); int templateNum = ReadWriteIOUtils.readInt(buffer); Map templateMap = new HashMap<>(); Template template; @@ -142,7 +159,14 @@ public static PlanNode deserialize(ByteBuffer buffer) { } PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); return new LevelTimeSeriesCountNode( - planNodeId, path, isPrefixPath, level, schemaFilter, templateMap, scope); + planNodeId, + path, + isPrefixPath, + level, + schemaFilter, + templateMap, + scope, + includeSystemDatabase); } @Override @@ -157,12 +181,12 @@ public boolean equals(Object o) { return false; } LevelTimeSeriesCountNode that = (LevelTimeSeriesCountNode) o; - return level == that.level; + return level == that.level && includeSystemDatabase == that.includeSystemDatabase; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), level); + return Objects.hash(super.hashCode(), level, includeSystemDatabase); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TimeSeriesCountNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TimeSeriesCountNode.java index 473d59b8d1532..6f7483132a31a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TimeSeriesCountNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/TimeSeriesCountNode.java @@ -49,16 +49,20 @@ public class TimeSeriesCountNode extends SchemaQueryScanNode { private final Map templateMap; + private final boolean includeSystemDatabase; + public TimeSeriesCountNode( PlanNodeId id, PartialPath partialPath, boolean isPrefixPath, SchemaFilter schemaFilter, @NotNull Map templateMap, - @NotNull PathPatternTree scope) { + @NotNull PathPatternTree scope, + boolean includeSystemDatabase) { super(id, partialPath, isPrefixPath, scope); this.schemaFilter = schemaFilter; this.templateMap = templateMap; + this.includeSystemDatabase = includeSystemDatabase; } public SchemaFilter getSchemaFilter() { @@ -69,6 +73,10 @@ public Map getTemplateMap() { return templateMap; } + public boolean isIncludeSystemDatabase() { + return includeSystemDatabase; + } + @Override public PlanNodeType getType() { return PlanNodeType.TIME_SERIES_COUNT; @@ -77,7 +85,13 @@ public PlanNodeType getType() { @Override public PlanNode clone() { return new TimeSeriesCountNode( - getPlanNodeId(), path, isPrefixPath, schemaFilter, templateMap, scope); + getPlanNodeId(), + path, + isPrefixPath, + schemaFilter, + templateMap, + scope, + includeSystemDatabase); } @Override @@ -94,6 +108,7 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { scope.serialize(byteBuffer); ReadWriteIOUtils.write(isPrefixPath, byteBuffer); SchemaFilter.serialize(schemaFilter, byteBuffer); + ReadWriteIOUtils.write(includeSystemDatabase, byteBuffer); ReadWriteIOUtils.write(templateMap.size(), byteBuffer); for (Template template : templateMap.values()) { template.serialize(byteBuffer); @@ -107,6 +122,7 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { scope.serialize(stream); ReadWriteIOUtils.write(isPrefixPath, stream); SchemaFilter.serialize(schemaFilter, stream); + ReadWriteIOUtils.write(includeSystemDatabase, stream); ReadWriteIOUtils.write(templateMap.size(), stream); for (Template template : templateMap.values()) { template.serialize(stream); @@ -124,6 +140,7 @@ public static PlanNode deserialize(ByteBuffer buffer) { PathPatternTree scope = PathPatternTree.deserialize(buffer); boolean isPrefixPath = ReadWriteIOUtils.readBool(buffer); SchemaFilter schemaFilter = SchemaFilter.deserialize(buffer); + boolean includeSystemDatabase = ReadWriteIOUtils.readBool(buffer); int templateNum = ReadWriteIOUtils.readInt(buffer); Map templateMap = new HashMap<>(); @@ -136,7 +153,7 @@ public static PlanNode deserialize(ByteBuffer buffer) { PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); return new TimeSeriesCountNode( - planNodeId, path, isPrefixPath, schemaFilter, templateMap, scope); + planNodeId, path, isPrefixPath, schemaFilter, templateMap, scope, includeSystemDatabase); } @Override @@ -153,11 +170,12 @@ public boolean equals(Object o) { if (!super.equals(o)) return false; TimeSeriesCountNode that = (TimeSeriesCountNode) o; return Objects.equals(schemaFilter, that.schemaFilter) - && Objects.equals(templateMap, that.templateMap); + && Objects.equals(templateMap, that.templateMap) + && includeSystemDatabase == that.includeSystemDatabase; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), schemaFilter, templateMap); + return Objects.hash(super.hashCode(), schemaFilter, templateMap, includeSystemDatabase); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/ExchangeNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/ExchangeNode.java index a8cfa04c41043..9a1eef8ae1411 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/ExchangeNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/ExchangeNode.java @@ -48,6 +48,9 @@ public class ExchangeNode extends SingleChildProcessNode { /** Exchange needs to know which child of IdentitySinkNode/ShuffleSinkNode it matches */ private int indexOfUpstreamSinkHandle = 0; + /** Planner-only flag: this exchange is forced and should keep independent upstream sink. */ + private transient boolean forcedExchange = false; + public ExchangeNode(PlanNodeId id) { super(id); } @@ -72,6 +75,7 @@ public PlanNode clone() { ExchangeNode node = new ExchangeNode(getPlanNodeId()); node.setOutputColumnNames(outputColumnNames); node.setIndexOfUpstreamSinkHandle(indexOfUpstreamSinkHandle); + node.setForcedExchange(forcedExchange); return node; } @@ -162,6 +166,14 @@ public void setIndexOfUpstreamSinkHandle(int indexOfUpstreamSinkHandle) { this.indexOfUpstreamSinkHandle = indexOfUpstreamSinkHandle; } + public boolean isForcedExchange() { + return forcedExchange; + } + + public void setForcedExchange(boolean forcedExchange) { + this.forcedExchange = forcedExchange; + } + public TEndPoint getUpstreamEndpoint() { return upstreamEndpoint; } @@ -188,11 +200,13 @@ public boolean equals(Object o) { ExchangeNode that = (ExchangeNode) o; return Objects.equals(upstreamEndpoint, that.upstreamEndpoint) && Objects.equals(upstreamInstanceId, that.upstreamInstanceId) - && Objects.equals(upstreamPlanNodeId, that.upstreamPlanNodeId); + && Objects.equals(upstreamPlanNodeId, that.upstreamPlanNodeId) + && forcedExchange == that.forcedExchange; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), upstreamEndpoint, upstreamInstanceId, upstreamPlanNodeId); + return Objects.hash( + super.hashCode(), upstreamEndpoint, upstreamInstanceId, upstreamPlanNodeId, forcedExchange); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/TimeseriesRegionScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/TimeseriesRegionScanNode.java index 5a5abed1aa020..d905c8eca2c4e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/TimeseriesRegionScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/TimeseriesRegionScanNode.java @@ -85,6 +85,13 @@ public void setDeviceToTimeseriesSchemaInfo( return deviceToTimeseriesSchemaInfo; } + public boolean hasActiveLogicalViewContext() { + return deviceToTimeseriesSchemaInfo.values().stream() + .flatMap(timeseriesContextMap -> timeseriesContextMap.values().stream()) + .flatMap(List::stream) + .anyMatch(context -> !context.getActiveLogicalViewContextMap().isEmpty()); + } + @Override public List getChildren() { return ImmutableList.of(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java index c4c4159f66dea..dda2d245f0921 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java @@ -36,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALReadUtils; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; import org.apache.tsfile.read.filter.factory.TimeFilterApi; @@ -129,7 +130,7 @@ public List getOutputColumnNames() { public int serializedSize() { int size = FIXED_SERIALIZED_SIZE; for (PartialPath path : pathList) { - size += ReadWriteIOUtils.sizeToWrite(path.getFullPath()); + size += WALWriteUtils.sizeToWrite(path.getFullPath()); } return size; } @@ -153,10 +154,9 @@ public static DeleteDataNode deserializeFromWAL(DataInputStream stream) throws I for (int i = 0; i < size; i++) { try { pathList.add( - DataNodeDevicePathCache.getInstance() - .getPartialPath(ReadWriteIOUtils.readString(stream))); + DataNodeDevicePathCache.getInstance().getPartialPath(WALReadUtils.readString(stream))); } catch (IllegalPathException e) { - throw new IllegalArgumentException("Cannot deserialize InsertRowNode", e); + throw new IllegalArgumentException("Cannot deserialize DeleteDataNode", e); } } long deleteStartTime = stream.readLong(); @@ -174,9 +174,9 @@ public static DeleteDataNode deserializeFromWAL(ByteBuffer buffer) { List pathList = new ArrayList<>(size); for (int i = 0; i < size; i++) { try { - pathList.add(new PartialPath(ReadWriteIOUtils.readString(buffer))); + pathList.add(new PartialPath(WALReadUtils.readString(buffer))); } catch (IllegalPathException e) { - throw new IllegalArgumentException("Cannot deserialize InsertRowNode", e); + throw new IllegalArgumentException("Cannot deserialize DeleteDataNode", e); } } long deleteStartTime = buffer.getLong(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java index c664bbb9ddfaa..aee5cc1ec9384 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java @@ -31,11 +31,14 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.storageengine.dataregion.memtable.DeviceIDFactory; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALReadUtils; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NotImplementedException; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.schema.MeasurementSchema; import java.io.DataInputStream; @@ -43,7 +46,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; @@ -213,6 +218,102 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { throw new NotImplementedException("serializeAttributes of InsertNode is not implemented"); } + protected static int serializeString(String value, ByteBuffer buffer) { + if (value == null) { + return ReadWriteIOUtils.write(-1, buffer); + } + byte[] bytes = value.getBytes(TSFileConfig.STRING_CHARSET); + int len = ReadWriteIOUtils.write(bytes.length, buffer); + buffer.put(bytes); + return len + bytes.length; + } + + protected static int serializeString(String value, DataOutputStream stream) throws IOException { + if (value == null) { + return ReadWriteIOUtils.write(-1, stream); + } + byte[] bytes = value.getBytes(TSFileConfig.STRING_CHARSET); + int len = ReadWriteIOUtils.write(bytes.length, stream); + stream.write(bytes); + return len + bytes.length; + } + + protected static String deserializeString(ByteBuffer buffer) { + int strLength = ReadWriteIOUtils.readInt(buffer); + if (strLength < 0) { + return null; + } else if (strLength == 0) { + return ""; + } + byte[] bytes = new byte[strLength]; + buffer.get(bytes); + return new String(bytes, TSFileConfig.STRING_CHARSET); + } + + protected static void serializeMeasurementSchema( + MeasurementSchema measurementSchema, ByteBuffer buffer) { + serializeString(measurementSchema.getMeasurementId(), buffer); + ReadWriteIOUtils.write(measurementSchema.getTypeInByte(), buffer); + ReadWriteIOUtils.write(measurementSchema.getEncodingType().serialize(), buffer); + ReadWriteIOUtils.write(measurementSchema.getCompressor().serialize(), buffer); + serializeProps(measurementSchema.getProps(), buffer); + } + + protected static void serializeMeasurementSchema( + MeasurementSchema measurementSchema, DataOutputStream stream) throws IOException { + serializeString(measurementSchema.getMeasurementId(), stream); + ReadWriteIOUtils.write(measurementSchema.getTypeInByte(), stream); + ReadWriteIOUtils.write(measurementSchema.getEncodingType().serialize(), stream); + ReadWriteIOUtils.write(measurementSchema.getCompressor().serialize(), stream); + serializeProps(measurementSchema.getProps(), stream); + } + + protected static MeasurementSchema deserializeMeasurementSchema(ByteBuffer buffer) { + String measurementId = deserializeString(buffer); + byte type = ReadWriteIOUtils.readByte(buffer); + byte encoding = ReadWriteIOUtils.readByte(buffer); + byte compressor = ReadWriteIOUtils.readByte(buffer); + Map props = deserializeProps(buffer); + return new MeasurementSchema(measurementId, type, encoding, compressor, props); + } + + private static void serializeProps(Map props, ByteBuffer buffer) { + if (props == null) { + ReadWriteIOUtils.write(0, buffer); + return; + } + ReadWriteIOUtils.write(props.size(), buffer); + for (Map.Entry entry : props.entrySet()) { + serializeString(entry.getKey(), buffer); + serializeString(entry.getValue(), buffer); + } + } + + private static void serializeProps(Map props, DataOutputStream stream) + throws IOException { + if (props == null) { + ReadWriteIOUtils.write(0, stream); + return; + } + ReadWriteIOUtils.write(props.size(), stream); + for (Map.Entry entry : props.entrySet()) { + serializeString(entry.getKey(), stream); + serializeString(entry.getValue(), stream); + } + } + + private static Map deserializeProps(ByteBuffer buffer) { + int size = ReadWriteIOUtils.readInt(buffer); + if (size <= 0) { + return null; + } + Map props = new HashMap<>(); + for (int i = 0; i < size; i++) { + props.put(deserializeString(buffer), deserializeString(buffer)); + } + return props; + } + // region Serialization methods for WAL /** Serialized size of measurement schemas, ignoring failed time series */ protected int serializeMeasurementSchemasSize() { @@ -244,7 +345,7 @@ protected void serializeMeasurementSchemasToWAL(IWALByteBufferView buffer) { */ protected void deserializeMeasurementSchemas(DataInputStream stream) throws IOException { for (int i = 0; i < measurements.length; i++) { - measurementSchemas[i] = MeasurementSchema.deserializeFrom(stream); + measurementSchemas[i] = WALReadUtils.readMeasurementSchema(stream); measurements[i] = measurementSchemas[i].getMeasurementId(); dataTypes[i] = measurementSchemas[i].getType(); } @@ -252,7 +353,7 @@ protected void deserializeMeasurementSchemas(DataInputStream stream) throws IOEx protected void deserializeMeasurementSchemas(ByteBuffer buffer) { for (int i = 0; i < measurements.length; i++) { - measurementSchemas[i] = MeasurementSchema.deserializeFrom(buffer); + measurementSchemas[i] = WALReadUtils.readMeasurementSchema(buffer); measurements[i] = measurementSchemas[i].getMeasurementId(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index 6518385c48a86..23c17808d493c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALReadUtils; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; import org.apache.iotdb.db.utils.TypeInferenceUtils; @@ -238,13 +239,13 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { void subSerialize(ByteBuffer buffer) { ReadWriteIOUtils.write(time, buffer); - ReadWriteIOUtils.write(devicePath.getFullPath(), buffer); + serializeString(devicePath.getFullPath(), buffer); serializeMeasurementsAndValues(buffer); } void subSerialize(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(time, stream); - ReadWriteIOUtils.write(devicePath.getFullPath(), stream); + serializeString(devicePath.getFullPath(), stream); serializeMeasurementsAndValues(stream); } @@ -281,9 +282,9 @@ private void serializeMeasurementsOrSchemas(ByteBuffer buffer) { } // serialize measurement schemas when exist if (measurementSchemas != null) { - measurementSchemas[i].serializeTo(buffer); + serializeMeasurementSchema(measurementSchemas[i], buffer); } else { - ReadWriteIOUtils.write(measurements[i], buffer); + serializeString(measurements[i], buffer); } } } @@ -303,9 +304,9 @@ private void serializeMeasurementsOrSchemas(DataOutputStream stream) throws IOEx } // serialize measurement schemas when exist if (measurementSchemas != null) { - measurementSchemas[i].serializeTo(stream); + serializeMeasurementSchema(measurementSchemas[i], stream); } else { - ReadWriteIOUtils.write(measurements[i], stream); + serializeString(measurements[i], stream); } } } @@ -331,7 +332,7 @@ private void putDataTypesAndValues(ByteBuffer buffer) { // and is forwarded to other nodes if (isNeedInferType) { ReadWriteIOUtils.write(TYPE_RAW_STRING, buffer); - ReadWriteIOUtils.write(values[i].toString(), buffer); + serializeString(values[i].toString(), buffer); } else { ReadWriteIOUtils.write(dataTypes[i], buffer); switch (dataTypes[i]) { @@ -386,7 +387,7 @@ private void putDataTypesAndValues(DataOutputStream stream) throws IOException { // and is forwarded to other nodes if (isNeedInferType) { ReadWriteIOUtils.write(TYPE_RAW_STRING, stream); - ReadWriteIOUtils.write(values[i].toString(), stream); + serializeString(values[i].toString(), stream); } else { ReadWriteIOUtils.write(dataTypes[i], stream); switch (dataTypes[i]) { @@ -431,8 +432,7 @@ void subDeserialize(ByteBuffer byteBuffer) { time = byteBuffer.getLong(); try { devicePath = - DataNodeDevicePathCache.getInstance() - .getPartialPath(ReadWriteIOUtils.readString(byteBuffer)); + DataNodeDevicePathCache.getInstance().getPartialPath(deserializeString(byteBuffer)); } catch (IllegalPathException e) { throw new IllegalArgumentException(DESERIALIZE_ERROR, e); } @@ -447,12 +447,12 @@ void deserializeMeasurementsAndValues(ByteBuffer buffer) { if (hasSchema) { measurementSchemas = new MeasurementSchema[measurementSize]; for (int i = 0; i < measurementSize; i++) { - measurementSchemas[i] = MeasurementSchema.deserializeFrom(buffer); + measurementSchemas[i] = deserializeMeasurementSchema(buffer); measurements[i] = measurementSchemas[i].getMeasurementId(); } } else { for (int i = 0; i < measurementSize; i++) { - measurements[i] = ReadWriteIOUtils.readString(buffer); + measurements[i] = deserializeString(buffer); } } @@ -476,7 +476,7 @@ private void fillDataTypesAndValues(ByteBuffer buffer) { // and is forwarded to other nodes byte typeNum = (byte) ReadWriteIOUtils.read(buffer); if (typeNum == TYPE_RAW_STRING || typeNum == TYPE_NULL) { - values[i] = typeNum == TYPE_RAW_STRING ? ReadWriteIOUtils.readString(buffer) : null; + values[i] = typeNum == TYPE_RAW_STRING ? deserializeString(buffer) : null; continue; } dataTypes[i] = TSDataType.values()[typeNum]; @@ -524,7 +524,7 @@ public int serializedSize() { protected int subSerializeSize() { int size = 0; size += Long.BYTES; - size += ReadWriteIOUtils.sizeToWrite(devicePath.getFullPath()); + size += WALWriteUtils.sizeToWrite(devicePath.getFullPath()); return size + serializeMeasurementsAndValuesSize(); } @@ -671,8 +671,7 @@ protected static InsertRowNode subDeserializeFromWAL(DataInputStream stream) thr insertNode.setTime(stream.readLong()); try { insertNode.setDevicePath( - DataNodeDevicePathCache.getInstance() - .getPartialPath(ReadWriteIOUtils.readString(stream))); + DataNodeDevicePathCache.getInstance().getPartialPath(WALReadUtils.readString(stream))); } catch (IllegalPathException e) { throw new IllegalArgumentException(DESERIALIZE_ERROR, e); } @@ -757,8 +756,7 @@ protected static InsertRowNode subDeserializeFromWAL(ByteBuffer buffer) { insertNode.setTime(buffer.getLong()); try { insertNode.setDevicePath( - DataNodeDevicePathCache.getInstance() - .getPartialPath(ReadWriteIOUtils.readString(buffer))); + DataNodeDevicePathCache.getInstance().getPartialPath(WALReadUtils.readString(buffer))); } catch (IllegalPathException e) { throw new IllegalArgumentException(DESERIALIZE_ERROR, e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java index c7b193841d45c..d7a6e8b6f5f59 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java @@ -236,8 +236,7 @@ public static InsertRowsOfOneDeviceNode deserialize(ByteBuffer byteBuffer) { try { devicePath = - DataNodeDevicePathCache.getInstance() - .getPartialPath((ReadWriteIOUtils.readString(byteBuffer))); + DataNodeDevicePathCache.getInstance().getPartialPath((deserializeString(byteBuffer))); } catch (IllegalPathException e) { throw new IllegalArgumentException("Cannot deserialize InsertRowsOfOneDeviceNode", e); } @@ -269,7 +268,7 @@ public static InsertRowsOfOneDeviceNode deserialize(ByteBuffer byteBuffer) { @Override protected void serializeAttributes(ByteBuffer byteBuffer) { PlanNodeType.INSERT_ROWS_OF_ONE_DEVICE.serialize(byteBuffer); - ReadWriteIOUtils.write(devicePath.getFullPath(), byteBuffer); + serializeString(devicePath.getFullPath(), byteBuffer); ReadWriteIOUtils.write(insertRowNodeList.size(), byteBuffer); @@ -285,7 +284,7 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { @Override protected void serializeAttributes(DataOutputStream stream) throws IOException { PlanNodeType.INSERT_ROWS_OF_ONE_DEVICE.serialize(stream); - ReadWriteIOUtils.write(devicePath.getFullPath(), stream); + serializeString(devicePath.getFullPath(), stream); ReadWriteIOUtils.write(insertRowNodeList.size(), stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index a366aed71b3e1..472c1e79d7e47 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; @@ -35,8 +36,11 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALReadUtils; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; +import org.apache.iotdb.db.utils.BitMapUtils; import org.apache.iotdb.db.utils.QueryDataSetUtils; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.NotImplementedException; @@ -275,7 +279,7 @@ public List splitByPartition(IAnalysis analysis) { long[] subTimes = new long[count]; int destLoc = 0; Object[] values = initTabletValues(dataTypes.length, count, dataTypes); - BitMap[] bitMaps = this.bitMaps == null ? null : initBitmaps(dataTypes.length, count); + BitMap[] bitMaps = initBitmapsForSplit(dataTypes.length, count); System.arraycopy(times, start, subTimes, destLoc, end - start); for (int k = 0; k < values.length; k++) { if (dataTypes[k] != null) { @@ -300,6 +304,7 @@ public List splitByPartition(IAnalysis analysis) { subNode.setFailedMeasurementNumber(getFailedMeasurementNumber()); subNode.setRange(locs); subNode.setDataRegionReplicaSet(entry.getKey()); + subNode.bitMaps = BitMapUtils.compactBitMaps(subNode.bitMaps, subNode.rowCount); result.add(subNode); } } @@ -364,6 +369,23 @@ private BitMap[] initBitmaps(int columnSize, int rowSize) { return bitMaps; } + protected BitMap[] initBitmapsForSplit(int columnSize, int rowSize) { + if (this.bitMaps == null) { + return null; + } + + final int sourceRowCount = rowCount > 0 ? rowCount : times == null ? 0 : times.length; + final BitMap[] splitBitMaps = new BitMap[columnSize]; + boolean hasBitMap = false; + for (int i = 0; i < columnSize && i < this.bitMaps.length; ++i) { + if (this.bitMaps[i] != null && !this.bitMaps[i].isAllUnmarked()) { + splitBitMaps[i] = new BitMap(rowSize); + hasBitMap = true; + } + } + return hasBitMap ? splitBitMaps : null; + } + @Override public void markFailedMeasurement(int index) { if (measurements[index] == null) { @@ -392,7 +414,7 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { } void subSerialize(ByteBuffer buffer) { - ReadWriteIOUtils.write(devicePath.getFullPath(), buffer); + serializeString(devicePath.getFullPath(), buffer); writeMeasurementsOrSchemas(buffer); writeDataTypes(buffer); writeTimes(buffer); @@ -402,7 +424,7 @@ void subSerialize(ByteBuffer buffer) { } void subSerialize(DataOutputStream stream) throws IOException { - ReadWriteIOUtils.write(devicePath.getFullPath(), stream); + serializeString(devicePath.getFullPath(), stream); writeMeasurementsOrSchemas(stream); writeDataTypes(stream); writeTimes(stream); @@ -423,9 +445,9 @@ private void writeMeasurementsOrSchemas(ByteBuffer buffer) { } // serialize measurement schemas when exist if (measurementSchemas != null) { - measurementSchemas[i].serializeTo(buffer); + serializeMeasurementSchema(measurementSchemas[i], buffer); } else { - ReadWriteIOUtils.write(measurements[i], buffer); + serializeString(measurements[i], buffer); } } } @@ -442,9 +464,9 @@ private void writeMeasurementsOrSchemas(DataOutputStream stream) throws IOExcept } // serialize measurement schemas when exist if (measurementSchemas != null) { - measurementSchemas[i].serializeTo(stream); + serializeMeasurementSchema(measurementSchemas[i], stream); } else { - ReadWriteIOUtils.write(measurements[i], stream); + serializeString(measurements[i], stream); } } } @@ -661,8 +683,7 @@ public static InsertTabletNode deserialize(ByteBuffer byteBuffer) { public void subDeserialize(ByteBuffer buffer) { try { devicePath = - DataNodeDevicePathCache.getInstance() - .getPartialPath((ReadWriteIOUtils.readString(buffer))); + DataNodeDevicePathCache.getInstance().getPartialPath((deserializeString(buffer))); } catch (IllegalPathException e) { throw new IllegalArgumentException("Cannot deserialize InsertTabletNode", e); } @@ -674,12 +695,12 @@ public void subDeserialize(ByteBuffer buffer) { if (hasSchema) { this.measurementSchemas = new MeasurementSchema[measurementSize]; for (int i = 0; i < measurementSize; i++) { - measurementSchemas[i] = MeasurementSchema.deserializeFrom(buffer); + measurementSchemas[i] = deserializeMeasurementSchema(buffer); measurements[i] = measurementSchemas[i].getMeasurementId(); } } else { for (int i = 0; i < measurementSize; i++) { - measurements[i] = ReadWriteIOUtils.readString(buffer); + measurements[i] = deserializeString(buffer); } } @@ -717,7 +738,7 @@ public int serializedSize(int start, int end) { int subSerializeSize(int start, int end) { int size = 0; size += Long.BYTES; - size += ReadWriteIOUtils.sizeToWrite(devicePath.getFullPath()); + size += WALWriteUtils.sizeToWrite(devicePath.getFullPath()); // measurements size size += Integer.BYTES; size += serializeMeasurementSchemasSize(); @@ -922,7 +943,7 @@ private void subDeserializeFromWAL(DataInputStream stream) throws IOException { searchIndex = stream.readLong(); try { devicePath = - DataNodeDevicePathCache.getInstance().getPartialPath(ReadWriteIOUtils.readString(stream)); + DataNodeDevicePathCache.getInstance().getPartialPath(WALReadUtils.readString(stream)); } catch (IllegalPathException e) { throw new IllegalArgumentException("Cannot deserialize InsertTabletNode", e); } @@ -958,8 +979,7 @@ private void subDeserializeFromWAL(ByteBuffer buffer) { searchIndex = buffer.getLong(); try { devicePath = - DataNodeDevicePathCache.getInstance() - .getPartialPath((ReadWriteIOUtils.readString(buffer))); + DataNodeDevicePathCache.getInstance().getPartialPath((WALReadUtils.readString(buffer))); } catch (IllegalPathException e) { throw new IllegalArgumentException("Cannot deserialize InsertTabletNode", e); } @@ -1076,68 +1096,105 @@ public R accept(PlanVisitor visitor, C context) { return visitor.visitInsertTablet(this, context); } - public TimeValuePair composeLastTimeValuePair(int measurementIndex) { + public TimeValuePair composeLastTimeValuePair( + int measurementIndex, int startOffset, int endOffset) { if (measurementIndex >= columns.length || Objects.isNull(dataTypes[measurementIndex])) { return null; } // get non-null value - int lastIdx = rowCount - 1; + int lastIdx = Math.min(endOffset - 1, rowCount - 1); if (bitMaps != null && bitMaps[measurementIndex] != null) { BitMap bitMap = bitMaps[measurementIndex]; - while (lastIdx >= 0) { + while (lastIdx >= startOffset) { if (!bitMap.isMarked(lastIdx)) { break; } lastIdx--; } } - if (lastIdx < 0) { + if (lastIdx < startOffset) { + return null; + } + return composeTimeValuePair(measurementIndex, lastIdx); + } + + protected TimeValuePair composeLastTimeValuePair( + final int measurementIndex, + final TSStatus[] results, + final int startOffset, + final int endOffset) { + if (results == null) { + return composeLastTimeValuePair(measurementIndex, startOffset, endOffset); + } + if (measurementIndex >= columns.length || Objects.isNull(dataTypes[measurementIndex])) { return null; } + final BitMap bitMap = bitMaps == null ? null : bitMaps[measurementIndex]; + int lastIdx = Math.min(endOffset - 1, rowCount - 1); + while (lastIdx >= startOffset) { + if (results[lastIdx] != null + && results[lastIdx].getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + lastIdx--; + continue; + } + if (bitMap != null && bitMap.isMarked(lastIdx)) { + lastIdx--; + continue; + } + break; + } + return lastIdx < startOffset ? null : composeTimeValuePair(measurementIndex, lastIdx); + } + + private TimeValuePair composeTimeValuePair(final int measurementIndex, final int rowIndex) { TsPrimitiveType value; switch (dataTypes[measurementIndex]) { case INT32: case DATE: int[] intValues = (int[]) columns[measurementIndex]; - value = new TsPrimitiveType.TsInt(intValues[lastIdx]); + value = new TsPrimitiveType.TsInt(intValues[rowIndex]); break; case INT64: case TIMESTAMP: long[] longValues = (long[]) columns[measurementIndex]; - value = new TsPrimitiveType.TsLong(longValues[lastIdx]); + value = new TsPrimitiveType.TsLong(longValues[rowIndex]); break; case FLOAT: float[] floatValues = (float[]) columns[measurementIndex]; - value = new TsPrimitiveType.TsFloat(floatValues[lastIdx]); + value = new TsPrimitiveType.TsFloat(floatValues[rowIndex]); break; case DOUBLE: double[] doubleValues = (double[]) columns[measurementIndex]; - value = new TsPrimitiveType.TsDouble(doubleValues[lastIdx]); + value = new TsPrimitiveType.TsDouble(doubleValues[rowIndex]); break; case BOOLEAN: boolean[] boolValues = (boolean[]) columns[measurementIndex]; - value = new TsPrimitiveType.TsBoolean(boolValues[lastIdx]); + value = new TsPrimitiveType.TsBoolean(boolValues[rowIndex]); break; case TEXT: case BLOB: case STRING: Binary[] binaryValues = (Binary[]) columns[measurementIndex]; - value = new TsPrimitiveType.TsBinary(binaryValues[lastIdx]); + value = new TsPrimitiveType.TsBinary(binaryValues[rowIndex]); break; default: throw new UnSupportedDataTypeException( String.format(DATATYPE_UNSUPPORTED, dataTypes[measurementIndex])); } - return new TimeValuePair(times[lastIdx], value); + return new TimeValuePair(times[rowIndex], value); } public void updateLastCache(final String databaseName) { + updateLastCache(databaseName, null); + } + + public void updateLastCache(final String databaseName, final TSStatus[] results) { final String[] rawMeasurements = getRawMeasurements(); final TimeValuePair[] timeValuePairs = new TimeValuePair[rawMeasurements.length]; for (int i = 0; i < rawMeasurements.length; i++) { - timeValuePairs[i] = composeLastTimeValuePair(i); + timeValuePairs[i] = composeLastTimeValuePair(i, results, 0, rowCount); } DataNodeSchemaCache.getInstance() .updateLastCacheIfExists( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/ClusterScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/ClusterScheduler.java index 33f1bd3a19455..283e480ae38f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/ClusterScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/ClusterScheduler.java @@ -39,7 +39,6 @@ import java.util.List; import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; @@ -74,8 +73,6 @@ public ClusterScheduler( QueryStateMachine stateMachine, DistributedQueryPlan distributedQueryPlan, QueryType queryType, - ExecutorService executor, - ExecutorService writeOperationExecutor, ScheduledExecutorService scheduledExecutor, IClientManager syncInternalServiceClientManager, IClientManager @@ -88,8 +85,6 @@ public ClusterScheduler( new FragmentInstanceDispatcherImpl( queryType, queryContext, - executor, - writeOperationExecutor, syncInternalServiceClientManager, asyncInternalServiceClientManager); if (queryType == QueryType.READ) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java index 7bcdd1fb6947f..3312444f8822e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java @@ -57,6 +57,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransportException; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,13 +65,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import static com.google.common.util.concurrent.Futures.immediateFuture; import static org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet.DISPATCH_READ; +import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onThriftFrameOversizeException; public class FragmentInstanceDispatcherImpl implements IFragInstanceDispatcher { @@ -79,8 +80,6 @@ public class FragmentInstanceDispatcherImpl implements IFragInstanceDispatcher { private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig(); - private final ExecutorService executor; - private final ExecutorService writeOperationExecutor; private final QueryType type; private final MPPQueryContext queryContext; private final String localhostIpAddr; @@ -102,15 +101,11 @@ public class FragmentInstanceDispatcherImpl implements IFragInstanceDispatcher { public FragmentInstanceDispatcherImpl( QueryType type, MPPQueryContext queryContext, - ExecutorService executor, - ExecutorService writeOperationExecutor, IClientManager syncInternalServiceClientManager, IClientManager asyncInternalServiceClientManager) { this.type = type; this.queryContext = queryContext; - this.executor = executor; - this.writeOperationExecutor = writeOperationExecutor; this.syncInternalServiceClientManager = syncInternalServiceClientManager; this.asyncInternalServiceClientManager = asyncInternalServiceClientManager; this.localhostIpAddr = IoTDBDescriptor.getInstance().getConfig().getInternalAddress(); @@ -518,6 +513,15 @@ private void dispatchRemoteHelper(final FragmentInstance instance, final TEndPoi TSStatusCode.EXECUTE_STATEMENT_ERROR, String.format("unknown read type [%s]", instance.getType()))); } + } catch (TException e) { + Throwable rootCause = ExceptionUtils.getRootCause(e); + if (rootCause instanceof TTransportException + && ((TTransportException) rootCause).getType() == TTransportException.CORRUPTED_DATA) { + // Don't set DISPATCH_ERROR status to avoid retry if dispatch failed because of thrift frame + // is oversize + throw new FragmentInstanceDispatchException(onThriftFrameOversizeException(rootCause)); + } + throw e; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java index c0703113cdf45..37f2fba832d4c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImpl.java @@ -38,7 +38,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.planner.plan.SubPlan; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadSingleTsFileNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.queryengine.plan.scheduler.FragInstanceDispatchResult; @@ -156,12 +155,7 @@ public void dispatchLocally(FragmentInstance instance) throws FragmentInstanceDi PlanNode planNode = instance.getFragment().getPlanNodeTree(); if (planNode instanceof LoadTsFilePieceNode) { // split - LoadTsFilePieceNode pieceNode = - (LoadTsFilePieceNode) PlanNodeType.deserialize(planNode.serializeToByteBuffer()); - if (pieceNode == null) { - throw new FragmentInstanceDispatchException( - new TSStatus(TSStatusCode.DESERIALIZE_PIECE_OF_TSFILE_ERROR.getStatusCode())); - } + LoadTsFilePieceNode pieceNode = (LoadTsFilePieceNode) planNode; TSStatus resultStatus = StorageEngine.getInstance().writeLoadTsFileNode((DataRegionId) groupId, pieceNode, uuid); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java index 4a85b6d0ebc0a..e61367ed896a0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java @@ -551,7 +551,7 @@ private void convertFailedTsFilesToTabletsAndRetry() { final TSStatus status = loadTsFileDataTypeConverter .convertForTreeModel( - new LoadTsFileStatement(filePath) + LoadTsFileStatement.createUnchecked(filePath) .setDeleteAfterLoad(failedNode.isDeleteAfterLoad()) .setConvertOnTypeMismatch(true)) .orElse(null); @@ -709,18 +709,29 @@ private void routeChunkData() throws LoadFileException { return; } + final List> partitionSlotList = new ArrayList<>(); + final int[] chunkPartitionIndexes = new int[nonDirectionalChunkData.size()]; + final Map> partitionSlotIndexes = new HashMap<>(); + for (int i = 0, size = nonDirectionalChunkData.size(); i < size; i++) { + final ChunkData chunkData = nonDirectionalChunkData.get(i); + final IDeviceID device = new PlainDeviceID(chunkData.getDevice()); + final TTimePartitionSlot timePartitionSlot = chunkData.getTimePartitionSlot(); + final Map slotIndexes = + partitionSlotIndexes.computeIfAbsent(device, key -> new HashMap<>()); + Integer partitionSlotIndex = slotIndexes.get(timePartitionSlot); + if (partitionSlotIndex == null) { + partitionSlotIndex = partitionSlotList.size(); + slotIndexes.put(timePartitionSlot, partitionSlotIndex); + partitionSlotList.add(new Pair<>(device, timePartitionSlot)); + } + chunkPartitionIndexes[i] = partitionSlotIndex; + } + List replicaSets = scheduler.partitionFetcher.queryDataPartition( - nonDirectionalChunkData.stream() - .map( - data -> - new Pair<>( - (IDeviceID) new PlainDeviceID(data.getDevice()), - data.getTimePartitionSlot())) - .collect(Collectors.toList()), - scheduler.queryContext.getSession().getUserName()); - for (int i = 0; i < replicaSets.size(); i++) { - final TRegionReplicaSet replicaSet = replicaSets.get(i); + partitionSlotList, scheduler.queryContext.getSession().getUserName()); + for (int i = 0, size = nonDirectionalChunkData.size(); i < size; i++) { + final TRegionReplicaSet replicaSet = replicaSets.get(chunkPartitionIndexes[i]); final TConsensusGroupId regionId = replicaSet.getRegionId(); if (regionId2ReplicaSetAndNode.containsKey(regionId) && !Objects.equals(regionId2ReplicaSetAndNode.get(regionId).getLeft(), replicaSet)) { @@ -790,7 +801,7 @@ public DataPartitionBatchFetcher(IPartitionFetcher fetcher) { public List queryDataPartition( List> slotList, String userName) { - List replicaSets = new ArrayList<>(); + List replicaSets = new ArrayList<>(slotList.size()); int size = slotList.size(); for (int i = 0; i < size; i += TRANSMIT_LIMIT) { @@ -798,31 +809,34 @@ public List queryDataPartition( slotList.subList(i, Math.min(size, i + TRANSMIT_LIMIT)); DataPartition dataPartition = fetcher.getOrCreateDataPartition(toQueryParam(subSlotList), userName); - replicaSets.addAll( - subSlotList.stream() - .map( - pair -> - dataPartition.getDataRegionReplicaSetForWriting( - ((PlainDeviceID) pair.left).toStringID(), pair.right)) - .collect(Collectors.toList())); + for (final Pair pair : subSlotList) { + replicaSets.add( + dataPartition.getDataRegionReplicaSetForWriting( + ((PlainDeviceID) pair.left).toStringID(), pair.right)); + } } return replicaSets; } private List toQueryParam( List> slots) { - return slots.stream() - .collect( - Collectors.groupingBy( - Pair::getLeft, Collectors.mapping(Pair::getRight, Collectors.toSet()))) - .entrySet() - .stream() - .map( - entry -> - new DataPartitionQueryParam( - ((PlainDeviceID) entry.getKey()).toStringID(), - new ArrayList<>(entry.getValue()))) - .collect(Collectors.toList()); + final Map> device2TimePartitionSlots = new HashMap<>(); + for (final Pair slot : slots) { + device2TimePartitionSlots + .computeIfAbsent(slot.left, key -> new HashSet<>()) + .add(slot.right); + } + + final List queryParams = + new ArrayList<>(device2TimePartitionSlots.size()); + for (final Map.Entry> entry : + device2TimePartitionSlots.entrySet()) { + final DataPartitionQueryParam queryParam = + new DataPartitionQueryParam( + ((PlainDeviceID) entry.getKey()).toStringID(), new ArrayList<>(entry.getValue())); + queryParams.add(queryParam); + } + return queryParams; } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index 2e393678c5d52..b5672cbbb4fc5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaValidation; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; +import org.apache.iotdb.db.utils.BitMapUtils; import org.apache.iotdb.db.utils.CommonUtils; import org.apache.tsfile.enums.TSDataType; @@ -326,7 +327,7 @@ public List getSplitList() { statement.setMeasurementSchemas(measurementSchemas); statement.setDataTypes(dataTypes); if (this.bitMaps != null) { - statement.setBitMaps(copiedBitMaps); + statement.setBitMaps(BitMapUtils.compactBitMaps(copiedBitMaps, rowCount)); } statement.setFailedMeasurementIndex2Info(failedMeasurementIndex2Info); insertTabletStatementList.add(statement); @@ -491,6 +492,169 @@ protected long calculateBytesUsed() { + InsertNodeMemoryEstimator.sizeOfColumns(columns, measurementSchemas); } + /** + * Convert this InsertTabletStatement to Tablet. This method constructs a Tablet object from this + * statement, converting all necessary fields. All arrays are copied to rowSize length to ensure + * immutability. + * + * @return Tablet object + * @throws MetadataException if conversion fails + */ + public Tablet convertToTablet() throws MetadataException { + try { + // Get deviceId/tableName from devicePath + final String deviceIdOrTableName = + this.getDevicePath() != null ? this.getDevicePath().getFullPath() : ""; + + // Get schemas from measurementSchemas + final MeasurementSchema[] measurementSchemas = this.getMeasurementSchemas(); + final String[] measurements = this.getMeasurements(); + final TSDataType[] dataTypes = this.getDataTypes(); + // If measurements and dataTypes are not null, use measurements.length as the standard length + final int originalSchemaSize = measurements != null ? measurements.length : 0; + + // Build schemas and track valid column indices (skip null columns) + // measurements and dataTypes being null is standard - skip those columns + final List schemas = new ArrayList<>(originalSchemaSize); + final int[] validColumnIndices = new int[originalSchemaSize]; + int validColumnCount = 0; + if (dataTypes != null) { + final int dataTypeSize = Math.min(originalSchemaSize, dataTypes.length); + for (int i = 0; i < dataTypeSize; i++) { + if (measurements[i] != null && dataTypes[i] != null) { + final MeasurementSchema measurementSchema = + measurementSchemas != null && i < measurementSchemas.length + ? measurementSchemas[i] + : null; + schemas.add( + measurementSchema != null + && Objects.equals(measurementSchema.getMeasurementId(), measurements[i]) + && measurementSchema.getType() == dataTypes[i] + ? measurementSchema + : new MeasurementSchema(measurements[i], dataTypes[i])); + validColumnIndices[validColumnCount++] = i; + } + } + } + + final int schemaSize = validColumnCount; + + // Get timestamps - always copy to ensure immutability + final long[] times = this.getTimes(); + final int rowSize = this.getRowCount(); + final long[] timestamps; + if (rowSize == 0) { + timestamps = new long[0]; + } else if (times != null && times.length >= rowSize) { + timestamps = Arrays.copyOf(times, rowSize); + } else { + LOGGER.warn( + "Times array is null or too small. times.length={}, rowSize={}, deviceId={}", + times != null ? times.length : 0, + rowSize, + deviceIdOrTableName); + timestamps = new long[0]; + } + + // Get values - convert Statement columns to Tablet format, only for valid columns + // All arrays are copied to rowSize length + final Object[] statementColumns = this.getColumns(); + final Object[] tabletValues = new Object[schemaSize]; + if (statementColumns != null && statementColumns.length > 0) { + for (int i = 0; i < schemaSize; i++) { + final int originalIndex = validColumnIndices[i]; + if (originalIndex < statementColumns.length + && statementColumns[originalIndex] != null + && dataTypes[originalIndex] != null) { + tabletValues[i] = + convertColumnToTablet( + statementColumns[originalIndex], dataTypes[originalIndex], rowSize); + } else { + tabletValues[i] = null; + } + } + } + + // Get bitMaps - copy and truncate to rowSize, only for valid columns + final BitMap[] originalBitMaps = this.getBitMaps(); + BitMap[] bitMaps = null; + if (originalBitMaps != null && originalBitMaps.length > 0) { + final BitMap[] copiedBitMaps = new BitMap[schemaSize]; + for (int i = 0; i < schemaSize; i++) { + final int originalIndex = validColumnIndices[i]; + if (originalIndex < originalBitMaps.length && originalBitMaps[originalIndex] != null) { + final BitMap originalBitMap = originalBitMaps[originalIndex]; + if (!originalBitMap.isAllUnmarked()) { + copiedBitMaps[i] = + originalBitMap.getRegion(0, Math.min(rowSize, originalBitMap.getSize())); + } + } else { + copiedBitMaps[i] = null; + } + } + bitMaps = BitMapUtils.compactBitMaps(copiedBitMaps, rowSize); + } + + return new Tablet(deviceIdOrTableName, schemas, timestamps, tabletValues, bitMaps, rowSize); + } catch (final Exception e) { + throw new MetadataException("Failed to convert InsertTabletStatement to Tablet: ", e); + } + } + + /** + * Convert a single column value from Statement format to Tablet format. Statement uses primitive + * arrays (e.g., int[], long[], float[]), while Tablet may need different format. All arrays are + * copied to rowSize length to ensure immutability - even if the original array is modified, the + * converted array remains unchanged. + * + * @param columnValue column value from Statement (primitive array) + * @param dataType data type of the column + * @param rowSize number of rows to copy + * @return column value in Tablet format (copied to rowSize) + */ + private Object convertColumnToTablet( + final Object columnValue, final TSDataType dataType, final int rowSize) { + + if (columnValue == null) { + return null; + } + + if (TSDataType.DATE.equals(dataType)) { + final int[] values = (int[]) columnValue; + final LocalDate[] localDateValue = new LocalDate[rowSize]; + final int size = Math.min(values.length, rowSize); + for (int i = 0; i < size; i++) { + localDateValue[i] = DateUtils.parseIntToLocalDate(values[i]); + } + return localDateValue; + } + + // For primitive arrays, copy to rowSize + if (columnValue instanceof boolean[]) { + final boolean[] original = (boolean[]) columnValue; + return Arrays.copyOf(original, rowSize); + } else if (columnValue instanceof int[]) { + final int[] original = (int[]) columnValue; + return Arrays.copyOf(original, rowSize); + } else if (columnValue instanceof long[]) { + final long[] original = (long[]) columnValue; + return Arrays.copyOf(original, rowSize); + } else if (columnValue instanceof float[]) { + final float[] original = (float[]) columnValue; + return Arrays.copyOf(original, rowSize); + } else if (columnValue instanceof double[]) { + final double[] original = (double[]) columnValue; + return Arrays.copyOf(original, rowSize); + } else if (columnValue instanceof Binary[]) { + // For Binary arrays, create a new array and copy references to rowSize + final Binary[] original = (Binary[]) columnValue; + return Arrays.copyOf(original, rowSize); + } + + // For other types, return as-is (should not happen for standard types) + return columnValue; + } + @Override public String toString() { final int size = CommonDescriptor.getInstance().getConfig().getPathLogMaxSize(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java index 3b10d3733e1b3..404b957c7865c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java @@ -20,7 +20,10 @@ package org.apache.iotdb.db.queryengine.plan.statement.crud; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.plan.statement.Statement; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; @@ -34,15 +37,20 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; + public class LoadTsFileStatement extends Statement { private final File file; private int databaseLevel; + private String database; private boolean verifySchema = true; private boolean deleteAfterLoad = false; private boolean convertOnTypeMismatch = true; @@ -56,6 +64,15 @@ public class LoadTsFileStatement extends Statement { private List writePointCountList; public LoadTsFileStatement(String filePath) throws FileNotFoundException { + this(filePath, true); + } + + public static LoadTsFileStatement createUnchecked(String filePath) throws FileNotFoundException { + return new LoadTsFileStatement(filePath, false); + } + + private LoadTsFileStatement(String filePath, boolean validateSourcePath) + throws FileNotFoundException { this.file = new File(filePath).getAbsoluteFile(); this.databaseLevel = IoTDBDescriptor.getInstance().getConfig().getDefaultDatabaseLevel(); this.verifySchema = true; @@ -65,13 +82,22 @@ public LoadTsFileStatement(String filePath) throws FileNotFoundException { IoTDBDescriptor.getInstance().getConfig().getLoadTabletConversionThresholdBytes(); this.autoCreateDatabase = IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled(); - this.tsFiles = processTsFile(file); + this.tsFiles = processTsFile(file, validateSourcePath); this.resources = new ArrayList<>(); this.writePointCountList = new ArrayList<>(); this.statementType = StatementType.MULTI_BATCH_INSERT; } public static List processTsFile(final File file) throws FileNotFoundException { + return processTsFile(file, true); + } + + public static List processTsFile(final File file, final boolean validateSourcePath) + throws FileNotFoundException { + if (validateSourcePath) { + validateLoadSourcePath(file); + } + final List tsFiles = new ArrayList<>(); if (file.isFile()) { tsFiles.add(file); @@ -82,7 +108,7 @@ public static List processTsFile(final File file) throws FileNotFoundExcep "Can not find %s on this machine, notice that load can only handle files on this machine.", file.getPath())); } - tsFiles.addAll(findAllTsFile(file)); + tsFiles.addAll(findAllTsFile(file, validateSourcePath)); } sortTsFiles(tsFiles); return tsFiles; @@ -101,7 +127,8 @@ protected LoadTsFileStatement() { this.statementType = StatementType.MULTI_BATCH_INSERT; } - private static List findAllTsFile(File file) { + private static List findAllTsFile(File file, boolean validateSourcePath) + throws FileNotFoundException { final File[] files = file.listFiles(); if (files == null) { return Collections.emptyList(); @@ -109,15 +136,55 @@ private static List findAllTsFile(File file) { final List tsFiles = new ArrayList<>(); for (File nowFile : files) { + if (validateSourcePath) { + validateLoadSourcePath(nowFile); + } if (nowFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) { tsFiles.add(nowFile); } else if (nowFile.isDirectory()) { - tsFiles.addAll(findAllTsFile(nowFile)); + tsFiles.addAll(findAllTsFile(nowFile, validateSourcePath)); } } return tsFiles; } + public static void validateLoadSourcePath(final String filePath) throws FileNotFoundException { + validateLoadSourcePath(new File(filePath)); + } + + private static void validateLoadSourcePath(final File file) throws FileNotFoundException { + final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + if (!config.isLoadTsFileSourcePathCheckEnabled()) { + return; + } + + final Path sourcePath = canonicalPath(file); + final String[] allowedDirs = config.getLoadTsFileAllowedDirs(); + final Path[] allowedDirCanonicalPaths = config.getLoadTsFileAllowedDirCanonicalPaths(); + + for (final Path allowedDirCanonicalPath : allowedDirCanonicalPaths) { + if (sourcePath.startsWith(allowedDirCanonicalPath)) { + return; + } + } + + throw new FileNotFoundException( + String.format( + "Load TsFile source path %s is outside allowed directories %s.", + sourcePath, Arrays.toString(allowedDirs))); + } + + private static Path canonicalPath(final File file) throws FileNotFoundException { + try { + return file.getCanonicalFile().toPath(); + } catch (final IOException e) { + throw new FileNotFoundException( + String.format( + "Failed to resolve canonical path for Load TsFile source %s: %s", + file.getPath(), e.getMessage())); + } + } + private static void sortTsFiles(List files) { files.sort( (o1, o2) -> { @@ -139,6 +206,14 @@ public int getDatabaseLevel() { return databaseLevel; } + public void setDatabase(String database) { + this.database = database; + } + + public String getDatabase() { + return database; + } + public void setVerifySchema(boolean verifySchema) { this.verifySchema = verifySchema; } @@ -219,6 +294,7 @@ public boolean isAsyncLoad() { private void initAttributes(final Map loadAttributes) { this.databaseLevel = LoadTsFileConfigurator.parseOrGetDefaultDatabaseLevel(loadAttributes); + this.database = LoadTsFileConfigurator.parseDatabaseName(loadAttributes); this.deleteAfterLoad = LoadTsFileConfigurator.parseOrGetDefaultOnSuccess(loadAttributes); this.convertOnTypeMismatch = LoadTsFileConfigurator.parseOrGetDefaultConvertOnTypeMismatch(loadAttributes); @@ -231,6 +307,28 @@ private void initAttributes(final Map loadAttributes) { } } + public void updateDatabaseLevelByTreeDatabase() { + final Integer databaseLevel = getDatabaseLevelByTreeDatabase(database); + if (databaseLevel != null) { + this.databaseLevel = databaseLevel; + } + } + + public static Integer getDatabaseLevelByTreeDatabase(final String database) { + if (database == null) { + return null; + } + try { + final String[] nodes = PathUtils.splitPathToDetachedNodes(database); + if (nodes.length > 1 && PATH_ROOT.equals(nodes[0])) { + return nodes.length - 1; + } + } catch (final IllegalPathException ignored) { + // Keep the configured database level when database is not a legal tree path. + } + return null; + } + public boolean reconstructStatementIfMiniFileConverted(final List isMiniTsFile) { int lastNonMiniTsFileIndex = -1; @@ -290,6 +388,7 @@ public List getSubStatements() { final LoadTsFileStatement statement = new LoadTsFileStatement(); statement.databaseLevel = this.databaseLevel; + statement.database = this.database; statement.verifySchema = this.verifySchema; statement.deleteAfterLoad = this.deleteAfterLoad; statement.convertOnTypeMismatch = this.convertOnTypeMismatch; @@ -333,6 +432,8 @@ public String toString() { + deleteAfterLoad + ", database-level=" + databaseLevel + + ", database=" + + database + ", verify-schema=" + verifySchema + ", convert-on-type-mismatch=" diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountStatement.java index 529a8660dfbfe..6ea44f6c11397 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountStatement.java @@ -19,9 +19,17 @@ package org.apache.iotdb.db.queryengine.plan.statement.metadata; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.rpc.TSStatusCode; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -34,6 +42,7 @@ */ public class CountStatement extends ShowStatement { protected PartialPath pathPattern; + private boolean canSeeSystemDB = true; public CountStatement(PartialPath pathPattern) { this.pathPattern = pathPattern; @@ -52,4 +61,76 @@ public void setPathPattern(PartialPath pathPattern) { public List getPaths() { return Collections.singletonList(pathPattern); } + + @Override + public TSStatus checkPermissionBeforeProcess(String userName) { + if (AuthorityChecker.SUPER_USER.equals(userName)) { + setCanSeeSystemDB(true); + return AuthorityChecker.SUCCEED; + } + TSStatus explicitInternalDatabaseStatus = checkExplicitInternalDatabase(userName); + if (explicitInternalDatabaseStatus != null) { + return explicitInternalDatabaseStatus; + } + setCanSeeSystemDB(userName); + TSStatus status = super.checkPermissionBeforeProcess(userName); + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + appendInternalDatabaseAuthorityScope(); + } + return status; + } + + public boolean isCanSeeSystemDB() { + return canSeeSystemDB; + } + + public void setCanSeeSystemDB(boolean canSeeSystemDB) { + this.canSeeSystemDB = canSeeSystemDB; + } + + protected void setCanSeeSystemDB(String userName) { + setCanSeeSystemDB( + AuthorityChecker.checkSystemPermission(userName, PrivilegeType.MAINTAIN.ordinal())); + } + + protected TSStatus checkExplicitInternalDatabase(String userName) { + if (!isExplicitSystemDatabasePath()) { + return null; + } + if (!AuthorityChecker.checkSystemPermission(userName, PrivilegeType.MAINTAIN.ordinal())) { + return AuthorityChecker.getTSStatus(false, PrivilegeType.MAINTAIN); + } + setCanSeeSystemDB(true); + authorityScope = createAuthorityScope(pathPattern); + return AuthorityChecker.SUCCEED; + } + + protected void appendInternalDatabaseAuthorityScope() { + if (!canSeeSystemDB || SchemaConstant.ALL_MATCH_SCOPE.equals(authorityScope)) { + return; + } + authorityScope.appendPathPattern(createInternalDatabasePathPattern()); + authorityScope.constructTree(); + } + + private boolean isExplicitSystemDatabasePath() { + String[] nodes = pathPattern.getNodes(); + return nodes.length >= 2 + && SchemaConstant.ROOT.equals(nodes[0]) + && SchemaConstant.SYSTEM_DATABASE.equals(SchemaConstant.ROOT + "." + nodes[1]); + } + + private PartialPath createInternalDatabasePathPattern() { + String[] databaseNodes = SchemaConstant.SYSTEM_DATABASE.split("\\."); + String[] pathPatternNodes = Arrays.copyOf(databaseNodes, databaseNodes.length + 1); + pathPatternNodes[databaseNodes.length] = IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; + return new PartialPath(pathPatternNodes); + } + + private PathPatternTree createAuthorityScope(PartialPath pathPattern) { + PathPatternTree authorityScope = new PathPatternTree(); + authorityScope.appendPathPattern(pathPattern); + authorityScope.constructTree(); + return authorityScope; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountTimeSeriesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountTimeSeriesStatement.java index c0910185421bd..870c0cf8cb8c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountTimeSeriesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountTimeSeriesStatement.java @@ -62,22 +62,27 @@ public WhereCondition getTimeCondition() { @Override public TSStatus checkPermissionBeforeProcess(String userName) { - if (hasTimeCondition()) { - try { - if (!AuthorityChecker.SUPER_USER.equals(userName)) { - this.authorityScope = - PathPatternTreeUtils.intersectWithFullPathPrefixTree( - AuthorityChecker.getAuthorizedPathTree( - userName, PrivilegeType.READ_SCHEMA.ordinal()), - AuthorityChecker.getAuthorizedPathTree( - userName, PrivilegeType.READ_DATA.ordinal())); - } - } catch (AuthException e) { - return new TSStatus(e.getCode().getStatusCode()); + if (!hasTimeCondition()) { + return super.checkPermissionBeforeProcess(userName); + } + try { + if (AuthorityChecker.SUPER_USER.equals(userName)) { + setCanSeeSystemDB(true); + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } + TSStatus explicitInternalDatabaseStatus = checkExplicitInternalDatabase(userName); + if (explicitInternalDatabaseStatus != null) { + return explicitInternalDatabaseStatus; + } + setCanSeeSystemDB(userName); + this.authorityScope = + PathPatternTreeUtils.intersectWithFullPathPrefixTree( + AuthorityChecker.getAuthorizedPathTree(userName, PrivilegeType.READ_SCHEMA.ordinal()), + AuthorityChecker.getAuthorizedPathTree(userName, PrivilegeType.READ_DATA.ordinal())); + appendInternalDatabaseAuthorityScope(); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); - } else { - return super.checkPermissionBeforeProcess(userName); + } catch (AuthException e) { + return new TSStatus(e.getCode().getStatusCode()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/CaseWhenThenColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/CaseWhenThenColumnTransformer.java index 68910f2aa4b3a..94156d1c71b21 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/CaseWhenThenColumnTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/CaseWhenThenColumnTransformer.java @@ -128,4 +128,14 @@ protected void evaluate() { protected void checkType() { // do nothing } + + @Override + public void close() { + super.close(); + for (Pair whenThenTransformer : whenThenTransformers) { + whenThenTransformer.left.close(); + whenThenTransformer.right.close(); + } + elseTransformer.close(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/BinaryColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/BinaryColumnTransformer.java index 6e074abe08f4a..996c6763afd52 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/BinaryColumnTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/BinaryColumnTransformer.java @@ -63,4 +63,11 @@ public ColumnTransformer getLeftTransformer() { public ColumnTransformer getRightTransformer() { return rightTransformer; } + + @Override + public void close() { + super.close(); + this.leftTransformer.close(); + this.rightTransformer.close(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/MappableUDFColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/MappableUDFColumnTransformer.java index b3562a2920884..74f15ab7055c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/MappableUDFColumnTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/MappableUDFColumnTransformer.java @@ -70,6 +70,10 @@ public ColumnTransformer[] getInputColumnTransformers() { @Override public void close() { + super.close(); + for (ColumnTransformer inputColumnTransformer : inputColumnTransformers) { + inputColumnTransformer.close(); + } // finalize executor executor.beforeDestroy(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/TernaryColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/TernaryColumnTransformer.java index 6d747d3dae835..1638d273bffb0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/TernaryColumnTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/TernaryColumnTransformer.java @@ -54,4 +54,12 @@ public ColumnTransformer getSecondColumnTransformer() { public ColumnTransformer getThirdColumnTransformer() { return thirdColumnTransformer; } + + @Override + public void close() { + super.close(); + firstColumnTransformer.close(); + secondColumnTransformer.close(); + thirdColumnTransformer.close(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/UnaryColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/UnaryColumnTransformer.java index 0f1f48cfee852..e29ea0bf56f73 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/UnaryColumnTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/UnaryColumnTransformer.java @@ -53,4 +53,10 @@ protected void checkType() { } protected abstract void doTransform(Column column, ColumnBuilder columnBuilder); + + @Override + public void close() { + super.close(); + childColumnTransformer.close(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java index 9b766b4c5140e..1e185e40cda15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java @@ -55,6 +55,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -365,13 +366,15 @@ public int getSchemaRegionNumber() { return schemaRegionMap.size(); } - public Map countDeviceNumBySchemaRegion(List schemaIds) { - Map deviceNum = new HashMap<>(); + public Map countDeviceNumBySchemaRegion(final List schemaIds) { + final Map deviceNum = new HashMap<>(); + final Collection targetSchemaIds = + schemaIds.size() > 1 ? new HashSet<>(schemaIds) : schemaIds; schemaRegionMap.entrySet().stream() .filter( entry -> - schemaIds.contains(entry.getKey().getId()) + targetSchemaIds.contains(entry.getKey().getId()) && SchemaRegionConsensusImpl.getInstance().isLeader(entry.getKey())) .forEach( entry -> @@ -381,12 +384,14 @@ public Map countDeviceNumBySchemaRegion(List schemaIds) return deviceNum; } - public Map countTimeSeriesNumBySchemaRegion(List schemaIds) { - Map timeSeriesNum = new HashMap<>(); + public Map countTimeSeriesNumBySchemaRegion(final List schemaIds) { + final Map timeSeriesNum = new HashMap<>(); + final Collection targetSchemaIds = + schemaIds.size() > 1 ? new HashSet<>(schemaIds) : schemaIds; schemaRegionMap.entrySet().stream() .filter( entry -> - schemaIds.contains(entry.getKey().getId()) + targetSchemaIds.contains(entry.getKey().getId()) && SchemaRegionConsensusImpl.getInstance().isLeader(entry.getKey())) .forEach( entry -> diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java index be960b3807b63..a3a833f06abf6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java @@ -168,7 +168,7 @@ public void clear() { memoryUsage.getAndSet(0); measurementNumber.getAndSet(0); devicesNumber.getAndSet(0); - viewNumber.getAndAdd(0); + viewNumber.getAndSet(0); templateUsage.forEach( (templateId, cnt) -> schemaEngineStatistics.deactivateTemplate(templateId, cnt)); templateUsage.clear(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java index 09a69f2b4f699..41ec8e649c599 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java @@ -39,6 +39,7 @@ public enum SchemaRegionPlanType { ACTIVATE_TEMPLATE_IN_CLUSTER((byte) 63), PRE_DELETE_TIMESERIES_IN_CLUSTER((byte) 64), ROLLBACK_PRE_DELETE_TIMESERIES((byte) 65), + // endregion PRE_DEACTIVATE_TEMPLATE((byte) 0), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 6fc556a1089da..0f6490eceb467 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -393,7 +393,9 @@ public synchronized void clear() { logWriter.close(); logWriter = null; } - tagManager.clear(); + if (tagManager != null) { + tagManager.clear(); + } isRecovering = true; initialized = false; @@ -857,8 +859,8 @@ public void createLogicalView(ICreateLogicalViewPlan plan) throws MetadataExcept regionStatistics.getGlobalMemoryUsage(), regionStatistics.getGlobalSeriesNumber()); } + final List pathList = plan.getViewPathList(); try { - List pathList = plan.getViewPathList(); Map viewPathToSourceMap = plan.getViewPathToSourceExpressionMap(); for (PartialPath path : pathList) { @@ -873,7 +875,7 @@ public void createLogicalView(ICreateLogicalViewPlan plan) throws MetadataExcept throw new RuntimeException(e); } // update statistics - regionStatistics.addView(1L); + regionStatistics.addView(pathList.size()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index b6f1221f7013f..a82250e229986 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -174,6 +174,59 @@ public synchronized boolean createSnapshot(File snapshotDir) { return store.createSnapshot(snapshotDir); } + private void applySubtreeMeasurementDelta(IMemMNode startNode, final long delta) { + if (delta == 0 || startNode == null) { + return; + } + IMemMNode current = startNode; + while (current != null) { + current.setSubtreeMeasurementCount(current.getSubtreeMeasurementCount() + delta); + current = current.getParent(); + } + } + + private IDeviceMNode setToEntityAndUpdateFlags(final IMemMNode node) { + final boolean wasDevice = node.isDevice(); + final IDeviceMNode deviceMNode = store.setToEntity(node); + if (!wasDevice) { + markAncestorsHavingDeviceDescendant(node); + } + return deviceMNode; + } + + private void markAncestorsHavingDeviceDescendant(final IMemMNode deviceNode) { + IMemMNode current = deviceNode.getParent(); + while (current != null && !current.hasDeviceDescendant()) { + current.setHasDeviceDescendant(true); + current = current.getParent(); + } + } + + private boolean hasDeviceDescendantInChildren(final IMemMNode node) { + try (final IMNodeIterator iterator = store.getChildrenIterator(node)) { + while (iterator.hasNext()) { + final IMemMNode child = iterator.next(); + if (child.isDevice() || child.hasDeviceDescendant()) { + return true; + } + } + return false; + } + } + + private void refreshAncestorsHavingDeviceDescendant(IMemMNode startNode) { + IMemMNode current = startNode; + while (current != null) { + current.setHasDeviceDescendant(hasDeviceDescendantInChildren(current)); + current = current.getParent(); + } + } + + private long getTemplateMeasurementCount(final int templateId) { + final Template template = ClusterTemplateManager.getInstance().getTemplate(templateId); + return template == null ? 0L : template.getMeasurementNumber(); + } + public static MTreeBelowSGMemoryImpl loadFromSnapshot( File snapshotDir, String storageGroupFullPath, @@ -184,13 +237,16 @@ public static MTreeBelowSGMemoryImpl loadFromSnapshot( Function, Map> tagGetter, Function, Map> attributeGetter) throws IOException, IllegalPathException { - return new MTreeBelowSGMemoryImpl( - new PartialPath(storageGroupFullPath), - MemMTreeStore.loadFromSnapshot( - snapshotDir, measurementProcess, deviceProcess, regionStatistics, metric), - tagGetter, - attributeGetter, - regionStatistics); + final MTreeBelowSGMemoryImpl mtree = + new MTreeBelowSGMemoryImpl( + new PartialPath(storageGroupFullPath), + MemMTreeStore.loadFromSnapshot( + snapshotDir, measurementProcess, deviceProcess, regionStatistics, metric), + tagGetter, + attributeGetter, + regionStatistics); + mtree.rebuildSubtreeMeasurementCount(); + return mtree; } // endregion @@ -268,7 +324,7 @@ public IMeasurementMNode createTimeSeries( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); } // create a non-aligned time series @@ -290,6 +346,7 @@ public IMeasurementMNode createTimeSeries( entityMNode.addAlias(alias, measurementMNode); } + applySubtreeMeasurementDelta(measurementMNode.getAsMNode(), 1L); return measurementMNode; } } @@ -360,7 +417,7 @@ public List> createAlignedTimeSeries( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); entityMNode.setAligned(true); } @@ -386,6 +443,7 @@ public List> createAlignedTimeSeries( if (aliasList != null && aliasList.get(i) != null) { entityMNode.addAlias(aliasList.get(i), measurementMNode); } + applySubtreeMeasurementDelta(measurementMNode.getAsMNode(), 1L); measurementMNodeList.add(measurementMNode); } return measurementMNodeList; @@ -539,6 +597,7 @@ public IMeasurementMNode deleteTimeseries(PartialPath path) throws Me if (deletedNode.getAlias() != null) { parent.getAsDeviceMNode().deleteAliasChild(deletedNode.getAlias()); } + applySubtreeMeasurementDelta(parent, -1L); } deleteEmptyInternalMNode(parent.getAsDeviceMNode()); return deletedNode; @@ -551,14 +610,15 @@ public void deleteEmptyInternalMNode(IDeviceMNode entityMNode) { boolean hasMeasurement = false; boolean hasNonViewMeasurement = false; IMemMNode child; - IMNodeIterator iterator = store.getChildrenIterator(curNode); - while (iterator.hasNext()) { - child = iterator.next(); - if (child.isMeasurement()) { - hasMeasurement = true; - if (!child.getAsMeasurementMNode().isLogicalView()) { - hasNonViewMeasurement = true; - break; + try (final IMNodeIterator iterator = store.getChildrenIterator(curNode)) { + while (iterator.hasNext()) { + child = iterator.next(); + if (child.isMeasurement()) { + hasMeasurement = true; + if (!child.getAsMeasurementMNode().isLogicalView()) { + hasNonViewMeasurement = true; + break; + } } } } @@ -567,6 +627,7 @@ public void deleteEmptyInternalMNode(IDeviceMNode entityMNode) { synchronized (this) { curNode = store.setToInternal(entityMNode); } + refreshAncestorsHavingDeviceDescendant(curNode.getParent()); } else if (!hasNonViewMeasurement) { // has some measurement but they are all logical view entityMNode.setAligned(null); @@ -889,7 +950,7 @@ public void activateTemplate(PartialPath activatePath, Template template) if (cur.isDevice()) { entityMNode = cur.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); } } @@ -907,6 +968,7 @@ public void activateTemplate(PartialPath activatePath, Template template) entityMNode.setUseTemplate(true); entityMNode.setSchemaTemplateId(template.getId()); regionStatistics.activateTemplate(template.getId()); + applySubtreeMeasurementDelta(entityMNode.getAsMNode(), (long) template.getMeasurementNumber()); } public Map> constructSchemaBlackListWithTemplate( @@ -968,6 +1030,8 @@ protected void updateEntity(IDeviceMNode node) { resultTemplateSetInfo.put( node.getPartialPath(), Collections.singletonList(node.getSchemaTemplateId())); regionStatistics.deactivateTemplate(node.getSchemaTemplateId()); + applySubtreeMeasurementDelta( + node.getAsMNode(), -getTemplateMeasurementCount(node.getSchemaTemplateId())); node.deactivateTemplate(); deleteEmptyInternalMNode(node); } @@ -991,7 +1055,7 @@ public void activateTemplateWithoutCheck( if (cur.isDevice()) { entityMNode = cur.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); } if (!entityMNode.isAligned()) { @@ -1000,6 +1064,7 @@ public void activateTemplateWithoutCheck( entityMNode.setUseTemplate(true); entityMNode.setSchemaTemplateId(templateId); regionStatistics.activateTemplate(templateId); + applySubtreeMeasurementDelta(entityMNode.getAsMNode(), getTemplateMeasurementCount(templateId)); } public long countPathsUsingTemplate(PartialPath pathPattern, int templateId) @@ -1011,6 +1076,30 @@ public long countPathsUsingTemplate(PartialPath pathPattern, int templateId) } } + public void rebuildSubtreeMeasurementCount() { + rebuildSubtreeMeasurementCountFromNode(rootNode); + } + + private long rebuildSubtreeMeasurementCountFromNode(final IMemMNode node) { + long count = node.isMeasurement() ? 1L : 0L; + boolean hasDeviceDescendant = false; + try (final IMNodeIterator iterator = store.getChildrenIterator(node)) { + while (iterator.hasNext()) { + final IMemMNode child = iterator.next(); + count += rebuildSubtreeMeasurementCountFromNode(child); + if (child.isDevice() || child.hasDeviceDescendant()) { + hasDeviceDescendant = true; + } + } + } + if (node.isDevice() && node.getAsDeviceMNode().isUseTemplate()) { + count += getTemplateMeasurementCount(node.getAsDeviceMNode().getSchemaTemplateId()); + } + node.setSubtreeMeasurementCount(count); + node.setHasDeviceDescendant(hasDeviceDescendant); + return count; + } + // endregion // region Interfaces for schema reader @@ -1278,7 +1367,7 @@ public IMeasurementMNode createLogicalView( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); // this parent has no measurement before. The leafName is his first child who is a logical // view. entityMNode.setAligned(null); @@ -1287,6 +1376,7 @@ public IMeasurementMNode createLogicalView( measurementMNode.setParent(entityMNode.getAsMNode()); store.addChild(entityMNode.getAsMNode(), leafName, measurementMNode.getAsMNode()); + applySubtreeMeasurementDelta(measurementMNode.getAsMNode(), 1L); return measurementMNode; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java index d3d055928b14a..257d7caeef8ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java @@ -19,5 +19,33 @@ package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode; import org.apache.iotdb.commons.schema.node.IMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.basic.BasicMNode; -public interface IMemMNode extends IMNode {} +public interface IMemMNode extends IMNode { + + BasicMNode getBasicMNode(); + + /** + * The count of measurement nodes contained in the subtree rooted at this node. The counter is + * maintained in memory only. + */ + default long getSubtreeMeasurementCount() { + return getBasicMNode().getSubtreeMeasurementCount(); + } + + default void setSubtreeMeasurementCount(final long subtreeMeasurementCount) { + getBasicMNode().setSubtreeMeasurementCount(subtreeMeasurementCount); + } + + /** + * Whether there is any device node in the subtree rooted at this node, excluding the node itself. + * This flag is maintained in memory only. + */ + default boolean hasDeviceDescendant() { + return getBasicMNode().hasDeviceDescendant(); + } + + default void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + getBasicMNode().setHasDeviceDescendant(hasDeviceDescendant); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java index ccacef6c7ebe8..e70885c7d03e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java @@ -46,6 +46,12 @@ public class BasicMNode implements IMemMNode { private IMemMNode parent; private final BasicMNodeInfo basicMNodeInfo; + /** Cached count of measurements in this node's subtree, rebuilt on restart. */ + private long subtreeMeasurementCount = 0L; + + /** Cached flag showing whether there is any device in the subtree below this node. */ + private boolean hasDeviceDescendant = false; + /** from root to this node, only be set when used once for InternalMNode */ private String fullPath; @@ -55,6 +61,11 @@ public BasicMNode(IMemMNode parent, String name) { this.basicMNodeInfo = new BasicMNodeInfo(name); } + @Override + public BasicMNode getBasicMNode() { + return this; + } + @Override public String getName() { return basicMNodeInfo.getName(); @@ -98,6 +109,25 @@ public void setFullPath(String fullPath) { this.fullPath = fullPath; } + public long getSubtreeMeasurementCount() { + return subtreeMeasurementCount; + } + + @Override + public void setSubtreeMeasurementCount(final long subtreeMeasurementCount) { + this.subtreeMeasurementCount = subtreeMeasurementCount; + } + + @Override + public boolean hasDeviceDescendant() { + return hasDeviceDescendant; + } + + @Override + public void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + this.hasDeviceDescendant = hasDeviceDescendant; + } + @Override public PartialPath getPartialPath() { List detachedPath = new ArrayList<>(); @@ -224,6 +254,8 @@ public R accept(MNodeVisitor visitor, C context) { *

  • basicMNodeInfo reference, 8B *
  • parent reference, 8B *
  • fullPath reference, 8B + *
  • subtreeMeasurementCount, 8B + *
  • hasDeviceDescendant, 1B * *
  • MapEntry in parent *
      @@ -235,7 +267,7 @@ public R accept(MNodeVisitor visitor, C context) { */ @Override public int estimateSize() { - return 8 + 8 + 8 + 8 + 8 + 8 + 28 + basicMNodeInfo.estimateSize(); + return 8 + 8 + 8 + 8 + 8 + 1 + 8 + 8 + 28 + basicMNodeInfo.estimateSize(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/AbstractTraverserIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/AbstractTraverserIterator.java index f7f6d7d1ae65f..e9aa30333c7db 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/AbstractTraverserIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/AbstractTraverserIterator.java @@ -107,12 +107,15 @@ public boolean hasNext() { if (skipPreDeletedSchema && nextMatchedNode.isMeasurement() && nextMatchedNode.getAsMeasurementMNode().isPreDeleted()) { + releaseSkippedNode(nextMatchedNode); nextMatchedNode = null; } } return true; } + protected abstract void releaseSkippedNode(final N node); + @Override public N next() { if (!hasNext()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/MemoryTraverserIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/MemoryTraverserIterator.java index 15eb95df4ab90..684b398c47249 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/MemoryTraverserIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/iterator/MemoryTraverserIterator.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.iterator; import org.apache.iotdb.commons.exception.MetadataException; @@ -30,11 +31,16 @@ // only use for IConfigMNode and IMemMNode public class MemoryTraverserIterator> extends AbstractTraverserIterator { public MemoryTraverserIterator( - IMTreeStore store, - IDeviceMNode parent, - Map templateMap, - IMNodeFactory nodeFactory) + final IMTreeStore store, + final IDeviceMNode parent, + final Map templateMap, + final IMNodeFactory nodeFactory) throws MetadataException { super(store, parent, templateMap, nodeFactory); } + + @Override + protected void releaseSkippedNode(final N node) { + // Do nothing + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java index f79a8c2e05743..829d242187e61 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java @@ -132,6 +132,68 @@ public class MTreeBelowSGCachedImpl { private final int levelOfDB; private final CachedSchemaRegionStatistics regionStatistics; + private IDeviceMNode setToEntityAndUpdateFlags(final ICachedMNode node) + throws MetadataException { + final boolean wasDevice = node.isDevice(); + if (!node.isDeviceDescendantComputed()) { + node.setHasDeviceDescendant(hasDeviceDescendantInChildren(node)); + node.setDeviceDescendantComputed(true); + } + final IDeviceMNode deviceMNode = store.setToEntity(node); + if (!wasDevice) { + markAncestorsHavingDeviceDescendant(node); + } + return deviceMNode; + } + + private void markAncestorsHavingDeviceDescendant(final ICachedMNode deviceNode) { + ICachedMNode current = deviceNode.getParent(); + while (current != null && !current.hasDeviceDescendant()) { + current.setHasDeviceDescendant(true); + current.setDeviceDescendantComputed(true); + current = current.getParent(); + } + } + + private boolean hasDeviceDescendantInChildren(final ICachedMNode node) throws MetadataException { + try (final IMNodeIterator iterator = store.getChildrenIterator(node)) { + while (iterator.hasNext()) { + final ICachedMNode child = iterator.next(); + try { + if (child.isDevice() || hasDeviceDescendant(child)) { + return true; + } + } finally { + unPinMNode(child); + } + } + return false; + } + } + + private boolean hasDeviceDescendant(final ICachedMNode node) throws MetadataException { + if (node.isMeasurement()) { + node.setHasDeviceDescendant(false); + node.setDeviceDescendantComputed(true); + return false; + } + if (!node.isDeviceDescendantComputed()) { + node.setHasDeviceDescendant(hasDeviceDescendantInChildren(node)); + node.setDeviceDescendantComputed(true); + } + return node.hasDeviceDescendant(); + } + + private void refreshAncestorsHavingDeviceDescendant(ICachedMNode startNode) + throws MetadataException { + ICachedMNode current = startNode; + while (current != null) { + current.setHasDeviceDescendant(hasDeviceDescendantInChildren(current)); + current.setDeviceDescendantComputed(true); + current = current.getParent(); + } + } + // region MTree initialization, clear and serialization public MTreeBelowSGCachedImpl( PartialPath storageGroupPath, @@ -348,7 +410,7 @@ public IMeasurementMNode createTimeSeriesWithPinnedReturn( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); device = entityMNode.getAsMNode(); } @@ -453,7 +515,7 @@ public List> createAlignedTimeSeries( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); entityMNode.setAligned(true); device = entityMNode.getAsMNode(); } @@ -674,8 +736,7 @@ private void deleteAndUnpinEmptyInternalMNode(IDeviceMNode entityM boolean hasMeasurement = false; boolean hasNonViewMeasurement = false; ICachedMNode child; - IMNodeIterator iterator = store.getChildrenIterator(curNode); - try { + try (final IMNodeIterator iterator = store.getChildrenIterator(curNode)) { while (iterator.hasNext()) { child = iterator.next(); unPinMNode(child); @@ -687,12 +748,11 @@ private void deleteAndUnpinEmptyInternalMNode(IDeviceMNode entityM } } } - } finally { - iterator.close(); } if (!hasMeasurement) { curNode = store.setToInternal(entityMNode); + refreshAncestorsHavingDeviceDescendant(curNode.getParent()); } else if (!hasNonViewMeasurement) { // has some measurement but they are all logical view store.updateMNode(entityMNode.getAsMNode(), o -> o.getAsDeviceMNode().setAligned(null)); @@ -719,14 +779,11 @@ private void deleteAndUnpinEmptyInternalMNode(IDeviceMNode entityM } private boolean isEmptyInternalMNode(ICachedMNode node) throws MetadataException { - IMNodeIterator iterator = store.getChildrenIterator(node); - try { + try (final IMNodeIterator iterator = store.getChildrenIterator(node)) { return !IoTDBConstant.PATH_ROOT.equals(node.getName()) && !node.isMeasurement() && !(node.isDevice() && node.getAsDeviceMNode().isUseTemplate()) && !iterator.hasNext(); - } finally { - iterator.close(); } } @@ -1064,7 +1121,7 @@ public void createLogicalView(PartialPath path, ViewExpression viewExpression) if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); // this parent has no measurement before. The leafName is his first child who is a // logical // view. @@ -1186,7 +1243,7 @@ public void activateTemplate(PartialPath activatePath, Template template) if (cur.isDevice()) { entityMNode = cur.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); } if (entityMNode.isUseTemplate()) { @@ -1232,7 +1289,7 @@ public void activateTemplateWithoutCheck( if (cur.isDevice()) { entityMNode = cur.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); } store.updateMNode( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/memory/ReleaseFlushMonitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/memory/ReleaseFlushMonitor.java index 4c3ec8d2c9de5..0aaa72899a46b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/memory/ReleaseFlushMonitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/memory/ReleaseFlushMonitor.java @@ -230,6 +230,7 @@ public List getRegionsToFlush(long windowsEndTime) { public void forceFlushAndRelease() { boolean needFlush; while (true) { + waitUntilWorkerTasksDone(); needFlush = false; for (CachedMTreeStore store : regionToStoreMap.values()) { if (store.getMemoryManager().getBufferNodeNum() > 0) { @@ -239,13 +240,29 @@ public void forceFlushAndRelease() { } if (needFlush) { scheduler.scheduleFlushAll().join(); + waitUntilWorkerTasksDone(); scheduler.scheduleRelease(true); } else { + // No volatile nodes left, but clean unpinned cache may still remain after previous flushes. + scheduler.scheduleRelease(true); + waitUntilWorkerTasksDone(); break; } } } + @TestOnly + private void waitUntilWorkerTasksDone() { + while (scheduler.getActiveWorkerNum() > 0 || !flushingRegionSet.isEmpty()) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + public void clear() { if (releaseMonitor != null) { releaseMonitor.shutdownNow(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/ICachedMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/ICachedMNode.java index 489b1a642aad9..244e077013c57 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/ICachedMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/ICachedMNode.java @@ -22,8 +22,27 @@ import org.apache.iotdb.commons.schema.node.IMNode; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.lock.LockEntry; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.cache.CacheEntry; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.basic.CachedBasicMNode; public interface ICachedMNode extends IMNode { + CachedBasicMNode getBasicMNode(); + + default boolean hasDeviceDescendant() { + return getBasicMNode().hasDeviceDescendant(); + } + + default void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + getBasicMNode().setHasDeviceDescendant(hasDeviceDescendant); + } + + default boolean isDeviceDescendantComputed() { + return getBasicMNode().isDeviceDescendantComputed(); + } + + default void setDeviceDescendantComputed(final boolean deviceDescendantComputed) { + getBasicMNode().setDeviceDescendantComputed(deviceDescendantComputed); + } + CacheEntry getCacheEntry(); void setCacheEntry(CacheEntry cacheEntry); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/basic/CachedBasicMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/basic/CachedBasicMNode.java index d34fde474abee..14bb9f3609b78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/basic/CachedBasicMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/basic/CachedBasicMNode.java @@ -47,6 +47,17 @@ public class CachedBasicMNode implements ICachedMNode { private ICachedMNode parent; private final CacheMNodeInfo cacheMNodeInfo; + /** Cached flag showing whether there is any device in the subtree below this node. */ + private boolean hasDeviceDescendant = false; + + /** + * Whether {@link #hasDeviceDescendant} is trusted for the current in-memory node instance. + * + *

      This state is intentionally not persisted. A node reloaded from PBTree can lazily recompute + * it when the wildcard-suffix optimization needs it. + */ + private boolean deviceDescendantComputed = false; + /** from root to this node, only be set when used once for InternalMNode */ private String fullPath; @@ -56,6 +67,11 @@ public CachedBasicMNode(ICachedMNode parent, String name) { this.cacheMNodeInfo = new CacheMNodeInfo(name); } + @Override + public CachedBasicMNode getBasicMNode() { + return this; + } + @Override public String getName() { return cacheMNodeInfo.getName(); @@ -99,6 +115,26 @@ public void setFullPath(String fullPath) { this.fullPath = fullPath; } + @Override + public boolean hasDeviceDescendant() { + return hasDeviceDescendant; + } + + @Override + public void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + this.hasDeviceDescendant = hasDeviceDescendant; + } + + @Override + public boolean isDeviceDescendantComputed() { + return deviceDescendantComputed; + } + + @Override + public void setDeviceDescendantComputed(final boolean deviceDescendantComputed) { + this.deviceDescendantComputed = deviceDescendantComputed; + } + @Override public PartialPath getPartialPath() { List detachedPath = new ArrayList<>(); @@ -245,6 +281,8 @@ public void setLockEntry(LockEntry lockEntry) { *

    1. basicMNodeInfo reference, 8B *
    2. parent reference, 8B *
    3. fullPath reference, 8B + *
    4. hasDeviceDescendant, 1B + *
    5. deviceDescendantComputed, 1B *
    *
  • MapEntry in parent *
      @@ -256,7 +294,7 @@ public void setLockEntry(LockEntry lockEntry) { */ @Override public int estimateSize() { - return 8 + 8 + 8 + 8 + 8 + 8 + 28 + cacheMNodeInfo.estimateSize(); + return 8 + 8 + 8 + 8 + 1 + 1 + 8 + 8 + 28 + cacheMNodeInfo.estimateSize(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/iterator/CachedTraverserIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/iterator/CachedTraverserIterator.java index 0d14e2251c4d5..0654c93b834ff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/iterator/CachedTraverserIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/iterator/CachedTraverserIterator.java @@ -48,4 +48,11 @@ public void close() { } super.close(); } + + @Override + protected void releaseSkippedNode(final ICachedMNode node) { + if (usingDirectChildrenIterator) { + store.unPin(node); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/WrappedSegment.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/WrappedSegment.java index e46f1aea0251c..65e1edebcf999 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/WrappedSegment.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/WrappedSegment.java @@ -105,17 +105,16 @@ public WrappedSegment(ByteBuffer buffer, boolean override) { } } - public WrappedSegment(ByteBuffer buffer) throws RecordDuplicatedException { + public WrappedSegment(ByteBuffer buffer) { this(buffer, true); } @TestOnly - public WrappedSegment(int size) throws RecordDuplicatedException { + public WrappedSegment(int size) { this(ByteBuffer.allocate(size)); } - public static ISegment initAsSegment(ByteBuffer buffer) - throws RecordDuplicatedException { + public static ISegment initAsSegment(ByteBuffer buffer) { if (buffer == null) { return null; } @@ -559,7 +558,6 @@ public Queue getAllRecords() throws MetadataException { @Override public int updateRecord(String key, ByteBuffer uBuffer) throws MetadataException { - int idx = binarySearchOnKeys(key); if (idx < 0) { throw new MetadataException(String.format("Record[key:%s] Not Existed.", key)); @@ -614,8 +612,8 @@ public int removeRecord(String key) { } // shift offsets forward - if (idx != recordNum) { - int shift = recordNum - idx; + if (idx != recordNum - 1) { + int shift = recordNum - idx - 1; this.buffer.position(SchemaFileConfig.SEG_HEADER_SIZE + idx * 2); ShortBuffer lb = this.buffer.asReadOnlyBuffer().asShortBuffer(); lb.get(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java index f1ac33d99ed60..00fdb1b81b4bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java @@ -25,15 +25,18 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.path.fa.IFAState; import org.apache.iotdb.commons.path.fa.IFATransition; +import org.apache.iotdb.commons.path.fa.match.IStateMatchInfo; import org.apache.iotdb.commons.schema.node.IMNode; import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; import org.apache.iotdb.commons.schema.tree.AbstractTreeVisitor; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.IMTreeStore; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.iterator.MNodeIterator; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.ReentrantReadOnlyCachedMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.ReleaseFlushMonitor; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.ICachedMNode; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MNodeUtils; import org.apache.iotdb.db.schemaengine.template.Template; @@ -41,9 +44,11 @@ import org.slf4j.LoggerFactory; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.Set; import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; @@ -180,6 +185,7 @@ private N getChild(N parent, String childName, boolean skipTemplateChildren) && skipPreDeletedSchema && child.isMeasurement() && child.getAsMeasurementMNode().isPreDeleted()) { + releaseNode(child); child = null; } } @@ -248,10 +254,107 @@ protected Iterator getChildrenIterator(N parent) throws MetadataException { if (parent.isAboveDatabase()) { return new MNodeIterator<>(parent.getChildren().values().iterator()); } else { + final Iterator optimizedIterator = + shouldUseLeafDeviceMeasurementOptimization() + ? getOptimizedChildrenIterator(parent) + : null; + if (optimizedIterator != null) { + return optimizedIterator; + } return store.getTraverserIterator(parent, templateMap, skipPreDeletedSchema); } } + protected boolean shouldUseLeafDeviceMeasurementOptimization() { + return false; + } + + @SuppressWarnings("unchecked") + private Iterator getOptimizedChildrenIterator(final N parent) throws MetadataException { + if (!parent.isDevice()) { + return null; + } + + if (parent instanceof IMemMNode) { + if (((IMemMNode) parent).hasDeviceDescendant()) { + return null; + } + } else if (parent instanceof ICachedMNode) { + if (hasDeviceDescendant((ICachedMNode) parent)) { + return null; + } + } else { + return null; + } + + final IStateMatchInfo stateMatchInfo = getCurrentStateMatchInfo(); + final Set candidateNames = new LinkedHashSet<>(); + + for (int i = 0; i < stateMatchInfo.getMatchedStateSize(); i++) { + final IFAState matchedState = stateMatchInfo.getMatchedState(i); + + for (final IFATransition transition : + patternFA.getPreciseMatchTransition(matchedState).values()) { + if (patternFA.getNextState(matchedState, transition).isFinal()) { + candidateNames.add(transition.getAcceptEvent()); + } + } + + final Iterator fuzzyTransitionIterator = + patternFA.getFuzzyMatchTransitionIterator(matchedState); + while (fuzzyTransitionIterator.hasNext()) { + if (patternFA.getNextState(matchedState, fuzzyTransitionIterator.next()).isFinal()) { + return null; + } + } + } + + if (candidateNames.isEmpty()) { + return null; + } + + // For leaf devices, `**.measurement` does not need to enumerate every measurement child. + try { + return getChildrenIterator(parent, candidateNames.iterator()); + } catch (final MetadataException e) { + throw e; + } catch (final Exception e) { + throw new MetadataException(e.getMessage(), e); + } + } + + @SuppressWarnings("unchecked") + private boolean hasDeviceDescendant(final ICachedMNode node) throws MetadataException { + if (node.isMeasurement()) { + node.setHasDeviceDescendant(false); + node.setDeviceDescendantComputed(true); + return false; + } + if (!node.isDeviceDescendantComputed()) { + node.setHasDeviceDescendant(hasDeviceDescendantInChildren(node)); + node.setDeviceDescendantComputed(true); + } + return node.hasDeviceDescendant(); + } + + @SuppressWarnings("unchecked") + private boolean hasDeviceDescendantInChildren(final ICachedMNode node) throws MetadataException { + final IMTreeStore cachedStore = (IMTreeStore) store; + try (final IMNodeIterator iterator = cachedStore.getChildrenIterator(node)) { + while (iterator.hasNext()) { + final ICachedMNode child = iterator.next(); + try { + if (child.isDevice() || hasDeviceDescendant(child)) { + return true; + } + } finally { + cachedStore.unPin(child); + } + } + return false; + } + } + @Override protected void releaseNodeIterator(Iterator nodeIterator) { if (nodeIterator instanceof IMNodeIterator) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/basic/MeasurementTraverser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/basic/MeasurementTraverser.java index 4996e3f0c8b36..490145d5371fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/basic/MeasurementTraverser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/basic/MeasurementTraverser.java @@ -84,4 +84,9 @@ protected boolean shouldVisitSubtreeOfFullMatchedNode(N node) { protected boolean shouldVisitSubtreeOfInternalMatchedNode(N node) { return !node.isMeasurement(); } + + @Override + protected boolean shouldUseLeafDeviceMeasurementOptimization() { + return true; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index b00c8737bd9b0..fa2f603d6facc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.exception.MemoryNotEnoughException; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; @@ -148,7 +149,7 @@ protected Map prepareTvListMapForQuery( // mutable tvlist TVList list = memChunk.getWorkingTVList(); TVList cloneList = null; - long tvListRamSize = list.calculateRamSize(); + TVList.RamInfo listRamInfo = list.calculateRamSize(); list.lockQueryList(); try { if (copyTimeFilter != null @@ -157,10 +158,46 @@ protected Map prepareTvListMapForQuery( } if (!isWorkMemTable) { + /* + * 1. Q1 queries this TVList while it is still in the working memtable and records a smaller + * visible row count. + * 2. Later writes append out-of-order rows to the same TVList, then FLUSH moves the + * memtable to the flushing list. + * 3. Q2 queries the flushing memtable. If Q2 directly reuses the original mutable TVList, + * Q2's query-side sort may reorder the indices in place. + * 4. Q1 continues to read with its old row count and the reordered indices. The converted + * value index can exceed Q1's bitmap range and cause out-of-bound access. + * + * Therefore, this flushing branch can reuse the original list only when it is already + * sorted or no active query is using it. Otherwise, Q2 should read from + * workingListForFlush. + */ + boolean canUseListDirectly = list.isSorted() || list.getQueryContextSet().isEmpty(); LOGGER.debug( "Flushing MemTable - add current query context to mutable TVList's query list"); - list.getQueryContextSet().add(context); - tvListQueryMap.put(list, list.rowCount()); + if (canUseListDirectly) { + list.getQueryContextSet().add(context); + tvListQueryMap.put(list, list.rowCount()); + } else { + TVList workingListForFlushSort = memChunk.initWorkingListForFlushIfNecessary(list, true); + /* + * The query will read from workingListForFlushSort, but cloneForFlushSort() only clones + * times and indices. The value arrays and bitmaps are still shared with the original + * list. + * + * Therefore, this query must also hold the original list until it finishes. Adding + * context to list.getQueryContextSet() lets flush/query cleanup see that the original + * list is still in use. Adding list to context.tvListSet makes + * releaseTVListOwnedByQuery() remove this context from the original list later. + * + * Do not put the original list into tvListQueryMap here. The actual read path must use + * workingListForFlushSort to avoid sorting the original list in place. + */ + list.getQueryContextSet().add(context); + context.addTVListToSet(Collections.singleton(list)); + workingListForFlushSort.getQueryContextSet().add(context); + tvListQueryMap.put(workingListForFlushSort, workingListForFlushSort.rowCount()); + } } else { if (list.isSorted() || list.getQueryContextSet().isEmpty()) { LOGGER.debug( @@ -189,8 +226,8 @@ protected Map prepareTvListMapForQuery( if (firstQuery instanceof FragmentInstanceContext) { MemoryReservationManager memoryReservationManager = ((FragmentInstanceContext) firstQuery).getMemoryReservationContext(); - memoryReservationManager.reserveMemoryCumulatively(tvListRamSize); - list.setReservedMemoryBytes(tvListRamSize); + memoryReservationManager.reserveMemoryCumulatively(listRamInfo.getRamSize()); + list.setReservedMemoryBytes(listRamInfo.getRamSize()); } list.setOwnerQuery(firstQuery); @@ -200,6 +237,15 @@ protected Map prepareTvListMapForQuery( tvListQueryMap.put(cloneList, cloneList.rowCount()); } } + } catch (MemoryNotEnoughException ex) { + LOGGER.warn( + "Failed to reserve memory for TVList: ramSize {}, timestampsSize {}, arrayMemCost {}, rowCount {}, dataTypes {}", + listRamInfo.getRamSize(), + listRamInfo.getTimestampsSize(), + listRamInfo.getArrayMemCost(), + listRamInfo.getRowCount(), + listRamInfo.getDataTypes()); + throw ex; } finally { list.unlockQueryList(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index c8439e0218399..1c772cdb20b47 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -90,7 +90,9 @@ import java.net.URL; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -1038,12 +1040,14 @@ private void stopTimedServiceAndThrow(ScheduledExecutorService pool, String pool public void getDiskSizeByDataRegion( Map dataRegionDisk, List dataRegionIds) { - dataRegionMap.forEach( - (dataRegionId, dataRegion) -> { - if (dataRegionIds.contains(dataRegionId.getId())) { - dataRegionDisk.put(dataRegionId.getId(), dataRegion.countRegionDiskSize()); - } - }); + final Collection targetDataRegionIds = + dataRegionIds.size() > 1 ? new HashSet<>(dataRegionIds) : dataRegionIds; + for (Integer dataRegionId : targetDataRegionIds) { + final DataRegion dataRegion = dataRegionMap.get(new DataRegionId(dataRegionId)); + if (dataRegion != null) { + dataRegionDisk.put(dataRegionId, dataRegion.countRegionDiskSize()); + } + } } public static File getDataRegionSystemDir(String dataBaseName, String dataRegionId) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 3cad57962e51e..d06362ffd3794 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -1193,7 +1193,7 @@ public void insertTablet(InsertTabletNode insertTabletNode) if (!insertTabletNode.isGeneratedByRemoteConsensusLeader()) { // disable updating last cache on follower startTime = System.nanoTime(); - tryToUpdateInsertTabletLastCache(insertTabletNode); + tryToUpdateInsertTabletLastCache(insertTabletNode, results); PERFORMANCE_OVERVIEW_METRICS.recordScheduleUpdateLastCacheCost( System.nanoTime() - startTime); } @@ -1261,13 +1261,13 @@ private boolean insertTabletToTsFileProcessor( return true; } - TsFileProcessor tsFileProcessor = getOrCreateTsFileProcessor(timePartitionId, sequence); - if (tsFileProcessor == null) { + final TsFileProcessor tsFileProcessor; + try { + tsFileProcessor = getOrCreateTsFileProcessor(timePartitionId, sequence); + } catch (WriteProcessException e) { + final TSStatus failureStatus = RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); for (int i = start; i < end; i++) { - results[i] = - RpcUtils.getStatus( - TSStatusCode.INTERNAL_SERVER_ERROR, - "can not create TsFileProcessor, timePartitionId: " + timePartitionId); + results[i] = failureStatus; } return false; } @@ -1289,8 +1289,9 @@ private boolean insertTabletToTsFileProcessor( return true; } - private void tryToUpdateInsertTabletLastCache(InsertTabletNode node) { - node.updateLastCache(getDatabaseName()); + private void tryToUpdateInsertTabletLastCache( + final InsertTabletNode node, final TSStatus[] results) { + node.updateLastCache(getDatabaseName(), results); } private TsFileProcessor insertToTsFileProcessor( @@ -1300,9 +1301,6 @@ private TsFileProcessor insertToTsFileProcessor( return null; } TsFileProcessor tsFileProcessor = getOrCreateTsFileProcessor(timePartitionId, sequence); - if (tsFileProcessor == null) { - return null; - } long[] costsForMetrics = new long[4]; tsFileProcessor.insert(insertRowNode, costsForMetrics); PERFORMANCE_OVERVIEW_METRICS.recordCreateMemtableBlockCost(costsForMetrics[0]); @@ -1325,9 +1323,11 @@ private List insertToTsFileProcessors( if (insertRowNode.allMeasurementFailed()) { continue; } - TsFileProcessor tsFileProcessor = - getOrCreateTsFileProcessor(timePartitionIds[i], areSequence[i]); - if (tsFileProcessor == null) { + final TsFileProcessor tsFileProcessor; + try { + tsFileProcessor = getOrCreateTsFileProcessor(timePartitionIds[i], areSequence[i]); + } catch (WriteProcessException e) { + insertRowsNode.getResults().put(i, RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); continue; } int finalI = i; @@ -1335,43 +1335,28 @@ private List insertToTsFileProcessors( tsFileProcessor, (k, v) -> { if (v == null) { - v = new InsertRowsNode(insertRowsNode.getPlanNodeId()); - v.setSearchIndex(insertRowNode.getSearchIndex()); - v.setAligned(insertRowNode.isAligned()); - if (insertRowNode.isGeneratedByPipe()) { - v.markAsGeneratedByPipe(); - } - if (insertRowNode.isGeneratedByRemoteConsensusLeader()) { - v.markAsGeneratedByRemoteConsensusLeader(); - } + v = createGroupedInsertRowsNode(insertRowsNode, insertRowNode); } - if (v.isAligned() != insertRowNode.isAligned()) { - v.setMixingAlignment(true); - } - v.addOneInsertRowNode(insertRowNode, finalI); - v.updateProgressIndex(insertRowNode.getProgressIndex()); + appendInsertRowNode(v, insertRowNode, finalI); return v; }); } List executedInsertRowNodeList = new ArrayList<>(); for (Map.Entry entry : tsFileProcessorMap.entrySet()) { - TsFileProcessor tsFileProcessor = entry.getKey(); InsertRowsNode subInsertRowsNode = entry.getValue(); try { - tsFileProcessor.insert(subInsertRowsNode, costsForMetrics); + List insertedProcessors = + insertRowsWithTypeConsistencyCheck(entry.getKey(), subInsertRowsNode, costsForMetrics); + executedInsertRowNodeList.addAll(subInsertRowsNode.getInsertRowNodeList()); + for (TsFileProcessor tsFileProcessor : insertedProcessors) { + // check memtable size and may asyncTryToFlush the work memtable + if (tsFileProcessor.shouldFlush()) { + fileFlushPolicy.apply(this, tsFileProcessor, tsFileProcessor.isSequence()); + } + } } catch (WriteProcessException e) { - insertRowsNode - .getResults() - .put( - subInsertRowsNode.getInsertRowNodeIndexList().get(0), - RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); - } - executedInsertRowNodeList.addAll(subInsertRowsNode.getInsertRowNodeList()); - - // check memtable size and may asyncTryToFlush the work memtable - if (entry.getKey().shouldFlush()) { - fileFlushPolicy.apply(this, tsFileProcessor, tsFileProcessor.isSequence()); + recordInsertRowsFailure(insertRowsNode, subInsertRowsNode, e); } } @@ -1382,6 +1367,121 @@ private List insertToTsFileProcessors( return executedInsertRowNodeList; } + private List insertRowsWithTypeConsistencyCheck( + TsFileProcessor tsFileProcessor, InsertRowsNode subInsertRowsNode, long[] costsForMetrics) + throws WriteProcessException { + tsFileProcessor.insert(subInsertRowsNode, costsForMetrics); + return Collections.singletonList(tsFileProcessor); + } + + private InsertRowsNode createGroupedInsertRowsNode( + final InsertRowsNode sourceInsertRowsNode, final InsertRowNode firstInsertRowNode) { + final InsertRowsNode groupedInsertRowsNode = + new InsertRowsNode(sourceInsertRowsNode.getPlanNodeId()); + initializeGroupedInsertRowsNode(groupedInsertRowsNode, firstInsertRowNode); + return groupedInsertRowsNode; + } + + private InsertRowsNode createGroupedInsertRowsNode( + final InsertRowsOfOneDeviceNode sourceInsertRowsNode, + final InsertRowNode firstInsertRowNode) { + final InsertRowsNode groupedInsertRowsNode = + new InsertRowsNode(sourceInsertRowsNode.getPlanNodeId()); + initializeGroupedInsertRowsNode(groupedInsertRowsNode, firstInsertRowNode); + return groupedInsertRowsNode; + } + + private void initializeGroupedInsertRowsNode( + final InsertRowsNode groupedInsertRowsNode, final InsertRowNode firstInsertRowNode) { + groupedInsertRowsNode.setSearchIndex(firstInsertRowNode.getSearchIndex()); + groupedInsertRowsNode.setAligned(firstInsertRowNode.isAligned()); + if (firstInsertRowNode.isGeneratedByPipe()) { + groupedInsertRowsNode.markAsGeneratedByPipe(); + } + if (firstInsertRowNode.isGeneratedByRemoteConsensusLeader()) { + groupedInsertRowsNode.markAsGeneratedByRemoteConsensusLeader(); + } + } + + private void appendInsertRowNode( + final InsertRowsNode groupedInsertRowsNode, + final InsertRowNode insertRowNode, + final int insertRowNodeIndex) { + if (groupedInsertRowsNode.isAligned() != insertRowNode.isAligned()) { + groupedInsertRowsNode.setMixingAlignment(true); + } + groupedInsertRowsNode.addOneInsertRowNode(insertRowNode, insertRowNodeIndex); + groupedInsertRowsNode.updateProgressIndex(insertRowNode.getProgressIndex()); + } + + private void flushWorkingProcessorsForTimePartition(final long timePartitionId) { + TsFileProcessor workSequenceProcessor = workSequenceTsFileProcessors.get(timePartitionId); + if (workSequenceProcessor != null) { + fileFlushPolicy.apply(this, workSequenceProcessor, workSequenceProcessor.isSequence()); + } + TsFileProcessor workUnsequenceProcessor = workUnsequenceTsFileProcessors.get(timePartitionId); + if (workUnsequenceProcessor != null) { + fileFlushPolicy.apply(this, workUnsequenceProcessor, workUnsequenceProcessor.isSequence()); + } + } + + private List retryInsertRowsAfterFlush( + final InsertRowsNode subInsertRowsNode, + final long timePartitionId, + final long[] costsForMetrics) + throws WriteProcessException { + final Map retriedProcessorMap = new HashMap<>(); + for (int i = 0; i < subInsertRowsNode.getInsertRowNodeList().size(); i++) { + final InsertRowNode insertRowNode = subInsertRowsNode.getInsertRowNodeList().get(i); + final boolean isSequence = + config.isEnableSeparateData() + && insertRowNode.getTime() + > lastFlushTimeMap.getFlushedTime(timePartitionId, insertRowNode.getDeviceID()); + final TsFileProcessor retriedProcessor = + getOrCreateTsFileProcessor(timePartitionId, isSequence); + final int insertRowNodeIndex = subInsertRowsNode.getInsertRowNodeIndexList().get(i); + retriedProcessorMap.compute( + retriedProcessor, + (k, v) -> { + if (v == null) { + v = createGroupedInsertRowsNode(subInsertRowsNode, insertRowNode); + } + appendInsertRowNode(v, insertRowNode, insertRowNodeIndex); + return v; + }); + } + + final List insertedProcessors = new ArrayList<>(retriedProcessorMap.size()); + for (Entry retriedEntry : retriedProcessorMap.entrySet()) { + final TsFileProcessor retriedProcessor = retriedEntry.getKey(); + retriedProcessor.insert(retriedEntry.getValue(), costsForMetrics); + insertedProcessors.add(retriedProcessor); + } + return insertedProcessors; + } + + private void recordInsertRowsFailure( + final InsertRowsNode sourceInsertRowsNode, + final InsertRowsNode failedInsertRowsNode, + final WriteProcessException exception) { + final TSStatus failureStatus = + RpcUtils.getStatus(exception.getErrorCode(), exception.getMessage()); + for (Integer failedIndex : failedInsertRowsNode.getInsertRowNodeIndexList()) { + sourceInsertRowsNode.getResults().put(failedIndex, failureStatus); + } + } + + private void recordInsertRowsFailure( + final InsertRowsOfOneDeviceNode sourceInsertRowsNode, + final InsertRowsNode failedInsertRowsNode, + final WriteProcessException exception) { + final TSStatus failureStatus = + RpcUtils.getStatus(exception.getErrorCode(), exception.getMessage()); + for (Integer failedIndex : failedInsertRowsNode.getInsertRowNodeIndexList()) { + sourceInsertRowsNode.getResults().put(failedIndex, failureStatus); + } + } + private void tryToUpdateInsertRowsLastCache(List nodeList) { for (InsertRowNode node : nodeList) { node.updateLastCache(databaseName); @@ -1440,7 +1540,8 @@ public void submitAFlushTaskWhenShouldFlush(TsFileProcessor tsFileProcessor) { } } - private TsFileProcessor getOrCreateTsFileProcessor(long timeRangeId, boolean sequence) { + protected TsFileProcessor getOrCreateTsFileProcessor(long timeRangeId, boolean sequence) + throws WriteProcessException { TsFileProcessor tsFileProcessor = null; int retryCnt = 0; do { @@ -1466,7 +1567,7 @@ private TsFileProcessor getOrCreateTsFileProcessor(long timeRangeId, boolean seq "disk space is insufficient when creating TsFile processor, change system mode to read-only", e); CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); - break; + throw new WriteProcessException(e.getMessage(), e.getErrorCode(), true); } catch (IOException e) { if (retryCnt < 3) { logger.warn("meet IOException when creating TsFileProcessor, retry it again", e); @@ -1475,11 +1576,15 @@ private TsFileProcessor getOrCreateTsFileProcessor(long timeRangeId, boolean seq logger.error( "meet IOException when creating TsFileProcessor, change system mode to error", e); CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); - break; + throw new WriteProcessException( + String.format( + "Failed to create TsFileProcessor for database %s, timePartitionId %s", + databaseName, timeRangeId), + e); } } catch (ExceedQuotaException e) { logger.error(e.getMessage()); - break; + throw new WriteProcessException(e.getMessage(), e.getErrorCode(), true); } } while (tsFileProcessor == null); return tsFileProcessor; @@ -3663,8 +3768,13 @@ public void insert(InsertRowsOfOneDeviceNode insertRowsOfOneDeviceNode) config.isEnableSeparateData() && insertRowNode.getTime() > lastFlushTimeMap.getFlushedTime(timePartitionId, insertRowNode.getDeviceID()); - TsFileProcessor tsFileProcessor = getOrCreateTsFileProcessor(timePartitionId, isSequence); - if (tsFileProcessor == null) { + final TsFileProcessor tsFileProcessor; + try { + tsFileProcessor = getOrCreateTsFileProcessor(timePartitionId, isSequence); + } catch (WriteProcessException e) { + insertRowsOfOneDeviceNode + .getResults() + .put(i, RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); continue; } int finalI = i; @@ -3672,39 +3782,28 @@ public void insert(InsertRowsOfOneDeviceNode insertRowsOfOneDeviceNode) tsFileProcessor, (k, v) -> { if (v == null) { - v = new InsertRowsNode(insertRowsOfOneDeviceNode.getPlanNodeId()); - v.setSearchIndex(insertRowNode.getSearchIndex()); - v.setAligned(insertRowNode.isAligned()); - if (insertRowNode.isGeneratedByPipe()) { - v.markAsGeneratedByPipe(); - } - if (insertRowNode.isGeneratedByRemoteConsensusLeader()) { - v.markAsGeneratedByRemoteConsensusLeader(); - } + v = createGroupedInsertRowsNode(insertRowsOfOneDeviceNode, insertRowNode); } - v.addOneInsertRowNode(insertRowNode, finalI); - v.updateProgressIndex(insertRowNode.getProgressIndex()); + appendInsertRowNode(v, insertRowNode, finalI); return v; }); } List executedInsertRowNodeList = new ArrayList<>(); for (Map.Entry entry : tsFileProcessorMap.entrySet()) { - TsFileProcessor tsFileProcessor = entry.getKey(); InsertRowsNode subInsertRowsNode = entry.getValue(); try { - tsFileProcessor.insert(subInsertRowsNode, costsForMetrics); + List insertedProcessors = + insertRowsWithTypeConsistencyCheck( + entry.getKey(), subInsertRowsNode, costsForMetrics); + executedInsertRowNodeList.addAll(subInsertRowsNode.getInsertRowNodeList()); + for (TsFileProcessor tsFileProcessor : insertedProcessors) { + // check memtable size and may asyncTryToFlush the work memtable + if (tsFileProcessor.shouldFlush()) { + fileFlushPolicy.apply(this, tsFileProcessor, tsFileProcessor.isSequence()); + } + } } catch (WriteProcessException e) { - insertRowsOfOneDeviceNode - .getResults() - .put( - subInsertRowsNode.getInsertRowNodeIndexList().get(0), - RpcUtils.getStatus(e.getErrorCode(), e.getMessage())); - } - executedInsertRowNodeList.addAll(subInsertRowsNode.getInsertRowNodeList()); - - // check memtable size and may asyncTryToFlush the work memtable - if (tsFileProcessor.shouldFlush()) { - fileFlushPolicy.apply(this, tsFileProcessor, tsFileProcessor.isSequence()); + recordInsertRowsFailure(insertRowsOfOneDeviceNode, subInsertRowsNode, e); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleTaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleTaskManager.java index 8348b87813768..516b1489a206d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleTaskManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleTaskManager.java @@ -65,6 +65,7 @@ public class CompactionScheduleTaskManager implements IService { ConcurrentHashMap.newKeySet(); private ReentrantLock lock = new ReentrantLock(); private volatile boolean init = false; + private volatile boolean isStoppingAllScheduleTask = false; @Override public void start() throws StartupException { @@ -76,8 +77,13 @@ public void start() throws StartupException { logger.info("Compaction schedule task manager started."); } + public boolean isStoppingAllScheduleTask() { + return isStoppingAllScheduleTask; + } + public void stopCompactionScheduleTasks() throws InterruptedException { lock.lock(); + isStoppingAllScheduleTask = true; try { for (Future task : submitCompactionScheduleTaskFutures) { task.cancel(true); @@ -121,6 +127,7 @@ public void checkAndMayApplyConfigurationChange() throws InterruptedException { public void startScheduleTasks() { lock.lock(); + isStoppingAllScheduleTask = false; try { // compaction selector for (int workerId = 0; workerId < compactionSelectorNum; workerId++) { @@ -144,6 +151,7 @@ public void startScheduleTasks() { @Override public void stop() { lock.lock(); + isStoppingAllScheduleTask = true; try { if (!init) { return; @@ -160,6 +168,7 @@ public void stop() { @Override public void waitAndStop(long milliseconds) { lock.lock(); + isStoppingAllScheduleTask = true; try { if (!init) { return; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleTaskWorker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleTaskWorker.java index 17ad0dd4334e6..714d232d62538 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleTaskWorker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleTaskWorker.java @@ -72,9 +72,26 @@ public Void call() { dataRegion.executeCompaction(); } } catch (InterruptedException ignored) { + boolean isStoppedByUser = + CompactionScheduleTaskManager.getInstance().isStoppingAllScheduleTask(); logger.info( - "[CompactionScheduleTaskWorker-{}] compaction schedule is interrupted", workerId); - return null; + "[CompactionScheduleTaskWorker-{}] compaction schedule is interrupted, isStopByUser: {}", + workerId, + isStoppedByUser); + if (isStoppedByUser) { + return null; + } + } catch (Exception e) { + logger.error( + "[CompactionScheduleTaskWorker-{}] Failed to execute compaction schedule task", + workerId, + e); + } catch (Throwable t) { + logger.error( + "[CompactionScheduleTaskWorker-{}] Failed to execute compaction schedule task and cannot recover", + workerId, + t); + throw t; } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/TTLScheduleTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/TTLScheduleTask.java index 393a9f6d2dc68..c8cba2b52927b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/TTLScheduleTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/TTLScheduleTask.java @@ -66,8 +66,21 @@ public Void call() throws Exception { } } } catch (InterruptedException ignored) { - logger.info("[TTLCheckTask-{}] TTL checker is interrupted", workerId); - return null; + boolean isStoppedByUser = + CompactionScheduleTaskManager.getInstance().isStoppingAllScheduleTask(); + logger.info( + "[TTLCheckTask-{}] TTL checker is interrupted, isStoppedByUser: {}", + workerId, + isStoppedByUser); + if (isStoppedByUser) { + return null; + } + } catch (Exception e) { + logger.error("[TTLCheckTask-{}] Failed to execute ttl check", workerId, e); + } catch (Throwable t) { + logger.error( + "[TTLCheckTask-{}] Failed to execute ttl check and cannot recover", workerId, t); + throw t; } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index eb49d5325b0b3..ab95fc57662aa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.storageengine.dataregion.read.filescan.impl.MemAlignedChunkHandleImpl; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.impl.MemChunkHandleImpl; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALReadUtils; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; import org.apache.iotdb.db.utils.MemUtils; import org.apache.iotdb.db.utils.ModificationUtils; @@ -901,7 +902,7 @@ public int serializedSize() { } int size = FIXED_SERIALIZED_SIZE; for (Map.Entry entry : memTableMap.entrySet()) { - size += ReadWriteIOUtils.sizeToWrite(((PlainDeviceID) entry.getKey()).toStringID()); + size += WALWriteUtils.sizeToWrite(((PlainDeviceID) entry.getKey()).toStringID()); size += Byte.BYTES; size += entry.getValue().serializedSize(); } @@ -948,7 +949,7 @@ protected void deserialize(DataInputStream stream, boolean multiTvListMemChunk) int memTableMapSize = stream.readInt(); for (int i = 0; i < memTableMapSize; ++i) { - IDeviceID deviceID = deviceIDFactory.getDeviceID(ReadWriteIOUtils.readString(stream)); + IDeviceID deviceID = deviceIDFactory.getDeviceID(WALReadUtils.readString(stream)); boolean isAligned = ReadWriteIOUtils.readBool(stream); IWritableMemChunkGroup memChunkGroup; if (multiTvListMemChunk) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java index 9db0196a3a113..6c773942fb72b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java @@ -45,6 +45,8 @@ public abstract class AbstractWritableMemChunk implements IWritableMemChunk { protected static long RETRY_INTERVAL_MS = 100L; protected static long MAX_WAIT_QUERY_MS = 60 * 1000L; + protected TVList workingListForFlush; + /** * Release the TVList if there is no query on it. Otherwise, it should set the first query as the * owner. TVList is released until all queries finish. If it throws memory-not-enough exception @@ -99,7 +101,7 @@ protected void maybeReleaseTvList(TVList tvList) { } private void tryReleaseTvList(TVList tvList) { - long tvListRamSize = tvList.calculateRamSize(); + long tvListRamSize = tvList.calculateRamSize().getRamSize(); tvList.lockQueryList(); try { if (tvList.getQueryContextSet().isEmpty()) { @@ -194,7 +196,50 @@ public abstract void writeAlignedTablet( public abstract IMeasurementSchema getSchema(); @Override - public abstract void sortTvListForFlush(); + public void sortTvListForFlush() { + TVList workingList = getWorkingTVList(); + if (workingList.isSorted()) { + workingListForFlush = workingList; + return; + } + + /* + * Concurrency background: + * + * A query may start earlier and record the current row count (rows) of the TVList as its + * visible range. After that, new unseq writes may arrive and immediately trigger a flush, which + * will sort the TVList. + * + * During sorting, the underlying indices array of the TVList may be reordered. + * If the query continues to use the previously recorded rows as its upper bound, + * it may convert a logical index to a physical index via the updated indices array. + * + * In this case, the converted physical index may exceed the previously visible + * rows range, leading to invalid access or unexpected behavior. + * + * To avoid this issue, when there are active queries on the working TVList, we must + * clone the times and indices before sorting, so that the flush sort does not mutate + * the data structures that concurrent queries rely on. + * + * Flushing-memtable queries may also reuse workingListForFlush instead of the original working + * TVList for the same reason. + */ + boolean needCloneTimesAndIndicesInWorkingTVList; + workingList.lockQueryList(); + try { + needCloneTimesAndIndicesInWorkingTVList = !workingList.getQueryContextSet().isEmpty(); + } finally { + workingList.unlockQueryList(); + } + workingListForFlush = + initWorkingListForFlushIfNecessary(workingList, needCloneTimesAndIndicesInWorkingTVList); + workingListForFlush.sort(); + } + + @Override + public void releaseTemporaryTvListForFlush() { + workingListForFlush = null; + } @Override public abstract int delete(long lowerBound, long upperBound); @@ -226,4 +271,14 @@ public abstract void encode( @Override public abstract int serializedSize(); + + @Override + public synchronized TVList initWorkingListForFlushIfNecessary( + TVList workingList, boolean needCloneTimesAndIndicesInWorkingTVList) { + if (workingListForFlush == null) { + workingListForFlush = + needCloneTimesAndIndicesInWorkingTVList ? workingList.cloneForFlushSort() : workingList; + } + return workingListForFlush; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index bbab08d444284..bb2ee311d3047 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -104,7 +104,7 @@ public AlignedReadOnlyMemChunk( this.valueStatisticsList = new ArrayList<>(); this.alignedTvListQueryMap = alignedTvListQueryMap; this.columnIndexList = columnIndexList; - this.context.addTVListToSet(alignedTvListQueryMap); + this.context.addTVListToSet(alignedTvListQueryMap.keySet()); } @Override @@ -122,7 +122,7 @@ public void sortTvLists() { // We must update queryRowCount here, otherwise, it may be used later to build // BitMaps, causing bitmap array size mismatch and possible out of bound. entry.setValue(alignedTvList.sort()); - long alignedTvListRamSize = alignedTvList.calculateRamSize(); + long alignedTvListRamSize = alignedTvList.calculateRamSize().getRamSize(); alignedTvList.lockQueryList(); try { FragmentInstanceContext ownerQuery = @@ -367,7 +367,7 @@ public IPointReader getPointReader() { int queryLength = entry.getValue(); if (!alignedTvList.isSorted() && queryLength > alignedTvList.seqRowCount()) { entry.setValue(alignedTvList.sort()); - long alignedTvListRamSize = alignedTvList.calculateRamSize(); + long alignedTvListRamSize = alignedTvList.calculateRamSize().getRamSize(); alignedTvList.lockQueryList(); try { FragmentInstanceContext ownerQuery = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index 2fca3ca398a51..06bd5404be0f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -63,6 +63,7 @@ public class AlignedWritableMemChunk extends AbstractWritableMemChunk { private final Map measurementIndexMap; private List dataTypes; private final List schemaList; + // Note: Use AbstractWritableMemChunk.workingListForFlush instead of list in FlushTask private AlignedTVList list; private List sortedList; private long sortedRowCount = 0; @@ -273,13 +274,6 @@ public long getMinTime() { return minTime; } - @Override - public synchronized void sortTvListForFlush() { - if (!list.isSorted()) { - list.sort(); - } - } - @Override public int delete(long lowerBound, long upperBound) { int deletedNumber = list.delete(lowerBound, upperBound); @@ -322,7 +316,8 @@ public void encodeWorkingAlignedTVList( BlockingQueue ioTaskQueue, long maxNumberOfPointsInChunk, int maxNumberOfPointsInPage) { - BitMap allValueColDeletedMap = list.getAllValueColDeletedMap(); + AlignedTVList alignedWorkingListForFlush = (AlignedTVList) workingListForFlush; + BitMap allValueColDeletedMap = alignedWorkingListForFlush.getAllValueColDeletedMap(); boolean[] timeDuplicateInfo = null; @@ -334,8 +329,10 @@ public void encodeWorkingAlignedTVList( int pointNumInPage = 0; int pointNumInChunk = 0; - for (int sortedRowIndex = 0; sortedRowIndex < list.rowCount(); sortedRowIndex++) { - long time = list.getTime(sortedRowIndex); + for (int sortedRowIndex = 0; + sortedRowIndex < alignedWorkingListForFlush.rowCount(); + sortedRowIndex++) { + long time = alignedWorkingListForFlush.getTime(sortedRowIndex); if (pointNumInPage == 0) { pageRange.add(sortedRowIndex); } @@ -356,14 +353,16 @@ public void encodeWorkingAlignedTVList( } int nextRowIndex = sortedRowIndex + 1; - while (nextRowIndex < list.rowCount() + while (nextRowIndex < alignedWorkingListForFlush.rowCount() && allValueColDeletedMap != null - && allValueColDeletedMap.isMarked(list.getValueIndex(nextRowIndex))) { + && allValueColDeletedMap.isMarked( + alignedWorkingListForFlush.getValueIndex(nextRowIndex))) { nextRowIndex++; } - if (nextRowIndex != list.rowCount() && time == list.getTime(nextRowIndex)) { + if (nextRowIndex != alignedWorkingListForFlush.rowCount() + && time == alignedWorkingListForFlush.getTime(nextRowIndex)) { if (Objects.isNull(timeDuplicateInfo)) { - timeDuplicateInfo = new boolean[list.rowCount()]; + timeDuplicateInfo = new boolean[alignedWorkingListForFlush.rowCount()]; } timeDuplicateInfo[sortedRowIndex] = true; } @@ -371,7 +370,7 @@ public void encodeWorkingAlignedTVList( } if (pointNumInPage != 0) { - pageRange.add(list.rowCount() - 1); + pageRange.add(alignedWorkingListForFlush.rowCount() - 1); } if (pointNumInChunk != 0) { chunkRange.add(pageRange); @@ -387,7 +386,8 @@ private void handleEncoding( boolean[] timeDuplicateInfo, BitMap allValueColDeletedMap, int maxNumberOfPointsInPage) { - List dataTypes = list.getTsDataTypes(); + AlignedTVList alignedWorkingListForFlush = (AlignedTVList) workingListForFlush; + List dataTypes = alignedWorkingListForFlush.getTsDataTypes(); Pair[] lastValidPointIndexForTimeDupCheck = new Pair[dataTypes.size()]; for (List pageRange : chunkRange) { AlignedChunkWriterImpl alignedChunkWriter = new AlignedChunkWriterImpl(schemaList); @@ -404,16 +404,18 @@ private void handleEncoding( sortedRowIndex++) { // skip empty row if (allValueColDeletedMap != null - && allValueColDeletedMap.isMarked(list.getValueIndex(sortedRowIndex))) { + && allValueColDeletedMap.isMarked( + alignedWorkingListForFlush.getValueIndex(sortedRowIndex))) { continue; } // skip time duplicated rows - long time = list.getTime(sortedRowIndex); + long time = alignedWorkingListForFlush.getTime(sortedRowIndex); if (Objects.nonNull(timeDuplicateInfo)) { - if (!list.isNullValue(list.getValueIndex(sortedRowIndex), columnIndex)) { + if (!alignedWorkingListForFlush.isNullValue( + alignedWorkingListForFlush.getValueIndex(sortedRowIndex), columnIndex)) { lastValidPointIndexForTimeDupCheck[columnIndex].left = time; lastValidPointIndexForTimeDupCheck[columnIndex].right = - list.getValueIndex(sortedRowIndex); + alignedWorkingListForFlush.getValueIndex(sortedRowIndex); } if (timeDuplicateInfo[sortedRowIndex]) { continue; @@ -434,41 +436,55 @@ private void handleEncoding( && (time == lastValidPointIndexForTimeDupCheck[columnIndex].left)) { originRowIndex = lastValidPointIndexForTimeDupCheck[columnIndex].right; } else { - originRowIndex = list.getValueIndex(sortedRowIndex); + originRowIndex = alignedWorkingListForFlush.getValueIndex(sortedRowIndex); } - boolean isNull = list.isNullValue(originRowIndex, columnIndex); + boolean isNull = alignedWorkingListForFlush.isNullValue(originRowIndex, columnIndex); switch (tsDataType) { case BOOLEAN: alignedChunkWriter.writeByColumn( time, - !isNull && list.getBooleanByValueIndex(originRowIndex, columnIndex), + !isNull + && alignedWorkingListForFlush.getBooleanByValueIndex( + originRowIndex, columnIndex), isNull); break; case INT32: case DATE: alignedChunkWriter.writeByColumn( time, - isNull ? 0 : list.getIntByValueIndex(originRowIndex, columnIndex), + isNull + ? 0 + : alignedWorkingListForFlush.getIntByValueIndex( + originRowIndex, columnIndex), isNull); break; case INT64: case TIMESTAMP: alignedChunkWriter.writeByColumn( time, - isNull ? 0 : list.getLongByValueIndex(originRowIndex, columnIndex), + isNull + ? 0 + : alignedWorkingListForFlush.getLongByValueIndex( + originRowIndex, columnIndex), isNull); break; case FLOAT: alignedChunkWriter.writeByColumn( time, - isNull ? 0 : list.getFloatByValueIndex(originRowIndex, columnIndex), + isNull + ? 0 + : alignedWorkingListForFlush.getFloatByValueIndex( + originRowIndex, columnIndex), isNull); break; case DOUBLE: alignedChunkWriter.writeByColumn( time, - isNull ? 0 : list.getDoubleByValueIndex(originRowIndex, columnIndex), + isNull + ? 0 + : alignedWorkingListForFlush.getDoubleByValueIndex( + originRowIndex, columnIndex), isNull); break; case TEXT: @@ -476,7 +492,10 @@ private void handleEncoding( case BLOB: alignedChunkWriter.writeByColumn( time, - isNull ? null : list.getBinaryByValueIndex(originRowIndex, columnIndex), + isNull + ? null + : alignedWorkingListForFlush.getBinaryByValueIndex( + originRowIndex, columnIndex), isNull); break; default: @@ -486,18 +505,20 @@ private void handleEncoding( alignedChunkWriter.nextColumn(); } - long[] times = new long[Math.min(maxNumberOfPointsInPage, list.rowCount())]; + long[] times = + new long[Math.min(maxNumberOfPointsInPage, alignedWorkingListForFlush.rowCount())]; int pointsInPage = 0; for (int sortedRowIndex = pageRange.get(pageNum * 2); sortedRowIndex <= pageRange.get(pageNum * 2 + 1); sortedRowIndex++) { // skip empty row if (allValueColDeletedMap != null - && allValueColDeletedMap.isMarked(list.getValueIndex(sortedRowIndex))) { + && allValueColDeletedMap.isMarked( + alignedWorkingListForFlush.getValueIndex(sortedRowIndex))) { continue; } if (Objects.isNull(timeDuplicateInfo) || !timeDuplicateInfo[sortedRowIndex]) { - times[pointsInPage++] = list.getTime(sortedRowIndex); + times[pointsInPage++] = alignedWorkingListForFlush.getTime(sortedRowIndex); } } alignedChunkWriter.write(times, pointsInPage, 0); @@ -513,8 +534,7 @@ private void handleEncoding( } @Override - public synchronized void encode( - BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo, long[] times) { + public void encode(BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo, long[] times) { if (TVLIST_SORT_THRESHOLD == 0) { encodeWorkingAlignedTVList( ioTaskQueue, encodeInfo.maxNumberOfPointsInChunk, encodeInfo.maxNumberOfPointsInPage); @@ -525,7 +545,7 @@ public synchronized void encode( // create MergeSortAlignedTVListIterator. List alignedTvLists = new ArrayList<>(sortedList); - alignedTvLists.add(list); + alignedTvLists.add((AlignedTVList) workingListForFlush); List columnIndexList = buildColumnIndexList(schemaList); MemPointIterator timeValuePairIterator = MemPointIteratorFactory.create( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java index 18ca5201bb049..376136e1d122e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java @@ -94,6 +94,8 @@ void writeAlignedTablet( */ void sortTvListForFlush(); + void releaseTemporaryTvListForFlush(); + default long getMaxTime() { return Long.MAX_VALUE; } @@ -124,4 +126,7 @@ default long getMinTime() { TVList getWorkingTVList(); void setWorkingTVList(TVList list); + + TVList initWorkingListForFlushIfNecessary( + TVList workingList, boolean needCloneTimesAndIndicesInWorkingTVList); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java index 38c8e9e2d13cc..c0a71bf7edcdc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/ReadOnlyMemChunk.java @@ -127,7 +127,7 @@ public ReadOnlyMemChunk( this.deletionList = deletionList; this.tvListQueryMap = tvListQueryMap; this.pageStatisticsList = new ArrayList<>(); - this.context.addTVListToSet(tvListQueryMap); + this.context.addTVListToSet(tvListQueryMap.keySet()); } public void sortTvLists() { @@ -136,7 +136,7 @@ public void sortTvLists() { int queryRowCount = entry.getValue(); if (!tvList.isSorted() && queryRowCount > tvList.seqRowCount()) { entry.setValue(tvList.sort()); - long tvListRamSize = tvList.calculateRamSize(); + long tvListRamSize = tvList.calculateRamSize().getRamSize(); tvList.lockQueryList(); try { FragmentInstanceContext ownerQuery = (FragmentInstanceContext) tvList.getOwnerQuery(); @@ -288,7 +288,7 @@ public IPointReader getPointReader() { int queryLength = entry.getValue(); if (!tvList.isSorted() && queryLength > tvList.seqRowCount()) { entry.setValue(tvList.sort()); - long tvListRamSize = tvList.calculateRamSize(); + long tvListRamSize = tvList.calculateRamSize().getRamSize(); tvList.lockQueryList(); try { FragmentInstanceContext ownerQuery = (FragmentInstanceContext) tvList.getOwnerQuery(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java index fd616ac94fa4a..90e96597b0cd2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java @@ -54,6 +54,7 @@ public class WritableMemChunk extends AbstractWritableMemChunk { private IMeasurementSchema schema; + // Note: Use AbstractWritableMemChunk.workingListForFlush instead of list in FlushTask private TVList list; private List sortedList; private long sortedRowCount = 0; @@ -241,13 +242,6 @@ public void putAlignedTablet(long[] t, Object[] v, BitMap[] bitMaps, int start, throw new UnSupportedDataTypeException(UNSUPPORTED_TYPE + schema.getType()); } - @Override - public synchronized void sortTvListForFlush() { - if (!list.isSorted()) { - list.sort(); - } - } - @Override public TVList getWorkingTVList() { return list; @@ -374,50 +368,53 @@ public void encodeWorkingTVList( ChunkWriterImpl chunkWriterImpl = createIChunkWriter(); long dataSizeInCurrentChunk = 0; int pointNumInCurrentChunk = 0; - for (int sortedRowIndex = 0; sortedRowIndex < list.rowCount(); sortedRowIndex++) { - if (list.isNullValue(list.getValueIndex(sortedRowIndex))) { + for (int sortedRowIndex = 0; + sortedRowIndex < workingListForFlush.rowCount(); + sortedRowIndex++) { + if (workingListForFlush.isNullValue(workingListForFlush.getValueIndex(sortedRowIndex))) { continue; } - long time = list.getTime(sortedRowIndex); + long time = workingListForFlush.getTime(sortedRowIndex); // skip duplicated data - if ((sortedRowIndex + 1 < list.rowCount() && (time == list.getTime(sortedRowIndex + 1)))) { + if ((sortedRowIndex + 1 < workingListForFlush.rowCount() + && (time == workingListForFlush.getTime(sortedRowIndex + 1)))) { continue; } // store last point for SDT - if (sortedRowIndex + 1 == list.rowCount()) { + if (sortedRowIndex + 1 == workingListForFlush.rowCount()) { chunkWriterImpl.setLastPoint(true); } switch (tsDataType) { case BOOLEAN: - chunkWriterImpl.write(time, list.getBoolean(sortedRowIndex)); + chunkWriterImpl.write(time, workingListForFlush.getBoolean(sortedRowIndex)); dataSizeInCurrentChunk += 8L + 1L; break; case INT32: case DATE: - chunkWriterImpl.write(time, list.getInt(sortedRowIndex)); + chunkWriterImpl.write(time, workingListForFlush.getInt(sortedRowIndex)); dataSizeInCurrentChunk += 8L + 4L; break; case INT64: case TIMESTAMP: - chunkWriterImpl.write(time, list.getLong(sortedRowIndex)); + chunkWriterImpl.write(time, workingListForFlush.getLong(sortedRowIndex)); dataSizeInCurrentChunk += 8L + 8L; break; case FLOAT: - chunkWriterImpl.write(time, list.getFloat(sortedRowIndex)); + chunkWriterImpl.write(time, workingListForFlush.getFloat(sortedRowIndex)); dataSizeInCurrentChunk += 8L + 4L; break; case DOUBLE: - chunkWriterImpl.write(time, list.getDouble(sortedRowIndex)); + chunkWriterImpl.write(time, workingListForFlush.getDouble(sortedRowIndex)); dataSizeInCurrentChunk += 8L + 8L; break; case TEXT: case BLOB: case STRING: - Binary value = list.getBinary(sortedRowIndex); + Binary value = workingListForFlush.getBinary(sortedRowIndex); chunkWriterImpl.write(time, value); dataSizeInCurrentChunk += 8L + getBinarySize(value); break; @@ -452,8 +449,7 @@ public void encodeWorkingTVList( } @Override - public synchronized void encode( - BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo, long[] times) { + public void encode(BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo, long[] times) { if (TVLIST_SORT_THRESHOLD == 0) { encodeWorkingTVList( ioTaskQueue, encodeInfo.maxNumberOfPointsInChunk, encodeInfo.targetChunkSize); @@ -467,7 +463,7 @@ public synchronized void encode( // create MultiTvListIterator. It need not handle float/double precision here. List tvLists = new ArrayList<>(sortedList); - tvLists.add(list); + tvLists.add(workingListForFlush); MemPointIterator timeValuePairIterator = MemPointIteratorFactory.create( schema.getType(), tvLists, encodeInfo.maxNumberOfPointsInPage); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java index 226c2fd398098..ff519a2206aa1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java @@ -22,10 +22,10 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternUtil; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALReadUtils; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; import org.apache.tsfile.utils.BitMap; -import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.schema.IMeasurementSchema; import java.io.DataInputStream; @@ -180,7 +180,7 @@ public int serializedSize() { int size = 0; size += Integer.BYTES; for (Map.Entry entry : memChunkMap.entrySet()) { - size += ReadWriteIOUtils.sizeToWrite(entry.getKey()); + size += WALWriteUtils.sizeToWrite(entry.getKey()); size += entry.getValue().serializedSize(); } return size; @@ -200,7 +200,7 @@ public static WritableMemChunkGroup deserialize(DataInputStream stream) throws I WritableMemChunkGroup memChunkGroup = new WritableMemChunkGroup(); int memChunkMapSize = stream.readInt(); for (int i = 0; i < memChunkMapSize; ++i) { - String measurement = ReadWriteIOUtils.readString(stream); + String measurement = WALReadUtils.readString(stream); IWritableMemChunk memChunk = WritableMemChunk.deserialize(stream); memChunkGroup.memChunkMap.put(measurement, memChunk); } @@ -212,7 +212,7 @@ public static WritableMemChunkGroup deserializeSingleTVListMemChunks(DataInputSt WritableMemChunkGroup memChunkGroup = new WritableMemChunkGroup(); int memChunkMapSize = stream.readInt(); for (int i = 0; i < memChunkMapSize; ++i) { - String measurement = ReadWriteIOUtils.readString(stream); + String measurement = WALReadUtils.readString(stream); IWritableMemChunk memChunk = WritableMemChunk.deserializeSingleTVListMemChunks(stream); memChunkGroup.memChunkMap.put(measurement, memChunk); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/Deletion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/Deletion.java index 07f8aecbfc026..c4c74df1623e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/Deletion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/Deletion.java @@ -24,10 +24,10 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.utils.RamUsageEstimator; -import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -113,7 +113,7 @@ public long serializeWithoutFileOffset(DataOutputStream stream) throws IOExcepti serializeSize += Long.BYTES; stream.writeLong(getEndTime()); serializeSize += Long.BYTES; - serializeSize += ReadWriteIOUtils.write(getPathString(), stream); + serializeSize += writeString(getPathString(), stream); return serializeSize; } @@ -121,8 +121,30 @@ public static Deletion deserializeWithoutFileOffset(DataInputStream stream) throws IOException, IllegalPathException { long startTime = stream.readLong(); long endTime = stream.readLong(); - return new Deletion( - getMeasurementPath(ReadWriteIOUtils.readString(stream)), 0, startTime, endTime); + return new Deletion(getMeasurementPath(readString(stream)), 0, startTime, endTime); + } + + private static int writeString(String value, DataOutputStream stream) throws IOException { + if (value == null) { + stream.writeInt(-1); + return Integer.BYTES; + } + byte[] bytes = value.getBytes(TSFileConfig.STRING_CHARSET); + stream.writeInt(bytes.length); + stream.write(bytes); + return Integer.BYTES + bytes.length; + } + + private static String readString(DataInputStream stream) throws IOException { + int strLength = stream.readInt(); + if (strLength < 0) { + return null; + } else if (strLength == 0) { + return ""; + } + byte[] bytes = new byte[strLength]; + stream.readFully(bytes); + return new String(bytes, TSFileConfig.STRING_CHARSET); } private static PartialPath getMeasurementPath(String path) throws IllegalPathException { @@ -137,7 +159,14 @@ private static PartialPath getMeasurementPath(String path) throws IllegalPathExc } public long getSerializedSize() { - return Long.BYTES * 2 + Integer.BYTES + (long) getPathString().length() * Character.BYTES; + return Long.BYTES * 2L + sizeToWriteString(getPathString()); + } + + private static int sizeToWriteString(String value) { + if (value == null) { + return Integer.BYTES; + } + return Integer.BYTES + value.getBytes(TSFileConfig.STRING_CHARSET).length; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/ClosedFileScanHandleImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/ClosedFileScanHandleImpl.java index 055b8517b48e8..1552b7d81fb0a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/ClosedFileScanHandleImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/ClosedFileScanHandleImpl.java @@ -53,11 +53,11 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; public class ClosedFileScanHandleImpl implements IFileScanHandle { @@ -65,13 +65,16 @@ public class ClosedFileScanHandleImpl implements IFileScanHandle { private final TsFileResource tsFileResource; private final QueryContext queryContext; private PatternTreeMap curFileMods = null; + // Used to cache the device-level modifications + private final Map> deviceToDeletionRanges; // Used to cache the modifications of each timeseries private final Map>> deviceToModifications; public ClosedFileScanHandleImpl(TsFileResource tsFileResource, QueryContext context) { this.tsFileResource = tsFileResource; this.queryContext = context; - this.deviceToModifications = new HashMap<>(); + this.deviceToDeletionRanges = new ConcurrentHashMap<>(); + this.deviceToModifications = new ConcurrentHashMap<>(); } @Override @@ -89,14 +92,26 @@ public boolean isDeviceTimeDeleted(IDeviceID deviceID, long timestamp) curFileMods != null ? curFileMods : queryContext.loadAllModificationsFromDisk(tsFileResource); - List modifications = queryContext.getPathModifications(curFileMods, deviceID); - List timeRangeList = - modifications.stream() - .filter(Deletion.class::isInstance) - .map(Deletion.class::cast) - .map(Deletion::getTimeRange) - .collect(Collectors.toList()); - return ModificationUtils.isPointDeletedWithoutOrderedRange(timestamp, timeRangeList); + List timeRangeList = deviceToDeletionRanges.get(deviceID); + if (timeRangeList == null) { + final List computedTimeRangeList = + getMergedTimeRanges(queryContext.getPathModifications(curFileMods, deviceID)); + final List existingTimeRangeList = + deviceToDeletionRanges.putIfAbsent(deviceID, computedTimeRangeList); + timeRangeList = existingTimeRangeList == null ? computedTimeRangeList : existingTimeRangeList; + } + return ModificationUtils.isPointDeleted(timestamp, timeRangeList); + } + + private static List getMergedTimeRanges(List modifications) { + List timeRangeList = new ArrayList<>(modifications.size()); + for (Modification modification : modifications) { + if (modification instanceof Deletion) { + timeRangeList.add(((Deletion) modification).getTimeRange()); + } + } + TimeRange.sortAndMerge(timeRangeList); + return timeRangeList; } @Override @@ -114,15 +129,9 @@ public boolean isTimeSeriesTimeDeleted(IDeviceID deviceID, String timeSeriesName : queryContext.loadAllModificationsFromDisk(tsFileResource); List modifications = queryContext.getPathModifications(curFileMods, new PartialPath(deviceID, timeSeriesName)); - List timeRangeList = - modifications.stream() - .filter(Deletion.class::isInstance) - .map(Deletion.class::cast) - .map(Deletion::getTimeRange) - .collect(Collectors.toList()); - TimeRange.sortAndMerge(timeRangeList); + List timeRangeList = getMergedTimeRanges(modifications); deviceToModifications - .computeIfAbsent(deviceID, k -> new HashMap<>()) + .computeIfAbsent(deviceID, k -> new ConcurrentHashMap<>()) .put(timeSeriesName, timeRangeList); return ModificationUtils.isPointDeleted(timestamp, timeRangeList); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/UnclosedFileScanHandleImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/UnclosedFileScanHandleImpl.java index 20ae0958a87d4..ef74c0e43369f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/UnclosedFileScanHandleImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/UnclosedFileScanHandleImpl.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileDeviceStartEndTimeIterator; +import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -39,15 +40,19 @@ import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; public class UnclosedFileScanHandleImpl implements IFileScanHandle { private final TsFileResource tsFileResource; private final Map>> deviceToChunkMetadataMap; private final Map>> deviceToMemChunkHandleMap; + private final Map> deviceToDeletionRanges; + private final Map>> deviceToTimeSeriesDeletionRanges; public UnclosedFileScanHandleImpl( Map>> deviceToChunkMetadataMap, @@ -56,6 +61,8 @@ public UnclosedFileScanHandleImpl( this.deviceToChunkMetadataMap = deviceToChunkMetadataMap; this.deviceToMemChunkHandleMap = deviceToMemChunkHandleMap; this.tsFileResource = tsFileResource; + this.deviceToDeletionRanges = new ConcurrentHashMap<>(); + this.deviceToTimeSeriesDeletionRanges = new ConcurrentHashMap<>(); } @Override @@ -68,19 +75,9 @@ public TsFileDeviceStartEndTimeIterator getDeviceStartEndTimeIterator() throws I @Override public boolean isDeviceTimeDeleted(IDeviceID deviceID, long timeArray) { - Map> chunkMetadataMap = deviceToChunkMetadataMap.get(deviceID); - for (List chunkMetadataList : chunkMetadataMap.values()) { - for (IChunkMetadata chunkMetadata : chunkMetadataList) { - if (chunkMetadata.getDeleteIntervalList() != null) { - for (TimeRange deleteInterval : chunkMetadata.getDeleteIntervalList()) { - if (deleteInterval.contains(timeArray)) { - return true; - } - } - } - } - } - return false; + List deletionRanges = + deviceToDeletionRanges.computeIfAbsent(deviceID, this::collectDeviceDeletionRanges); + return ModificationUtils.isPointDeleted(timeArray, deletionRanges); } @Override @@ -121,19 +118,13 @@ public Iterator getAllDeviceChunkMetaData() throws @Override public boolean isTimeSeriesTimeDeleted( IDeviceID deviceID, String timeSeriesName, long timestamp) { - List chunkMetadataList = - deviceToChunkMetadataMap.get(deviceID).get(timeSeriesName); - // check if timestamp is deleted by deleteInterval - for (IChunkMetadata chunkMetadata : chunkMetadataList) { - if (chunkMetadata.getDeleteIntervalList() != null) { - for (TimeRange deleteInterval : chunkMetadata.getDeleteIntervalList()) { - if (deleteInterval.contains(timestamp)) { - return true; - } - } - } - } - return false; + Map> timeSeriesDeletionRanges = + deviceToTimeSeriesDeletionRanges.computeIfAbsent( + deviceID, key -> new ConcurrentHashMap<>()); + List deletionRanges = + timeSeriesDeletionRanges.computeIfAbsent( + timeSeriesName, key -> collectTimeSeriesDeletionRanges(deviceID, key)); + return ModificationUtils.isPointDeleted(timestamp, deletionRanges); } @Override @@ -167,4 +158,44 @@ public boolean isDeleted() { public TsFileResource getTsResource() { return tsFileResource; } + + private List collectDeviceDeletionRanges(IDeviceID deviceID) { + Map> chunkMetadataMap = deviceToChunkMetadataMap.get(deviceID); + if (chunkMetadataMap == null || chunkMetadataMap.isEmpty()) { + return Collections.emptyList(); + } + List deletionRanges = new ArrayList<>(); + for (List chunkMetadataList : chunkMetadataMap.values()) { + appendDeletionRanges(deletionRanges, chunkMetadataList); + } + TimeRange.sortAndMerge(deletionRanges); + return deletionRanges; + } + + private List collectTimeSeriesDeletionRanges( + IDeviceID deviceID, String timeSeriesName) { + Map> chunkMetadataMap = deviceToChunkMetadataMap.get(deviceID); + if (chunkMetadataMap == null) { + return Collections.emptyList(); + } + List chunkMetadataList = chunkMetadataMap.get(timeSeriesName); + if (chunkMetadataList == null || chunkMetadataList.isEmpty()) { + return Collections.emptyList(); + } + List deletionRanges = new ArrayList<>(); + appendDeletionRanges(deletionRanges, chunkMetadataList); + TimeRange.sortAndMerge(deletionRanges); + return deletionRanges; + } + + private void appendDeletionRanges( + List deletionRanges, List chunkMetadataList) { + for (IChunkMetadata chunkMetadata : chunkMetadataList) { + if (chunkMetadata.getDeleteIntervalList() != null) { + for (TimeRange deletionRange : chunkMetadata.getDeleteIntervalList()) { + deletionRanges.add(new TimeRange(deletionRange.getMin(), deletionRange.getMax())); + } + } + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java index 1c83153499147..be7052536ed53 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileManager.java @@ -191,14 +191,7 @@ public Iterator getIterator(boolean sequence) { public void remove(TsFileResource tsFileResource, boolean sequence) { writeLock("remove"); try { - Map selectedMap = sequence ? sequenceFiles : unsequenceFiles; - for (Map.Entry entry : selectedMap.entrySet()) { - if (entry.getValue().contains(tsFileResource)) { - entry.getValue().remove(tsFileResource); - TsFileResourceManager.getInstance().removeTsFileResource(tsFileResource); - break; - } - } + removeFromPartitionFileList(tsFileResource, sequence); } finally { writeUnlock(); } @@ -208,10 +201,18 @@ public void removeAll(List tsFileResourceList, boolean sequence) writeLock("removeAll"); try { for (TsFileResource resource : tsFileResourceList) { - remove(resource, sequence); + removeFromPartitionFileList(resource, sequence); } } finally { - writeLock("removeAll"); + writeUnlock(); + } + } + + private void removeFromPartitionFileList(TsFileResource tsFileResource, boolean sequence) { + Map selectedMap = sequence ? sequenceFiles : unsequenceFiles; + TsFileResourceList tsFileResources = selectedMap.get(tsFileResource.getTimePartition()); + if (tsFileResources != null && tsFileResources.remove(tsFileResource)) { + TsFileResourceManager.getInstance().removeTsFileResource(tsFileResource); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 8571fd2e91528..9f344c1994bec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -639,11 +639,13 @@ public void removeModFile() throws IOException { * file physically. */ public boolean remove() { - forceMarkDeleted(); // To release the memory occupied by pipe if held by it // Note that pipe can safely handle the case that the time index does not exist isEmpty(); - degradeTimeIndex(); + if (getStatus() != TsFileResourceStatus.UNCLOSED) { + degradeTimeIndex(); + } + forceMarkDeleted(); try { fsFactory.deleteIfExists(file); fsFactory.deleteIfExists( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/IWALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/IWALBuffer.java index 978ea78116268..082dcca8c05bb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/IWALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/IWALBuffer.java @@ -51,7 +51,7 @@ public interface IWALBuffer extends AutoCloseable { * * @throws InterruptedException when interrupted by the flush thread */ - void waitForFlush() throws InterruptedException; + void waitForRollFile() throws InterruptedException; /** * Wait for next flush operation done, if the predicate == true after entering a locked @@ -60,14 +60,14 @@ public interface IWALBuffer extends AutoCloseable { * @param waitPredicate the condition which should be satisfied before waiting. * @throws InterruptedException when interrupted by the flush thread */ - public void waitForFlush(Predicate waitPredicate) throws InterruptedException; + public void waitForRollFile(Predicate waitPredicate) throws InterruptedException; /** * Wait for next flush operation done. * * @throws InterruptedException when interrupted by the flush thread */ - boolean waitForFlush(long time, TimeUnit unit) throws InterruptedException; + boolean waitForRollFile(long time, TimeUnit unit) throws InterruptedException; /** Return true when all wal entries all consumed and flushed. */ boolean isAllWALEntriesConsumed(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index 36c404faa7c2a..08d1ad384bbb7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -88,6 +88,7 @@ public class WALBuffer extends AbstractWALBuffer { private final Lock buffersLock = new ReentrantLock(); // condition to guarantee correctness of switching buffers private final Condition idleBufferReadyCondition = buffersLock.newCondition(); + private final Condition rollLogWriterCondition = buffersLock.newCondition(); // last writer position when fsync is called, help record each entry's position private long lastFsyncPosition; @@ -168,6 +169,13 @@ private int getCompressedByteBufferSize(int size) { protected File rollLogWriter(long searchIndex, WALFileStatus fileStatus) throws IOException { File file = super.rollLogWriter(searchIndex, fileStatus); currentWALFileWriter.setCompressedByteBuffer(compressedByteBuffer); + buffersLock.lock(); + try { + // notify WALReader that new file is generated, and it can read new file + rollLogWriterCondition.signalAll(); + } finally { + buffersLock.unlock(); + } return file; } @@ -640,7 +648,7 @@ private void switchSyncingBufferToIdle() { } @Override - public void waitForFlush() throws InterruptedException { + public void waitForRollFile() throws InterruptedException { buffersLock.lock(); try { idleBufferReadyCondition.await(); @@ -650,11 +658,11 @@ public void waitForFlush() throws InterruptedException { } @Override - public void waitForFlush(Predicate waitPredicate) throws InterruptedException { + public void waitForRollFile(Predicate waitPredicate) throws InterruptedException { buffersLock.lock(); try { if (waitPredicate.test(this)) { - idleBufferReadyCondition.await(); + rollLogWriterCondition.await(); } } finally { buffersLock.unlock(); @@ -662,10 +670,10 @@ public void waitForFlush(Predicate waitPredicate) throws InterruptedE } @Override - public boolean waitForFlush(long time, TimeUnit unit) throws InterruptedException { + public boolean waitForRollFile(long time, TimeUnit unit) throws InterruptedException { buffersLock.lock(); try { - return idleBufferReadyCondition.await(time, unit); + return rollLogWriterCondition.await(time, unit); } finally { buffersLock.unlock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java index 15c8faa8993d7..a9af8f81c7d62 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java @@ -64,7 +64,14 @@ public WALInfoEntry(long memTableId, InsertTabletNode value, int tabletStart, in @Override public int serializedSize() { - return FIXED_SERIALIZED_SIZE + (value == null ? 0 : value.serializedSize()); + if (value == null) { + return FIXED_SERIALIZED_SIZE; + } + if (value instanceof InsertTabletNode && tabletInfo != null) { + return FIXED_SERIALIZED_SIZE + + ((InsertTabletNode) value).serializedSize(tabletInfo.tabletStart, tabletInfo.tabletEnd); + } + return FIXED_SERIALIZED_SIZE + value.serializedSize(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java index 003c74763da77..b0e19364ddb21 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java @@ -92,7 +92,7 @@ public class WALNode implements IWALNode { // no iot consensus, all insert nodes can be safely deleted public static final long DEFAULT_SAFELY_DELETED_SEARCH_INDEX = Long.MAX_VALUE; // timeout threshold when waiting for next wal entry - private static final long WAIT_FOR_NEXT_WAL_ENTRY_TIMEOUT_IN_SEC = 30; + public static final long WAIT_FOR_NEXT_WAL_ENTRY_TIMEOUT_IN_SEC = 30; private static final WritingMetrics WRITING_METRICS = WritingMetrics.getInstance(); // unique identifier of this WALNode @@ -733,7 +733,7 @@ public void waitForNextReady() throws InterruptedException { while (!hasNext()) { if (!walFileRolled) { boolean timeout = - !buffer.waitForFlush(WAIT_FOR_NEXT_WAL_ENTRY_TIMEOUT_IN_SEC, TimeUnit.SECONDS); + !buffer.waitForRollFile(WAIT_FOR_NEXT_WAL_ENTRY_TIMEOUT_IN_SEC, TimeUnit.SECONDS); if (timeout) { bufferLastSearchIndex = buffer.getCurrentSearchIndex(); logger.info( @@ -746,7 +746,7 @@ public void waitForNextReady() throws InterruptedException { } else { // only wait when the search index of the buffer remains the same as the previous check long finalBufferLastSearchIndex = bufferLastSearchIndex; - buffer.waitForFlush(buf -> buf.getCurrentSearchIndex() == finalBufferLastSearchIndex); + buffer.waitForRollFile(buf -> buf.getCurrentSearchIndex() == finalBufferLastSearchIndex); } } } @@ -755,8 +755,8 @@ public void waitForNextReady() throws InterruptedException { public void waitForNextReady(long time, TimeUnit unit) throws InterruptedException, TimeoutException { if (!hasNext()) { - boolean timeout = !buffer.waitForFlush(time, unit); - if (timeout || !hasNext()) { + boolean timeout = !buffer.waitForRollFile(time, unit); + if (timeout && !hasNext()) { throw new TimeoutException(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALReadUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALReadUtils.java new file mode 100644 index 0000000000000..6d467f19934a5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALReadUtils.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.wal.utils; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +/** Read methods paired with {@link WALWriteUtils}. */ +public class WALReadUtils { + + private WALReadUtils() {} + + public static String readString(DataInputStream stream) throws IOException { + int strLength = stream.readInt(); + if (strLength < 0) { + return null; + } else if (strLength == 0) { + return ""; + } + byte[] bytes = new byte[strLength]; + stream.readFully(bytes); + return new String(bytes, TSFileConfig.STRING_CHARSET); + } + + public static String readString(ByteBuffer buffer) { + int strLength = buffer.getInt(); + if (strLength < 0) { + return null; + } else if (strLength == 0) { + return ""; + } + byte[] bytes = new byte[strLength]; + buffer.get(bytes); + return new String(bytes, TSFileConfig.STRING_CHARSET); + } + + public static MeasurementSchema readMeasurementSchema(DataInputStream stream) throws IOException { + String measurementId = readString(stream); + byte type = stream.readByte(); + byte encoding = stream.readByte(); + byte compressor = stream.readByte(); + Map props = readProps(stream); + return new MeasurementSchema(measurementId, type, encoding, compressor, props); + } + + public static MeasurementSchema readMeasurementSchema(ByteBuffer buffer) { + String measurementId = readString(buffer); + byte type = buffer.get(); + byte encoding = buffer.get(); + byte compressor = buffer.get(); + Map props = readProps(buffer); + return new MeasurementSchema(measurementId, type, encoding, compressor, props); + } + + private static Map readProps(DataInputStream stream) throws IOException { + int size = stream.readInt(); + if (size <= 0) { + return null; + } + Map props = new HashMap<>(); + for (int i = 0; i < size; i++) { + props.put(readString(stream), readString(stream)); + } + return props; + } + + private static Map readProps(ByteBuffer buffer) { + int size = buffer.getInt(); + if (size <= 0) { + return null; + } + Map props = new HashMap<>(); + for (int i = 0; i < size; i++) { + props.put(readString(buffer), readString(buffer)); + } + return props; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java index e6c0eb02722ae..72dd2671fa313 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java @@ -21,6 +21,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; +import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; @@ -127,13 +128,20 @@ public static int write(String s, IWALByteBufferView buffer) { return write(NO_BYTE_TO_READ, buffer); } int len = 0; - byte[] bytes = s.getBytes(); + byte[] bytes = s.getBytes(TSFileConfig.STRING_CHARSET); len += write(bytes.length, buffer); buffer.put(bytes); len += bytes.length; return len; } + public static int sizeToWrite(String s) { + if (s == null) { + return INT_LEN; + } + return INT_LEN + s.getBytes(TSFileConfig.STRING_CHARSET).length; + } + /** * Write IDeviceID to byteBuffer. * @@ -196,15 +204,15 @@ public static int write(MeasurementSchema measurementSchema, IWALByteBufferView public static int sizeToWrite(MeasurementSchema measurementSchema) { int byteLen = 0; - byteLen += ReadWriteIOUtils.sizeToWrite(measurementSchema.getMeasurementId()); + byteLen += sizeToWrite(measurementSchema.getMeasurementId()); byteLen += 3 * Byte.BYTES; Map props = measurementSchema.getProps(); byteLen += Integer.BYTES; if (props != null) { for (Map.Entry entry : props.entrySet()) { - byteLen += ReadWriteIOUtils.sizeToWrite(entry.getKey()); - byteLen += ReadWriteIOUtils.sizeToWrite(entry.getValue()); + byteLen += sizeToWrite(entry.getKey()); + byteLen += sizeToWrite(entry.getValue()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index ce1bb680f21e2..0b85e3f0e644a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -128,6 +128,34 @@ public LoadTsFileManager() { activeLoadAgent.start(); } + private long getCleanupTaskDelayInMs() { + return CONFIG.getLoadCleanupTaskExecutionDelayTimeSeconds() * 1000L; + } + + private void createCleanupTaskIfAbsent(final String uuid) { + synchronized (uuid2CleanupTask) { + if (uuid2CleanupTask.containsKey(uuid)) { + return; + } + + final CleanupTask cleanupTask = new CleanupTask(uuid, getCleanupTaskDelayInMs()); + uuid2CleanupTask.put(uuid, cleanupTask); + cleanupTaskQueue.add(cleanupTask); + } + } + + private void rescheduleCleanupTask(final CleanupTask cleanupTask) { + synchronized (uuid2CleanupTask) { + if (uuid2CleanupTask.get(cleanupTask.uuid) != cleanupTask) { + return; + } + + cleanupTaskQueue.remove(cleanupTask); + cleanupTask.resetScheduledTime(); + cleanupTaskQueue.add(cleanupTask); + } + } + private void registerCleanupTaskExecutor() { PipeDataNodeAgent.runtime() .registerPeriodicalJob( @@ -199,26 +227,13 @@ private void recover() { uuid2WriterManager.put(uuid, writerManager); writerManager.close(); - synchronized (uuid2CleanupTask) { - final CleanupTask cleanupTask = - new CleanupTask( - uuid, CONFIG.getLoadCleanupTaskExecutionDelayTimeSeconds() * 1000); - uuid2CleanupTask.put(uuid, cleanupTask); - cleanupTaskQueue.add(cleanupTask); - } + createCleanupTaskIfAbsent(uuid); }); } public void writeToDataRegion(DataRegion dataRegion, LoadTsFilePieceNode pieceNode, String uuid) throws IOException, PageException { - if (!uuid2WriterManager.containsKey(uuid)) { - synchronized (uuid2CleanupTask) { - final CleanupTask cleanupTask = - new CleanupTask(uuid, CONFIG.getLoadCleanupTaskExecutionDelayTimeSeconds() * 1000); - uuid2CleanupTask.put(uuid, cleanupTask); - cleanupTaskQueue.add(cleanupTask); - } - } + createCleanupTaskIfAbsent(uuid); final Optional cleanupTask = Optional.ofNullable(uuid2CleanupTask.get(uuid)); cleanupTask.ifPresent(CleanupTask::markLoadTaskRunning); @@ -293,6 +308,8 @@ public boolean loadAll( return false; } + createCleanupTaskIfAbsent(uuid); + final Optional cleanupTask = Optional.ofNullable(uuid2CleanupTask.get(uuid)); cleanupTask.ifPresent(CleanupTask::markLoadTaskRunning); try { @@ -315,9 +332,10 @@ public boolean deleteAll(String uuid) { private void clean(String uuid) { synchronized (uuid2CleanupTask) { - final CleanupTask cleanupTask = uuid2CleanupTask.get(uuid); + final CleanupTask cleanupTask = uuid2CleanupTask.remove(uuid); if (cleanupTask != null) { cleanupTask.cancel(); + cleanupTaskQueue.remove(cleanupTask); } } @@ -744,12 +762,12 @@ private CleanupTask(String uuid, long delayInMs) { public void markLoadTaskRunning() { isLoadTaskRunning = true; - resetScheduledTime(); + rescheduleCleanupTask(this); } public void markLoadTaskNotRunning() { isLoadTaskRunning = false; - resetScheduledTime(); + rescheduleCleanupTask(this); } public void resetScheduledTime() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPathHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPathHelper.java index 965f2941dc659..2503b822b9eac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPathHelper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadPathHelper.java @@ -46,6 +46,7 @@ public final class ActiveLoadPathHelper { private static final List KEY_ORDER = Collections.unmodifiableList( Arrays.asList( + LoadTsFileConfigurator.DATABASE_NAME_KEY, LoadTsFileConfigurator.DATABASE_LEVEL_KEY, LoadTsFileConfigurator.CONVERT_ON_TYPE_MISMATCH_KEY, LoadTsFileConfigurator.TABLET_CONVERSION_THRESHOLD_KEY, @@ -62,8 +63,28 @@ public static Map buildAttributes( final Boolean verify, final Long tabletConversionThresholdBytes, final Boolean pipeGenerated) { + return buildAttributes( + null, + databaseLevel, + convertOnTypeMismatch, + verify, + tabletConversionThresholdBytes, + pipeGenerated); + } + + public static Map buildAttributes( + final String databaseName, + final Integer databaseLevel, + final Boolean convertOnTypeMismatch, + final Boolean verify, + final Long tabletConversionThresholdBytes, + final Boolean pipeGenerated) { final Map attributes = new LinkedHashMap<>(); + if (Objects.nonNull(databaseName) && !databaseName.isEmpty()) { + attributes.put(LoadTsFileConfigurator.DATABASE_NAME_KEY, databaseName); + } + if (Objects.nonNull(databaseLevel)) { attributes.put(LoadTsFileConfigurator.DATABASE_LEVEL_KEY, databaseLevel.toString()); } @@ -149,6 +170,10 @@ public static void applyAttributesToStatement( final LoadTsFileStatement statement, final boolean defaultVerify) { + Optional.ofNullable(attributes.get(LoadTsFileConfigurator.DATABASE_NAME_KEY)) + .filter(name -> !name.isEmpty()) + .ifPresent(statement::setDatabase); + Optional.ofNullable(attributes.get(LoadTsFileConfigurator.DATABASE_LEVEL_KEY)) .ifPresent( level -> { @@ -216,6 +241,11 @@ private static Optional extractAndValidateAttributeValue( private static void validateAttributeValue(final String key, final String value) { switch (key) { + case LoadTsFileConfigurator.DATABASE_NAME_KEY: + if (value == null || value.isEmpty()) { + throw new SemanticException("Database name must not be empty."); + } + break; case LoadTsFileConfigurator.DATABASE_LEVEL_KEY: LoadTsFileConfigurator.validateDatabaseLevelParam(value); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java index 8a1f305e4f28c..a228ef6c6514c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/active/ActiveLoadTsFileLoader.java @@ -218,7 +218,8 @@ private TSStatus loadTsFile( final ActiveLoadPendingQueue.ActiveLoadEntry entry, final IClientSession session) throws FileNotFoundException { final File tsFile = new File(entry.getFile()); - final LoadTsFileStatement statement = new LoadTsFileStatement(entry.getFile()); + final LoadTsFileStatement statement = + LoadTsFileStatement.createUnchecked(tsFile.getAbsolutePath()); statement.setDeleteAfterLoad(true); statement.setAutoCreateDatabase( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java index 510d47b0b23df..8b689c6fb22ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/config/LoadTsFileConfigurator.java @@ -40,6 +40,7 @@ public static void validateParameters(final String key, final String value) { case ON_SUCCESS_KEY: validateOnSuccessParam(value); break; + case DATABASE_NAME_KEY: case TABLET_CONVERSION_THRESHOLD_KEY: break; case CONVERT_ON_TYPE_MISMATCH_KEY: @@ -87,6 +88,12 @@ public static int parseOrGetDefaultDatabaseLevel(final Map loadA DATABASE_LEVEL_KEY, String.valueOf(DATABASE_LEVEL_DEFAULT_VALUE))); } + public static final String DATABASE_NAME_KEY = "database-name"; + + public static String parseDatabaseName(final Map loadAttributes) { + return loadAttributes.get(DATABASE_NAME_KEY); + } + public static final String ON_SUCCESS_KEY = "on-success"; public static final String ON_SUCCESS_DELETE_VALUE = "delete"; public static final String ON_SUCCESS_NONE_VALUE = "none"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java index 6c85d2280fd0b..c3f8881530050 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManager.java @@ -126,7 +126,18 @@ public synchronized LoadTsFileDataCacheMemoryBlock allocateDataCacheMemoryBlock( if (dataCacheMemoryBlock == null) { final long actuallyAllocateMemoryInBytes = tryAllocateFromQuery(MEMORY_TOTAL_SIZE_FROM_QUERY_IN_BYTES >> 2); - dataCacheMemoryBlock = new LoadTsFileDataCacheMemoryBlock(actuallyAllocateMemoryInBytes); + try { + dataCacheMemoryBlock = new LoadTsFileDataCacheMemoryBlock(actuallyAllocateMemoryInBytes); + } catch (RuntimeException e) { + if (actuallyAllocateMemoryInBytes > 0) { + try { + releaseToQuery(actuallyAllocateMemoryInBytes); + } catch (RuntimeException releaseException) { + e.addSuppressed(releaseException); + } + } + throw e; + } LOGGER.info( "Create Data Cache Memory Block {}, allocate memory {}", dataCacheMemoryBlock, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java index 86bd02c9947d1..8567d30660def 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/AlignedChunkData.java @@ -287,7 +287,7 @@ public void writeDecodeValuePage( } protected void deserializeTsFileData(TsFileIOWriter writer) throws IOException, PageException { - final InputStream stream = new ByteArrayInputStream(chunkData); + final InputStream stream = createTsFileDataInputStream(); if (needDecodeChunk) { buildChunkWriter(stream, writer); } else { @@ -295,6 +295,12 @@ protected void deserializeTsFileData(TsFileIOWriter writer) throws IOException, } } + private InputStream createTsFileDataInputStream() { + return chunkData == null + ? new ByteArrayInputStream(byteStream.getBuf(), 0, byteStream.size()) + : new ByteArrayInputStream(chunkData); + } + protected void deserializeTsFileDataByte(final InputStream stream) throws IOException { final int size = ReadWriteIOUtils.readInt(stream); this.chunkData = new byte[size]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java index 186426650fe50..dc4908ef2c38f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/DeletionData.java @@ -59,6 +59,10 @@ public void serialize(DataOutputStream stream) throws IOException { deletion.serializeWithoutFileOffset(stream); } + public Deletion getModEntry() { + return this.deletion; + } + public static DeletionData deserialize(InputStream stream) throws IllegalPathException, IOException { return new DeletionData(Deletion.deserializeWithoutFileOffset(new DataInputStream(stream))); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java index 2d8a75052ffb9..6c5504e7a99d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/NonAlignedChunkData.java @@ -38,6 +38,7 @@ import org.apache.tsfile.write.schema.MeasurementSchema; import org.apache.tsfile.write.writer.TsFileIOWriter; +import java.io.ByteArrayInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; @@ -113,6 +114,7 @@ public boolean isAligned() { @Override public void writeToFileWriter(final TsFileIOWriter writer) throws IOException { + ensureDataReadyForWriting(); if (chunk != null) { writer.writeChunk(chunk); } else { @@ -120,6 +122,18 @@ public void writeToFileWriter(final TsFileIOWriter writer) throws IOException { } } + private void ensureDataReadyForWriting() throws IOException { + if (chunk != null || chunkWriter != null) { + return; + } + + try { + deserializeTsFileData(new ByteArrayInputStream(byteStream.getBuf(), 0, byteStream.size())); + } catch (final PageException e) { + throw new IOException(e); + } + } + @Override public void serialize(final DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(isModification(), stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java index af401f84aa777..4571bdb1531d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java @@ -156,7 +156,6 @@ private void processTimeChunkOrNonAlignedChunk(TsFileSequenceReader reader, byte long chunkOffset = reader.position(); timeChunkIndexOfCurrentValueColumn = pageIndex2TimesList.size(); consumeAllAlignedChunkData(chunkOffset, pageIndex2ChunkData); - handleModification(offset2Deletions, chunkOffset); ChunkHeader header = reader.readChunkHeader(marker); String measurementId = header.getMeasurementID(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuota.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuota.java index e4f1170ddb36c..3da4db4b4083a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuota.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuota.java @@ -116,9 +116,7 @@ protected void updateEstimateConsumeQuota(int numWrites, int numReads, Statement case BATCH_INSERT: // InsertTabletStatement InsertTabletStatement insertTabletStatement = (InsertTabletStatement) s; - for (BitMap bitMap : insertTabletStatement.getBitMaps()) { - avgSize += bitMap.getSize(); - } + avgSize += calculationWrite(insertTabletStatement.getBitMaps()); break; case BATCH_INSERT_ONE_DEVICE: // InsertRowsOfOneDeviceStatement @@ -151,10 +149,12 @@ protected void updateEstimateConsumeQuota(int numWrites, int numReads, Statement for (int i = 0; i < insertMultiTabletsStatement.getInsertTabletStatementList().size(); i++) { - for (BitMap bitMap : - insertMultiTabletsStatement.getInsertTabletStatementList().get(i).getBitMaps()) { - avgSize += bitMap.getSize(); - } + avgSize += + calculationWrite( + insertMultiTabletsStatement + .getInsertTabletStatementList() + .get(i) + .getBitMaps()); } } break; @@ -178,6 +178,20 @@ private long calculationWrite(Object[] values) { return size; } + private long calculationWrite(BitMap[] bitMaps) { + if (bitMaps == null) { + return 0; + } + + long size = 0; + for (BitMap bitMap : bitMaps) { + if (bitMap != null) { + size += bitMap.getSize(); + } + } + return size; + } + private long estimateConsume(int numReqs, long avgSize) { if (numReqs > 0) { return avgSize * numReqs; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/stage/SubscriptionTaskSinkStage.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/stage/SubscriptionTaskSinkStage.java index 73fca57a1fd69..45c6d6f86cfc8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/stage/SubscriptionTaskSinkStage.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/stage/SubscriptionTaskSinkStage.java @@ -41,7 +41,7 @@ public SubscriptionTaskSinkStage( @Override protected void registerSubtask() { - this.connectorSubtaskId = + this.sinkSubtaskId = SubscriptionSinkSubtaskManager.instance() .register( executor.get(), @@ -56,22 +56,21 @@ public void createSubtask() throws PipeException { @Override public void startSubtask() throws PipeException { - SubscriptionSinkSubtaskManager.instance().start(connectorSubtaskId); + SubscriptionSinkSubtaskManager.instance().start(sinkSubtaskId); } @Override public void stopSubtask() throws PipeException { - SubscriptionSinkSubtaskManager.instance().stop(connectorSubtaskId); + SubscriptionSinkSubtaskManager.instance().stop(sinkSubtaskId); } @Override public void dropSubtask() throws PipeException { SubscriptionSinkSubtaskManager.instance() - .deregister(pipeName, creationTime, regionId, connectorSubtaskId); + .deregister(pipeName, creationTime, regionId, sinkSubtaskId); } public UnboundedBlockingPendingQueue getPipeSinkPendingQueue() { - return SubscriptionSinkSubtaskManager.instance() - .getPipeConnectorPendingQueue(connectorSubtaskId); + return SubscriptionSinkSubtaskManager.instance().getPipeConnectorPendingQueue(sinkSubtaskId); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionSinkSubtaskLifeCycle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionSinkSubtaskLifeCycle.java index 98163697374da..c24098f44fca6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionSinkSubtaskLifeCycle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionSinkSubtaskLifeCycle.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.subscription.task.subtask; import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.db.pipe.agent.task.execution.PipeSinkSubtaskExecutor; import org.apache.iotdb.db.pipe.agent.task.subtask.sink.PipeSinkSubtask; import org.apache.iotdb.db.pipe.agent.task.subtask.sink.PipeSinkSubtaskLifeCycle; @@ -63,7 +64,7 @@ public synchronized void register() { } @Override - public synchronized boolean deregister(final String pipeNameToDeregister, int regionId) { + public synchronized boolean deregister(final CommitterKey committerKey) { if (registeredTaskCount <= 0) { throw new IllegalStateException("registeredTaskCount <= 0"); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionSinkSubtaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionSinkSubtaskManager.java index 07def3ff4d39a..1c081888b3577 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionSinkSubtaskManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionSinkSubtaskManager.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.agent.task.progress.PipeEventCommitManager; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; @@ -167,7 +168,11 @@ public synchronized void deregister( final PipeSinkSubtaskLifeCycle lifeCycle = attributeSortedString2SubtaskLifeCycleMap.get(attributeSortedString); - if (lifeCycle.deregister(pipeName, regionId)) { + + final CommitterKey committerKey = + PipeEventCommitManager.getInstance().getCommitterKey(pipeName, creationTime, regionId); + + if (lifeCycle.deregister(committerKey)) { attributeSortedString2SubtaskLifeCycleMap.remove(attributeSortedString); } diff --git a/iotdb-client/client-cpp/src/test/main.cpp b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/BitMapUtils.java similarity index 53% rename from iotdb-client/client-cpp/src/test/main.cpp rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/BitMapUtils.java index 82b3e60ddb593..ba30c8847b4b5 100644 --- a/iotdb-client/client-cpp/src/test/main.cpp +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/BitMapUtils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an @@ -17,26 +17,30 @@ * under the License. */ -#define CATCH_CONFIG_MAIN +package org.apache.iotdb.db.utils; -#include -#include "Session.h" +import org.apache.tsfile.utils.BitMap; -std::shared_ptr session = std::make_shared("127.0.0.1", 6667, "root", "root"); +import java.util.Objects; -struct SessionListener : Catch::TestEventListenerBase { +public final class BitMapUtils { - using TestEventListenerBase::TestEventListenerBase; + private BitMapUtils() {} - void testCaseStarting(Catch::TestCaseInfo const &testInfo) override { - // Perform some setup before a test case is run - session->open(false); + public static BitMap[] compactBitMaps(final BitMap[] bitMaps, final int rowCount) { + if (Objects.isNull(bitMaps)) { + return null; } - void testCaseEnded(Catch::TestCaseStats const &testCaseStats) override { - // Tear-down after a test case is run - session->close(); + boolean hasMarkedBitMap = false; + for (int i = 0; i < bitMaps.length; ++i) { + if (Objects.nonNull(bitMaps[i]) && bitMaps[i].isAllUnmarked()) { + bitMaps[i] = null; + } + if (Objects.nonNull(bitMaps[i])) { + hasMarkedBitMap = true; + } } -}; - -CATCH_REGISTER_LISTENER( SessionListener ) \ No newline at end of file + return hasMarkedBitMap ? bitMaps : null; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java index 963cecceb104c..765f19c31bf0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java @@ -46,6 +46,7 @@ import java.util.concurrent.ExecutionException; import static org.apache.iotdb.commons.utils.StatusUtils.needRetry; +import static org.apache.iotdb.db.protocol.thrift.OperationType.DISPATCH_FRAGMENT_INSTANCE; public class ErrorHandlingUtils { @@ -60,10 +61,11 @@ private ErrorHandlingUtils() {} "The read statement is not allowed in batch: "; private static final String ERROR_OPERATION_LOG = "Status code: {}, operation: {} failed"; + private static final String EXCEPTION_PATTERN = "[%s] Exception occurred: %s failed. "; public static TSStatus onNpeOrUnexpectedException( Exception e, String operation, TSStatusCode statusCode) { - String message = String.format("[%s] Exception occurred: %s failed. ", statusCode, operation); + String message = String.format(EXCEPTION_PATTERN, statusCode, operation); if (e instanceof IOException || e instanceof NullPointerException) { LOGGER.error(ERROR_OPERATION_LOG, statusCode, operation, e); } else { @@ -87,6 +89,23 @@ public static TSStatus onNpeOrUnexpectedException( return onNpeOrUnexpectedException(e, operation.getName(), statusCode); } + public static Throwable getRootCause(Throwable e) { + while (e.getCause() != null) { + e = e.getCause(); + } + return e; + } + + public static TSStatus onThriftFrameOversizeException(Throwable t) { + TSStatus status = + new TSStatus(TSStatusCode.THRIFT_FRAME_OVERSIZE.getStatusCode()).setNeedRetry(false); + String message = + String.format(EXCEPTION_PATTERN, status, DISPATCH_FRAGMENT_INSTANCE) + + getRootCause(t).getMessage(); + LOGGER.warn(message); + return status.setMessage(message); + } + public static TSStatus onQueryException(Exception e, String operation, TSStatusCode statusCode) { TSStatus status = tryCatchQueryException(e); if (status != null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java index a9f28e24e0a35..982ac2a3e2223 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java @@ -79,7 +79,11 @@ public static void modifyChunkMetaData( if (range.contains(metaData.getStartTime(), metaData.getEndTime())) { return true; } else { - if (range.overlaps(new TimeRange(metaData.getStartTime(), metaData.getEndTime()))) { + if (overlap( + metaData.getStartTime(), + metaData.getEndTime(), + range.getMin(), + range.getMax())) { metaData.setModified(true); } } @@ -135,9 +139,11 @@ public static void modifyAlignedChunkMetaData( currentRemoved = true; break; } else { - if (range.overlaps( - new TimeRange( - valueChunkMetadata.getStartTime(), valueChunkMetadata.getEndTime()))) { + if (overlap( + valueChunkMetadata.getStartTime(), + valueChunkMetadata.getEndTime(), + range.getMin(), + range.getMax())) { valueChunkMetadata.setModified(true); modified = true; } @@ -209,6 +215,11 @@ public static boolean isPointDeleted(long timestamp, List deletionLis return isPointDeleted(timestamp, deletionList, deleteCursor); } + // Both ranges are closed. + public static boolean overlap(long startA, long endA, long startB, long endB) { + return endB >= startA && startB <= endA; + } + /** * Check whether the device with start time and end time is completely deleted by mods or not. * There are some slight differences from that in {@link SettleSelectorImpl}. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java index dec3864353cc0..90e405ebdea91 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java @@ -30,15 +30,14 @@ import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.write.UnSupportedDataTypeException; -import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; import java.util.Optional; @@ -57,14 +56,13 @@ public static Pair convertTsBlockByFetchSize( // indicate whether it is a null int columnNumWithTime = columnNum * 2 + 1; DataOutputStream[] dataOutputStreams = new DataOutputStream[columnNumWithTime]; - ByteArrayOutputStream[] byteArrayOutputStreams = new ByteArrayOutputStream[columnNumWithTime]; + PublicBAOS[] byteArrayOutputStreams = new PublicBAOS[columnNumWithTime]; for (int i = 0; i < columnNumWithTime; i++) { - byteArrayOutputStreams[i] = new ByteArrayOutputStream(); + byteArrayOutputStreams[i] = new PublicBAOS(); dataOutputStreams[i] = new DataOutputStream(byteArrayOutputStreams[i]); } int rowCount = 0; - int[] valueOccupation = new int[columnNum]; // used to record a bitmap for every 8 points int[] bitmaps = new int[columnNum]; @@ -77,14 +75,7 @@ public static Pair convertTsBlockByFetchSize( TsBlock tsBlock = optionalTsBlock.get(); if (!tsBlock.isEmpty()) { int currentCount = tsBlock.getPositionCount(); - serializeTsBlock( - rowCount, - currentCount, - tsBlock, - columnNum, - dataOutputStreams, - valueOccupation, - bitmaps); + serializeTsBlock(rowCount, currentCount, tsBlock, columnNum, dataOutputStreams, bitmaps); rowCount += currentCount; } } @@ -93,9 +84,9 @@ public static Pair convertTsBlockByFetchSize( TSQueryDataSet tsQueryDataSet = new TSQueryDataSet(); - fillTimeColumn(rowCount, byteArrayOutputStreams, tsQueryDataSet); + fillTimeColumn(byteArrayOutputStreams, tsQueryDataSet); - fillValueColumnsAndBitMaps(rowCount, byteArrayOutputStreams, valueOccupation, tsQueryDataSet); + fillValueColumnsAndBitMaps(byteArrayOutputStreams, tsQueryDataSet); return new Pair<>(tsQueryDataSet, finished); } @@ -109,14 +100,13 @@ public static TSQueryDataSet convertTsBlockByFetchSize(List tsBlocks) int columnNum = 1; int columnNumWithTime = columnNum * 2 + 1; DataOutputStream[] dataOutputStreams = new DataOutputStream[columnNumWithTime]; - ByteArrayOutputStream[] byteArrayOutputStreams = new ByteArrayOutputStream[columnNumWithTime]; + PublicBAOS[] byteArrayOutputStreams = new PublicBAOS[columnNumWithTime]; for (int i = 0; i < columnNumWithTime; i++) { - byteArrayOutputStreams[i] = new ByteArrayOutputStream(); + byteArrayOutputStreams[i] = new PublicBAOS(); dataOutputStreams[i] = new DataOutputStream(byteArrayOutputStreams[i]); } int rowCount = 0; - int[] valueOccupation = new int[columnNum]; // used to record a bitmap for every 8 points int[] bitmaps = new int[columnNum]; @@ -150,7 +140,6 @@ public static TSQueryDataSet convertTsBlockByFetchSize(List tsBlocks) } else { bitmaps[k] = (bitmaps[k] << 1) | FLAG; dataOutputStream.writeInt(column.getInt(i)); - valueOccupation[k] += 4; } if (rowCount != 0 && rowCount % 8 == 0) { dataBitmapOutputStream.writeByte(bitmaps[k]); @@ -168,7 +157,6 @@ public static TSQueryDataSet convertTsBlockByFetchSize(List tsBlocks) } else { bitmaps[k] = (bitmaps[k] << 1) | FLAG; dataOutputStream.writeLong(column.getLong(i)); - valueOccupation[k] += 8; } if (rowCount != 0 && rowCount % 8 == 0) { dataBitmapOutputStream.writeByte(bitmaps[k]); @@ -185,7 +173,6 @@ public static TSQueryDataSet convertTsBlockByFetchSize(List tsBlocks) } else { bitmaps[k] = (bitmaps[k] << 1) | FLAG; dataOutputStream.writeFloat(column.getFloat(i)); - valueOccupation[k] += 4; } if (rowCount != 0 && rowCount % 8 == 0) { dataBitmapOutputStream.writeByte(bitmaps[k]); @@ -202,7 +189,6 @@ public static TSQueryDataSet convertTsBlockByFetchSize(List tsBlocks) } else { bitmaps[k] = (bitmaps[k] << 1) | FLAG; dataOutputStream.writeDouble(column.getDouble(i)); - valueOccupation[k] += 8; } if (rowCount != 0 && rowCount % 8 == 0) { dataBitmapOutputStream.writeByte(bitmaps[k]); @@ -219,7 +205,6 @@ public static TSQueryDataSet convertTsBlockByFetchSize(List tsBlocks) } else { bitmaps[k] = (bitmaps[k] << 1) | FLAG; dataOutputStream.writeBoolean(column.getBoolean(i)); - valueOccupation[k] += 1; } if (rowCount != 0 && rowCount % 8 == 0) { dataBitmapOutputStream.writeByte(bitmaps[k]); @@ -240,7 +225,6 @@ public static TSQueryDataSet convertTsBlockByFetchSize(List tsBlocks) Binary binary = column.getBinary(i); dataOutputStream.writeInt(binary.getLength()); dataOutputStream.write(binary.getValues()); - valueOccupation[k] = valueOccupation[k] + 4 + binary.getLength(); } if (rowCount != 0 && rowCount % 8 == 0) { dataBitmapOutputStream.writeByte(bitmaps[k]); @@ -268,27 +252,14 @@ public static TSQueryDataSet convertTsBlockByFetchSize(List tsBlocks) } // calculate the time buffer size - int timeOccupation = rowCount * 8; - ByteBuffer timeBuffer = ByteBuffer.allocate(timeOccupation); - timeBuffer.put(byteArrayOutputStreams[0].toByteArray()); - timeBuffer.flip(); - tsQueryDataSet.setTime(timeBuffer); + tsQueryDataSet.setTime(wrapBuffer(byteArrayOutputStreams[0])); // calculate the bitmap buffer size - int bitmapOccupation = (rowCount + 7) / 8; - - List bitmapList = new LinkedList<>(); - List valueList = new LinkedList<>(); + List bitmapList = new ArrayList<>(columnNum); + List valueList = new ArrayList<>(columnNum); for (int i = 1; i < byteArrayOutputStreams.length; i += 2) { - ByteBuffer valueBuffer = ByteBuffer.allocate(valueOccupation[(i - 1) / 2]); - valueBuffer.put(byteArrayOutputStreams[i].toByteArray()); - valueBuffer.flip(); - valueList.add(valueBuffer); - - ByteBuffer bitmapBuffer = ByteBuffer.allocate(bitmapOccupation); - bitmapBuffer.put(byteArrayOutputStreams[i + 1].toByteArray()); - bitmapBuffer.flip(); - bitmapList.add(bitmapBuffer); + valueList.add(wrapBuffer(byteArrayOutputStreams[i])); + bitmapList.add(wrapBuffer(byteArrayOutputStreams[i + 1])); } tsQueryDataSet.setBitmapList(bitmapList); tsQueryDataSet.setValueList(valueList); @@ -301,7 +272,6 @@ private static void serializeTsBlock( TsBlock tsBlock, int columnNum, DataOutputStream[] dataOutputStreams, - int[] valueOccupation, int[] bitmaps) throws IOException { // serialize time column @@ -321,67 +291,27 @@ private static void serializeTsBlock( switch (type) { case INT32: case DATE: - doWithInt32Column( - rowCount, - column, - bitmaps, - k, - dataOutputStream, - valueOccupation, - dataBitmapOutputStream); + doWithInt32Column(rowCount, column, bitmaps, k, dataOutputStream, dataBitmapOutputStream); break; case INT64: case TIMESTAMP: - doWithInt64Column( - rowCount, - column, - bitmaps, - k, - dataOutputStream, - valueOccupation, - dataBitmapOutputStream); + doWithInt64Column(rowCount, column, bitmaps, k, dataOutputStream, dataBitmapOutputStream); break; case FLOAT: - doWithFloatColumn( - rowCount, - column, - bitmaps, - k, - dataOutputStream, - valueOccupation, - dataBitmapOutputStream); + doWithFloatColumn(rowCount, column, bitmaps, k, dataOutputStream, dataBitmapOutputStream); break; case DOUBLE: doWithDoubleColumn( - rowCount, - column, - bitmaps, - k, - dataOutputStream, - valueOccupation, - dataBitmapOutputStream); + rowCount, column, bitmaps, k, dataOutputStream, dataBitmapOutputStream); break; case BOOLEAN: doWithBooleanColumn( - rowCount, - column, - bitmaps, - k, - dataOutputStream, - valueOccupation, - dataBitmapOutputStream); + rowCount, column, bitmaps, k, dataOutputStream, dataBitmapOutputStream); break; case TEXT: case BLOB: case STRING: - doWithTextColumn( - rowCount, - column, - bitmaps, - k, - dataOutputStream, - valueOccupation, - dataBitmapOutputStream); + doWithTextColumn(rowCount, column, bitmaps, k, dataOutputStream, dataBitmapOutputStream); break; default: throw new UnSupportedDataTypeException( @@ -396,7 +326,6 @@ private static void doWithInt32Column( int[] bitmaps, int columnIndex, DataOutputStream dataOutputStream, - int[] valueOccupation, DataOutputStream dataBitmapOutputStream) throws IOException { for (int i = 0, size = column.getPositionCount(); i < size; i++) { @@ -406,7 +335,6 @@ private static void doWithInt32Column( } else { bitmaps[columnIndex] = (bitmaps[columnIndex] << 1) | FLAG; dataOutputStream.writeInt(column.getInt(i)); - valueOccupation[columnIndex] += 4; } if (rowCount != 0 && rowCount % 8 == 0) { dataBitmapOutputStream.writeByte(bitmaps[columnIndex]); @@ -422,7 +350,6 @@ private static void doWithInt64Column( int[] bitmaps, int columnIndex, DataOutputStream dataOutputStream, - int[] valueOccupation, DataOutputStream dataBitmapOutputStream) throws IOException { for (int i = 0, size = column.getPositionCount(); i < size; i++) { @@ -432,7 +359,6 @@ private static void doWithInt64Column( } else { bitmaps[columnIndex] = (bitmaps[columnIndex] << 1) | FLAG; dataOutputStream.writeLong(column.getLong(i)); - valueOccupation[columnIndex] += 8; } if (rowCount != 0 && rowCount % 8 == 0) { dataBitmapOutputStream.writeByte(bitmaps[columnIndex]); @@ -448,7 +374,6 @@ private static void doWithFloatColumn( int[] bitmaps, int columnIndex, DataOutputStream dataOutputStream, - int[] valueOccupation, DataOutputStream dataBitmapOutputStream) throws IOException { for (int i = 0, size = column.getPositionCount(); i < size; i++) { @@ -458,7 +383,6 @@ private static void doWithFloatColumn( } else { bitmaps[columnIndex] = (bitmaps[columnIndex] << 1) | FLAG; dataOutputStream.writeFloat(column.getFloat(i)); - valueOccupation[columnIndex] += 4; } if (rowCount != 0 && rowCount % 8 == 0) { dataBitmapOutputStream.writeByte(bitmaps[columnIndex]); @@ -474,7 +398,6 @@ private static void doWithDoubleColumn( int[] bitmaps, int columnIndex, DataOutputStream dataOutputStream, - int[] valueOccupation, DataOutputStream dataBitmapOutputStream) throws IOException { for (int i = 0, size = column.getPositionCount(); i < size; i++) { @@ -484,7 +407,6 @@ private static void doWithDoubleColumn( } else { bitmaps[columnIndex] = (bitmaps[columnIndex] << 1) | FLAG; dataOutputStream.writeDouble(column.getDouble(i)); - valueOccupation[columnIndex] += 8; } if (rowCount != 0 && rowCount % 8 == 0) { dataBitmapOutputStream.writeByte(bitmaps[columnIndex]); @@ -500,7 +422,6 @@ private static void doWithBooleanColumn( int[] bitmaps, int columnIndex, DataOutputStream dataOutputStream, - int[] valueOccupation, DataOutputStream dataBitmapOutputStream) throws IOException { for (int i = 0, size = column.getPositionCount(); i < size; i++) { @@ -510,7 +431,6 @@ private static void doWithBooleanColumn( } else { bitmaps[columnIndex] = (bitmaps[columnIndex] << 1) | FLAG; dataOutputStream.writeBoolean(column.getBoolean(i)); - valueOccupation[columnIndex] += 1; } if (rowCount != 0 && rowCount % 8 == 0) { dataBitmapOutputStream.writeByte(bitmaps[columnIndex]); @@ -526,7 +446,6 @@ private static void doWithTextColumn( int[] bitmaps, int columnIndex, DataOutputStream dataOutputStream, - int[] valueOccupation, DataOutputStream dataBitmapOutputStream) throws IOException { for (int i = 0, size = column.getPositionCount(); i < size; i++) { @@ -538,7 +457,6 @@ private static void doWithTextColumn( Binary binary = column.getBinary(i); dataOutputStream.writeInt(binary.getLength()); dataOutputStream.write(binary.getValues()); - valueOccupation[columnIndex] = valueOccupation[columnIndex] + 4 + binary.getLength(); } if (rowCount != 0 && rowCount % 8 == 0) { dataBitmapOutputStream.writeByte(bitmaps[columnIndex]); @@ -562,40 +480,27 @@ private static void fillRemainingBitMap( } private static void fillTimeColumn( - int rowCount, ByteArrayOutputStream[] byteArrayOutputStreams, TSQueryDataSet tsQueryDataSet) { - // calculate the time buffer size - int timeOccupation = rowCount * 8; - ByteBuffer timeBuffer = ByteBuffer.allocate(timeOccupation); - timeBuffer.put(byteArrayOutputStreams[0].toByteArray()); - timeBuffer.flip(); - tsQueryDataSet.setTime(timeBuffer); + PublicBAOS[] byteArrayOutputStreams, TSQueryDataSet tsQueryDataSet) { + tsQueryDataSet.setTime(wrapBuffer(byteArrayOutputStreams[0])); } private static void fillValueColumnsAndBitMaps( - int rowCount, - ByteArrayOutputStream[] byteArrayOutputStreams, - int[] valueOccupation, - TSQueryDataSet tsQueryDataSet) { - // calculate the bitmap buffer size - int bitmapOccupation = (rowCount + 7) / 8; - - List bitmapList = new LinkedList<>(); - List valueList = new LinkedList<>(); + PublicBAOS[] byteArrayOutputStreams, TSQueryDataSet tsQueryDataSet) { + int columnNum = byteArrayOutputStreams.length / 2; + List bitmapList = new ArrayList<>(columnNum); + List valueList = new ArrayList<>(columnNum); for (int i = 1; i < byteArrayOutputStreams.length; i += 2) { - ByteBuffer valueBuffer = ByteBuffer.allocate(valueOccupation[(i - 1) / 2]); - valueBuffer.put(byteArrayOutputStreams[i].toByteArray()); - valueBuffer.flip(); - valueList.add(valueBuffer); - - ByteBuffer bitmapBuffer = ByteBuffer.allocate(bitmapOccupation); - bitmapBuffer.put(byteArrayOutputStreams[i + 1].toByteArray()); - bitmapBuffer.flip(); - bitmapList.add(bitmapBuffer); + valueList.add(wrapBuffer(byteArrayOutputStreams[i])); + bitmapList.add(wrapBuffer(byteArrayOutputStreams[i + 1])); } tsQueryDataSet.setBitmapList(bitmapList); tsQueryDataSet.setValueList(valueList); } + private static ByteBuffer wrapBuffer(PublicBAOS outputStream) { + return ByteBuffer.wrap(outputStream.getBuf(), 0, outputStream.size()); + } + /** * To fetch required amounts of data and combine them through List * @@ -654,10 +559,8 @@ public static Optional readBitMapsFromBuffer(ByteBuffer buffer, int co for (int i = 0; i < columns; i++) { boolean hasBitMap = BytesUtils.byteToBool(buffer.get()); if (hasBitMap) { - byte[] bytes = new byte[size / Byte.SIZE + 1]; - for (int j = 0; j < bytes.length; j++) { - bytes[j] = buffer.get(); - } + byte[] bytes = new byte[getBitmapByteSize(size)]; + buffer.get(bytes); bitMaps[i] = new BitMap(size, bytes); } } @@ -673,16 +576,18 @@ public static Optional readBitMapsFromStream( for (int i = 0; i < columns; i++) { boolean hasBitMap = BytesUtils.byteToBool(stream.readByte()); if (hasBitMap) { - byte[] bytes = new byte[size / Byte.SIZE + 1]; - for (int j = 0; j < bytes.length; j++) { - bytes[j] = stream.readByte(); - } + byte[] bytes = new byte[getBitmapByteSize(size)]; + stream.readFully(bytes); bitMaps[i] = new BitMap(size, bytes); } } return Optional.of(bitMaps); } + private static int getBitmapByteSize(int size) { + return size / Byte.SIZE + 1; + } + public static Object[] readTabletValuesFromBuffer( ByteBuffer buffer, List types, int columns, int size) { TSDataType[] dataTypes = new TSDataType[types.size()]; @@ -850,11 +755,7 @@ private static void parseTextColumn( for (int index = 0; index < size; index++) { int binarySize = stream.readInt(); byte[] binaryValue = new byte[binarySize]; - int actualReadSize = stream.read(binaryValue); - if (actualReadSize != binarySize) { - throw new IllegalStateException( - "Expect to read " + binarySize + " bytes, actually read " + actualReadSize + "bytes."); - } + stream.readFully(binaryValue); binaryValues[index] = new Binary(binaryValue); } values[columnIndex] = binaryValues; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 090b057498748..b99e784faadb0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -137,6 +137,16 @@ public TVList getTvListByColumnIndex(List columnIndex, List return alignedTvList; } + @Override + public synchronized AlignedTVList cloneForFlushSort() { + AlignedTVList cloneList = AlignedTVList.newAlignedList(new ArrayList<>(dataTypes)); + cloneAs(cloneList); + cloneList.memoryBinaryChunkSize = this.memoryBinaryChunkSize; + cloneList.values = this.values; + cloneList.bitMaps = this.bitMaps; + return cloneList; + } + @Override public synchronized AlignedTVList clone() { AlignedTVList cloneList = AlignedTVList.newAlignedList(new ArrayList<>(dataTypes)); @@ -836,8 +846,9 @@ public TSDataType getDataType() { } @Override - public synchronized long calculateRamSize() { - return timestamps.size() * alignedTvListArrayMemCost(); + public synchronized RamInfo calculateRamSize() { + return new RamInfo( + timestamps.size(), alignedTvListArrayMemCost(), rowCount, new ArrayList<>(dataTypes)); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java index dc4ff5529d45b..b274bc8ef6586 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BinaryTVList.java @@ -63,6 +63,15 @@ public static BinaryTVList newList() { } } + @Override + public synchronized TVList cloneForFlushSort() { + BinaryTVList cloneList = BinaryTVList.newList(); + cloneAs(cloneList); + cloneList.bitMap = this.bitMap; + cloneList.values = this.values; + return cloneList; + } + @Override public synchronized BinaryTVList clone() { BinaryTVList cloneList = BinaryTVList.newList(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java index b8eb0e508bfe8..5e3461acb2c8e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/BooleanTVList.java @@ -62,6 +62,15 @@ public static BooleanTVList newList() { } } + @Override + public synchronized TVList cloneForFlushSort() { + BooleanTVList cloneList = BooleanTVList.newList(); + cloneAs(cloneList); + cloneList.bitMap = this.bitMap; + cloneList.values = this.values; + return cloneList; + } + @Override public synchronized BooleanTVList clone() { BooleanTVList cloneList = BooleanTVList.newList(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java index f61995ef0628a..753ca2020a89f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/DoubleTVList.java @@ -63,6 +63,15 @@ public static DoubleTVList newList() { } } + @Override + public synchronized TVList cloneForFlushSort() { + DoubleTVList cloneList = DoubleTVList.newList(); + cloneAs(cloneList); + cloneList.bitMap = this.bitMap; + cloneList.values = this.values; + return cloneList; + } + @Override public synchronized DoubleTVList clone() { DoubleTVList cloneList = DoubleTVList.newList(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java index 3623fa49a3efe..517dc208211fb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/FloatTVList.java @@ -63,6 +63,15 @@ public static FloatTVList newList() { } } + @Override + public synchronized TVList cloneForFlushSort() { + FloatTVList cloneList = FloatTVList.newList(); + cloneAs(cloneList); + cloneList.bitMap = this.bitMap; + cloneList.values = this.values; + return cloneList; + } + @Override public synchronized FloatTVList clone() { FloatTVList cloneList = FloatTVList.newList(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java index 758cd64053bc5..8593400fa79c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/IntTVList.java @@ -62,6 +62,15 @@ public static IntTVList newList() { } } + @Override + public synchronized TVList cloneForFlushSort() { + IntTVList cloneList = IntTVList.newList(); + cloneAs(cloneList); + cloneList.bitMap = this.bitMap; + cloneList.values = this.values; + return cloneList; + } + @Override public synchronized IntTVList clone() { IntTVList cloneList = IntTVList.newList(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java index 7b4bd8d82d264..544d2cb7dd0eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/LongTVList.java @@ -62,6 +62,15 @@ public static LongTVList newList() { } } + @Override + public synchronized TVList cloneForFlushSort() { + LongTVList cloneList = LongTVList.newList(); + cloneAs(cloneList); + cloneList.bitMap = this.bitMap; + cloneList.values = this.values; + return cloneList; + } + @Override public synchronized LongTVList clone() { LongTVList cloneList = LongTVList.newList(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index 0f4ab88441aa6..d21ecc83e0f1b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager; import org.apache.iotdb.db.utils.MathUtils; +import com.google.common.collect.ImmutableList; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; @@ -59,6 +60,42 @@ public abstract class TVList implements WALEntryValue { protected static final String ERR_DATATYPE_NOT_CONSISTENT = "DataType not consistent"; + + public static class RamInfo { + private final int timestampsSize; + private final long arrayMemCost; + private final int rowCount; + private final List dataTypes; + + public RamInfo( + int timestampCount, long arrayMemCost, int rowCount, List dataTypes) { + this.timestampsSize = timestampCount; + this.rowCount = rowCount; + this.arrayMemCost = arrayMemCost; + this.dataTypes = dataTypes; + } + + public long getRamSize() { + return timestampsSize * arrayMemCost; + } + + public int getTimestampsSize() { + return timestampsSize; + } + + public int getRowCount() { + return rowCount; + } + + public long getArrayMemCost() { + return arrayMemCost; + } + + public List getDataTypes() { + return dataTypes; + } + } + // list of timestamp array, add 1 when expanded -> data point timestamp array // index relation: arrayIndex -> elementIndex protected List timestamps; @@ -156,8 +193,9 @@ public static long tvListArrayMemCost(TSDataType type) { return size; } - public synchronized long calculateRamSize() { - return timestamps.size() * tvListArrayMemCost(); + public synchronized RamInfo calculateRamSize() { + return new RamInfo( + timestamps.size(), tvListArrayMemCost(), rowCount, ImmutableList.of(getDataType())); } public synchronized boolean isSorted() { @@ -510,6 +548,8 @@ public long getVersion() { protected abstract void expandValues(); + public abstract TVList cloneForFlushSort(); + @Override public abstract TVList clone(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/AuthorityCheckerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/AuthorityCheckerTest.java index ca730c377b763..2861059808a5f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/AuthorityCheckerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/auth/AuthorityCheckerTest.java @@ -20,18 +20,36 @@ package org.apache.iotdb.db.auth; import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.auth.entity.User; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountLevelTimeSeriesStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountTimeSeriesStatement; +import org.apache.iotdb.rpc.TSStatusCode; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import java.util.Arrays; +import java.util.Collections; public class AuthorityCheckerTest { + @Before + public void setup() { + AuthorityChecker.getAuthorityFetcher().getAuthorCache().invalidAllCache(); + } + + @After + public void teardown() { + AuthorityChecker.getAuthorityFetcher().getAuthorCache().invalidAllCache(); + } + @Test public void testLogReduce() throws IllegalPathException { final CommonConfig config = CommonDescriptor.getInstance().getConfig(); @@ -48,4 +66,60 @@ public void testLogReduce() throws IllegalPathException { .getMessage()); config.setPathLogMaxSize(oldSize); } + + @Test + public void testCountTimeSeriesExplicitSystemDatabasePermission() throws Exception { + User user = new User("user1", "password"); + AuthorityChecker.getAuthorityFetcher().getAuthorCache().putUserCache(user.getName(), user); + + CountTimeSeriesStatement systemStatement = + new CountTimeSeriesStatement(new PartialPath("root.__system.**")); + Assert.assertEquals( + TSStatusCode.NO_PERMISSION.getStatusCode(), + systemStatement.checkPermissionBeforeProcess(user.getName()).getCode()); + + user.addSysPrivilege(PrivilegeType.MAINTAIN.ordinal()); + systemStatement = new CountTimeSeriesStatement(new PartialPath("root.__system.**")); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + systemStatement.checkPermissionBeforeProcess(user.getName()).getCode()); + Assert.assertEquals( + Collections.singletonList(new PartialPath("root.__system.**")), + systemStatement.getAuthorityScope().getAllPathPatterns()); + Assert.assertTrue(systemStatement.isCanSeeSystemDB()); + + CountLevelTimeSeriesStatement systemLevelStatement = + new CountLevelTimeSeriesStatement(new PartialPath("root.__system.**"), 1); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + systemLevelStatement.checkPermissionBeforeProcess(user.getName()).getCode()); + Assert.assertEquals( + Collections.singletonList(new PartialPath("root.__system.**")), + systemLevelStatement.getAuthorityScope().getAllPathPatterns()); + Assert.assertTrue(systemLevelStatement.isCanSeeSystemDB()); + } + + @Test + public void testCountTimeSeriesImplicitSystemDatabasePermission() throws Exception { + User user = new User("user2", "password"); + AuthorityChecker.getAuthorityFetcher().getAuthorCache().putUserCache(user.getName(), user); + + CountTimeSeriesStatement statement = new CountTimeSeriesStatement(new PartialPath("root.**")); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + statement.checkPermissionBeforeProcess(user.getName()).getCode()); + Assert.assertFalse(statement.isCanSeeSystemDB()); + + user.addSysPrivilege(PrivilegeType.MAINTAIN.ordinal()); + statement = new CountTimeSeriesStatement(new PartialPath("root.**")); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + statement.checkPermissionBeforeProcess(user.getName()).getCode()); + Assert.assertTrue(statement.isCanSeeSystemDB()); + Assert.assertTrue( + statement + .getAuthorityScope() + .getAllPathPatterns() + .contains(new PartialPath("root.__system.**"))); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java index 144f2396ef7fd..bf15d216e67cb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java @@ -267,6 +267,43 @@ public void testUpdateLastCache() throws IllegalPathException { Assert.assertEquals(0, dataNodeSchemaCache.getDeviceSchemaCache().getMemoryUsage()); } + @Test + public void testInvalidateLastCacheByWildcardDevicePath() throws IllegalPathException { + final MeasurementSchema s0 = new MeasurementSchema("s0", TSDataType.INT32); + final PartialPath device0 = new PartialPath("root.sg1.d1"); + final PartialPath device1 = new PartialPath("root.sg2.d1"); + final MeasurementPath path0 = new MeasurementPath(device0.concatNode("s0"), s0); + final MeasurementPath path1 = new MeasurementPath(device1.concatNode("s0"), s0); + final TimeValuePair tv0 = new TimeValuePair(0L, new TsPrimitiveType.TsInt(0)); + + updateLastCache("root.sg1", device0, path0, s0, tv0); + updateLastCache("root.sg2", device1, path1, s0, tv0); + + Assert.assertEquals(tv0, dataNodeSchemaCache.getLastCache(path0)); + Assert.assertEquals(tv0, dataNodeSchemaCache.getLastCache(path1)); + + dataNodeSchemaCache.invalidateLastCache(new MeasurementPath("root.sg1.*.s0")); + + Assert.assertNull(dataNodeSchemaCache.getLastCache(path0)); + Assert.assertEquals(tv0, dataNodeSchemaCache.getLastCache(path1)); + } + + private void updateLastCache( + final String database, + final PartialPath devicePath, + final MeasurementPath path, + final MeasurementSchema schema, + final TimeValuePair timeValuePair) { + dataNodeSchemaCache.declareLastCache(database, path); + dataNodeSchemaCache.updateLastCacheIfExists( + database, + devicePath, + new String[] {path.getMeasurement()}, + new TimeValuePair[] {timeValuePair}, + false, + new MeasurementSchema[] {schema}); + } + @Test public void testPut() throws Exception { final ClusterSchemaTree clusterSchemaTree = new ClusterSchemaTree(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/mtree/lock/StampedWriterPreferredLockTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/mtree/lock/StampedWriterPreferredLockTest.java index 2c9fdbdfd3d54..b559aaf81529a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/mtree/lock/StampedWriterPreferredLockTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/mtree/lock/StampedWriterPreferredLockTest.java @@ -25,6 +25,7 @@ import org.junit.Assert; import org.junit.Test; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -143,22 +144,41 @@ public void testThreadReadLock() { } @Test - public void testAcquireReadLockWhileWriting() { + public void testAcquireReadLockWhileWriting() throws InterruptedException { StampedWriterPreferredLock lock = new StampedWriterPreferredLock(); lock.writeLock(); AtomicInteger counter = new AtomicInteger(); - new Thread( + CountDownLatch readerFinished = new CountDownLatch(1); + Thread readerThread = + new Thread( () -> { lock.threadReadLock(); - counter.incrementAndGet(); - lock.threadReadUnlock(); - }) - .start(); - // block reader util writer release write lock - Assert.assertEquals(0, counter.get()); - lock.writeUnlock(); - Awaitility.await().atMost(2, TimeUnit.SECONDS).until(() -> counter.get() == 1); - Assert.assertEquals(1, counter.get()); + try { + counter.incrementAndGet(); + } finally { + lock.threadReadUnlock(); + readerFinished.countDown(); + } + }); + readerThread.setDaemon(true); + readerThread.start(); + + boolean writeLocked = true; + try { + // block reader until writer release write lock + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .until(() -> readerThread.getState() == Thread.State.WAITING); + Assert.assertEquals(0, counter.get()); + lock.writeUnlock(); + writeLocked = false; + Assert.assertTrue(readerFinished.await(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, counter.get()); + } finally { + if (writeLocked) { + lock.writeUnlock(); + } + } } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java index 667ec4bc9b47b..913b5b151e8c1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java @@ -1185,6 +1185,28 @@ public void testGetMatchedDevicesWithSpecialPattern() throws Exception { Assert.assertEquals(expectedPathList, actualPathList); } + @Test + public void testShowTimeseriesWildcardSuffixWithNestedAndLeafDevices() throws Exception { + final ISchemaRegion schemaRegion = getSchemaRegion("root.test", 0); + + SchemaRegionTestUtil.createSimpleTimeSeriesByList( + schemaRegion, + Arrays.asList( + "root.test.d1.s1", "root.test.d1.a.d2.s1", "root.test.d3.s1", "root.test.d4.s2")); + + final List result = + SchemaRegionTestUtil.showTimeseries(schemaRegion, new PartialPath("root.test.**.s1")); + final Set expectedPathList = + new HashSet<>(Arrays.asList("root.test.d1.s1", "root.test.d1.a.d2.s1", "root.test.d3.s1")); + Assert.assertEquals(expectedPathList.size(), result.size()); + + final Set actualPathList = new HashSet<>(); + for (final ITimeSeriesSchemaInfo timeSeriesSchemaInfo : result) { + actualPathList.add(timeSeriesSchemaInfo.getFullPath()); + } + Assert.assertEquals(expectedPathList, actualPathList); + } + @Test public void testGetMatchedDevicesWithSpecialPattern2() throws Exception { final ISchemaRegion schemaRegion = getSchemaRegion("root.test", 0); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionManagementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionManagementTest.java index d027871121910..2d79a6a4766c8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionManagementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionManagementTest.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ISchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.SchemaRegionWritePlanFactory; @@ -40,6 +41,7 @@ import org.junit.Test; import java.io.File; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -229,4 +231,14 @@ public void testSnapshotPerformance() throws Exception { config.setSchemaRegionConsensusProtocolClass(schemaRegionConsensusProtocolClass); } } + + @Test + public void testSchemaRegionPlanType() throws Exception { + Assert.assertEquals( + SchemaRegionPlanType.values().length, + Arrays.stream(SchemaRegionPlanType.values()) + .map(SchemaRegionPlanType::getPlanType) + .distinct() + .count()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaStatisticsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaStatisticsTest.java index 04643a88c41e5..40aade6a63b95 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaStatisticsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaStatisticsTest.java @@ -75,24 +75,12 @@ public void testPBTreeMemoryStatistics() throws Exception { || testParams.getTestModeName().equals("PBTree-NonMemory")) { final IMNodeFactory nodeFactory = MNodeFactoryLoader.getInstance().getCachedMNodeIMNodeFactory(); - // wait release and flush task - Thread.sleep(6000); + ReleaseFlushMonitor.getInstance().forceFlushAndRelease(); // schemaRegion1 final IMNode sg1 = nodeFactory.createDatabaseMNode(null, "sg1"); sg1.setFullPath("root.sg1"); final long size1 = sg1.estimateSize(); - if (size1 != schemaRegion1.getSchemaRegionStatistics().getRegionMemoryUsage()) { - // There are two possibilities here in PartialMemory mode: - // 1. only the "sg1" node remains - // 2. the "sg1" node and the "n" node remain - Assert.assertEquals("PBTree-PartialMemory", testParams.getTestModeName()); - Assert.assertEquals( - size1 + nodeFactory.createDeviceMNode(sg1.getAsMNode(), "n").estimateSize(), - schemaRegion1.getSchemaRegionStatistics().getRegionMemoryUsage()); - ReleaseFlushMonitor.getInstance().forceFlushAndRelease(); - Assert.assertEquals( - size1, schemaRegion1.getSchemaRegionStatistics().getRegionMemoryUsage()); - } + Assert.assertEquals(size1, schemaRegion1.getSchemaRegionStatistics().getRegionMemoryUsage()); } Assert.assertEquals(0, schemaRegion1.getSchemaRegionStatistics().getSchemaRegionId()); checkPBTreeStatistics(engineStatistics); @@ -122,8 +110,7 @@ public void testMemoryStatistics2() throws Exception { final IMNodeFactory nodeFactory = MNodeFactoryLoader.getInstance().getCachedMNodeIMNodeFactory(); - // wait release and flush task - Thread.sleep(1000); + ReleaseFlushMonitor.getInstance().forceFlushAndRelease(); // schemaRegion1 final IMNode sg1 = nodeFactory.createDatabaseMNode(null, "sg1"); sg1.setFullPath("root.sg1"); @@ -231,36 +218,12 @@ public void testMemoryStatistics() throws Exception { || testParams.getTestModeName().equals("PBTree-NonMemory")) { final IMNodeFactory nodeFactory = MNodeFactoryLoader.getInstance().getCachedMNodeIMNodeFactory(); - // wait release and flush task - Thread.sleep(1000); + ReleaseFlushMonitor.getInstance().forceFlushAndRelease(); // schemaRegion1 final IMNode sg1 = nodeFactory.createDatabaseDeviceMNode(null, "sg1"); sg1.setFullPath("root.sg1"); final long size1 = sg1.estimateSize(); - if (sg1.estimateSize() != schemaRegion1.getSchemaRegionStatistics().getRegionMemoryUsage()) { - // "d0" or "d1" node may remain in PartialMemory mode - Assert.assertEquals("PBTree-PartialMemory", testParams.getTestModeName()); - final long d0ExistSize = - size1 - + nodeFactory - .createMeasurementMNode( - sg1.getAsDeviceMNode(), - "d0", - new MeasurementSchema( - "d0", TSDataType.INT64, TSEncoding.PLAIN, CompressionType.SNAPPY), - null) - .estimateSize(); - final long d1ExistSize = - size1 + nodeFactory.createInternalMNode(sg1.getAsMNode(), "d1").estimateSize(); - Assert.assertTrue( - d0ExistSize == schemaRegion1.getSchemaRegionStatistics().getRegionMemoryUsage() - || d1ExistSize == schemaRegion1.getSchemaRegionStatistics().getRegionMemoryUsage()); - ReleaseFlushMonitor.getInstance().forceFlushAndRelease(); - // wait release and flush task - Thread.sleep(1000); - Assert.assertEquals( - size1, schemaRegion1.getSchemaRegionStatistics().getRegionMemoryUsage()); - } + Assert.assertEquals(size1, schemaRegion1.getSchemaRegionStatistics().getRegionMemoryUsage()); // schemaRegion2 final IMNode sg2 = nodeFactory.createDatabaseMNode(null, "sg2"); sg2.setFullPath("root.sg2"); @@ -420,7 +383,11 @@ public void testPBTreeNodeStatistics() throws Exception { schemaRegion1.deleteTimeseriesInBlackList(patternTree); schemaRegion2.deleteTimeseriesInBlackList(patternTree); - Thread.sleep(1000); + if (testParams.getCachedMNodeSize() <= 3) { + ReleaseFlushMonitor.getInstance().forceFlushAndRelease(); + } else { + Thread.sleep(1000); + } final CachedSchemaRegionStatistics cachedRegionStatistics1 = schemaRegion1.getSchemaRegionStatistics().getAsCachedSchemaRegionStatistics(); final CachedSchemaRegionStatistics cachedRegionStatistics2 = @@ -433,13 +400,7 @@ public void testPBTreeNodeStatistics() throws Exception { Assert.assertEquals(4, cachedRegionStatistics2.getUnpinnedMNodeNum()); } else { Assert.assertEquals(1, cachedRegionStatistics1.getPinnedMNodeNum()); - if (0 != cachedRegionStatistics1.getUnpinnedMNodeNum()) { - // "d0" may remain in PartialMemory mode - Assert.assertEquals("PBTree-PartialMemory", testParams.getTestModeName()); - ReleaseFlushMonitor.getInstance().forceFlushAndRelease(); - Thread.sleep(1000); - Assert.assertEquals(0, cachedRegionStatistics1.getUnpinnedMNodeNum()); - } + Assert.assertEquals(0, cachedRegionStatistics1.getUnpinnedMNodeNum()); Assert.assertEquals(1, cachedRegionStatistics2.getPinnedMNodeNum()); Assert.assertEquals(0, cachedRegionStatistics2.getUnpinnedMNodeNum()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollectorTest.java new file mode 100644 index 0000000000000..d54db821007f8 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollectorTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.agent.task.connection; + +import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; +import org.apache.iotdb.db.pipe.agent.task.subtask.sink.PipeRealtimePriorityBlockingQueue; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter; +import org.apache.iotdb.pipe.api.event.Event; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +public class PipeEventCollectorTest { + + @Test + public void testCollectorDoesNotOfferEventsOfDroppedPipeToUnboundedPendingQueue() { + verifyCollectorDoesNotOfferEventsOfDroppedPipe( + new UnboundedBlockingPendingQueue<>(new PipeDataRegionEventCounter())); + } + + @Test + public void testCollectorDoesNotOfferEventsOfDroppedPipeToRealtimePendingQueue() { + verifyCollectorDoesNotOfferEventsOfDroppedPipe(new PipeRealtimePriorityBlockingQueue()); + } + + private void verifyCollectorDoesNotOfferEventsOfDroppedPipe( + final UnboundedBlockingPendingQueue pendingQueue) { + pendingQueue.discardEventsOfPipe("pipe", 1L, 1); + + final PipeEventCollector droppedPipeCollector = + new PipeEventCollector(pendingQueue, 1L, 1, false, false); + final PipeRawTabletInsertionEvent droppedPipeEvent = + createPipeRawTabletInsertionEvent("pipe", 1L); + droppedPipeCollector.collect(droppedPipeEvent); + + Assert.assertTrue(droppedPipeEvent.isReleased()); + Assert.assertEquals(0, pendingQueue.size()); + + final PipeEventCollector recreatedPipeCollector = + new PipeEventCollector(pendingQueue, 2L, 1, false, false); + final PipeRawTabletInsertionEvent recreatedPipeEvent = + createPipeRawTabletInsertionEvent("pipe", 2L); + recreatedPipeCollector.collect(recreatedPipeEvent); + + Assert.assertFalse(recreatedPipeEvent.isReleased()); + Assert.assertEquals(1, pendingQueue.size()); + + pendingQueue.discardAllEvents(); + Assert.assertTrue(recreatedPipeEvent.isReleased()); + } + + private PipeRawTabletInsertionEvent createPipeRawTabletInsertionEvent( + final String pipeName, final long creationTime) { + final List schemaList = + Arrays.asList(new MeasurementSchema("s1", TSDataType.INT64)); + final Tablet tablet = new Tablet("root.db.d1", schemaList, 1); + tablet.addTimestamp(0, 1L); + tablet.addValue("s1", 0, 1L); + return new PipeRawTabletInsertionEvent( + tablet, false, pipeName, creationTime, null, null, false); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskTest.java new file mode 100644 index 0000000000000..2a15fb9ea181b --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/agent/task/subtask/sink/PipeSinkSubtaskTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.agent.task.subtask.sink; + +import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; +import org.apache.iotdb.commons.pipe.sink.protocol.PipeConnectorWithEventDiscard; +import org.apache.iotdb.pipe.api.PipeConnector; + +import org.junit.Test; +import org.mockito.Mockito; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.withSettings; + +public class PipeSinkSubtaskTest { + + @Test + public void testDiscardEventsOfPipeDelegatesToConnector() { + final PipeConnector connector = + mock( + PipeConnector.class, + withSettings().extraInterfaces(PipeConnectorWithEventDiscard.class)); + final UnboundedBlockingPendingQueue pendingQueue = mock(UnboundedBlockingPendingQueue.class); + + final PipeSinkSubtask subtask = + Mockito.spy( + new PipeSinkSubtask( + "PipeSinkSubtaskTest", + System.currentTimeMillis(), + "data_test", + 0, + (UnboundedBlockingPendingQueue) pendingQueue, + connector)); + + try { + final CommitterKey committerKey = new CommitterKey("pipe", 1L, 1, -1); + subtask.discardEventsOfPipe(committerKey); + + verify((PipeConnectorWithEventDiscard) connector).discardEventsOfPipe(committerKey); + } finally { + subtask.close(); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeRealtimeEventTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeRealtimeEventTest.java new file mode 100644 index 0000000000000..525951fa4e85e --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeRealtimeEventTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.event; + +import org.apache.iotdb.commons.consensus.index.ProgressIndex; +import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; +import org.apache.iotdb.commons.pipe.datastructure.pattern.PipePattern; +import org.apache.iotdb.commons.pipe.event.EnrichedEvent; +import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent; + +import org.junit.Assert; +import org.junit.Test; + +public class PipeRealtimeEventTest { + + @Test + public void testSkipReportOnCommitIsDelegatedToInnerEvent() { + final TestEnrichedEvent innerEvent = new TestEnrichedEvent(); + final PipeRealtimeEvent realtimeEvent = new PipeRealtimeEvent(innerEvent, null, null, null); + + Assert.assertTrue(innerEvent.isShouldReportOnCommit()); + Assert.assertTrue(realtimeEvent.isShouldReportOnCommit()); + + realtimeEvent.skipReportOnCommit(); + + Assert.assertFalse(innerEvent.isShouldReportOnCommit()); + Assert.assertFalse(realtimeEvent.isShouldReportOnCommit()); + } + + private static class TestEnrichedEvent extends EnrichedEvent { + + private TestEnrichedEvent() { + super(null, 0, null, null, Long.MIN_VALUE, Long.MAX_VALUE); + } + + @Override + public boolean internallyIncreaseResourceReferenceCount(final String holderMessage) { + return true; + } + + @Override + public boolean internallyDecreaseResourceReferenceCount(final String holderMessage) { + return true; + } + + @Override + public ProgressIndex getProgressIndex() { + return MinimumProgressIndex.INSTANCE; + } + + @Override + public EnrichedEvent shallowCopySelfAndBindPipeTaskMetaForProgressReport( + final String pipeName, + final long creationTime, + final PipeTaskMeta pipeTaskMeta, + final PipePattern pattern, + final long startTime, + final long endTime) { + return this; + } + + @Override + public boolean isGeneratedByPipe() { + return false; + } + + @Override + public boolean mayEventTimeOverlappedWithTimeRange() { + return true; + } + + @Override + public boolean mayEventPathsOverlappedWithPattern() { + return true; + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java index 4a7d77eab8d29..308c5458dc9b9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.pipe.datastructure.pattern.PrefixPipePattern; import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeTabletUtils; import org.apache.iotdb.db.pipe.event.common.tablet.TabletInsertionDataContainer; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; @@ -217,7 +218,7 @@ private void createTablet() { tabletForInsertRowNode.values = values; tabletForInsertRowNode.timestamps = new long[] {times[0]}; tabletForInsertRowNode.rowSize = 1; - tabletForInsertRowNode.bitMaps = bitMapsForInsertRowNode; + tabletForInsertRowNode.bitMaps = PipeTabletUtils.compactBitMaps(bitMapsForInsertRowNode, 1); // create tablet for insertTabletNode BitMap[] bitMapsForInsertTabletNode = new BitMap[schemas.length]; @@ -253,7 +254,8 @@ private void createTablet() { tabletForInsertTabletNode.values = values; tabletForInsertTabletNode.timestamps = times; tabletForInsertTabletNode.rowSize = times.length; - tabletForInsertTabletNode.bitMaps = bitMapsForInsertTabletNode; + tabletForInsertTabletNode.bitMaps = + PipeTabletUtils.compactBitMaps(bitMapsForInsertTabletNode, times.length); } @Test @@ -318,6 +320,36 @@ public void convertToAlignedTabletForTest() { Assert.assertTrue(isAligned4); } + @Test + public void convertToTabletSkipsUnnecessaryBitMapsForTest() throws Exception { + final BitMap[] bitMaps = new BitMap[schemas.length]; + bitMaps[0] = new BitMap(times.length); + bitMaps[1] = new BitMap(times.length); + bitMaps[1].mark(1); + + final InsertTabletNode nodeWithSparseColumn = + new InsertTabletNode( + new PlanNodeId("plannode bitmap"), + new PartialPath(deviceId), + false, + measurementIds, + dataTypes, + schemas, + times, + bitMaps, + insertTabletNode.getColumns(), + times.length); + + final Tablet tablet = + new TabletInsertionDataContainer(nodeWithSparseColumn, new PrefixPipePattern(pattern)) + .convertToTablet(); + + Assert.assertNotNull(tablet.bitMaps); + Assert.assertNull(tablet.bitMaps[0]); + Assert.assertNotNull(tablet.bitMaps[1]); + Assert.assertTrue(tablet.bitMaps[1].isMarked(1)); + } + @Test public void convertToTabletWithFilteredRowsForTest() { TabletInsertionDataContainer container1 = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java new file mode 100644 index 0000000000000..fd824d2d25712 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.event; + +import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex; +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; + +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.PlainDeviceID; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +public class PipeTsFileInsertionEventTest { + + @Test(timeout = 5000) + public void testRealtimeEventCanSkipWaitingForClosedStatusAfterTsFileSealed() throws Exception { + final File tempDir = Files.createTempDirectory("pipeTsFileSealed").toFile(); + + try { + final TsFileResource resource = + createNonEmptyTsFileResource(tempDir, "realtime.tsfile", 1L, 1); + Assert.assertFalse(resource.isClosed()); + Assert.assertFalse(resource.isEmpty()); + + final PipeTsFileInsertionEvent sourceEvent = new PipeTsFileInsertionEvent(resource, false); + Assert.assertTrue(sourceEvent.waitForTsFileClose()); + + final PipeTsFileInsertionEvent copiedEvent = + sourceEvent.shallowCopySelfAndBindPipeTaskMetaForProgressReport( + "pipe", 1L, null, null, Long.MIN_VALUE, Long.MAX_VALUE); + Assert.assertTrue(copiedEvent.waitForTsFileClose()); + + copiedEvent.close(); + sourceEvent.close(); + } finally { + FileUtils.deleteFileOrDirectory(tempDir); + } + } + + private TsFileResource createNonEmptyTsFileResource( + final File tempDir, final String fileName, final long flushOrderId, final int dataRegionId) + throws IOException { + final File file = new File(tempDir, fileName); + Assert.assertTrue(file.createNewFile()); + + final TsFileResource resource = new TsFileResource(file); + resource.updateProgressIndex(new SimpleProgressIndex(1, flushOrderId)); + final ITimeIndex timeIndex = new DeviceTimeIndex(); + final IDeviceID deviceID = new PlainDeviceID("root.db.d" + dataRegionId); + timeIndex.putStartTime(deviceID, 1); + timeIndex.putEndTime(deviceID, 1); + resource.setTimeIndex(timeIndex); + return resource; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionDataContainerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionDataContainerTest.java index bd4e3923815f4..36f56e0e6068c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionDataContainerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionDataContainerTest.java @@ -26,19 +26,30 @@ import org.apache.iotdb.commons.pipe.datastructure.pattern.PrefixPipePattern; import org.apache.iotdb.db.pipe.event.common.tsfile.container.TsFileInsertionDataContainer; import org.apache.iotdb.db.pipe.event.common.tsfile.container.query.TsFileInsertionQueryDataContainer; +import org.apache.iotdb.db.pipe.event.common.tsfile.container.scan.AlignedSinglePageWholeChunkReader; +import org.apache.iotdb.db.pipe.event.common.tsfile.container.scan.SinglePageWholeChunkReader; import org.apache.iotdb.db.pipe.event.common.tsfile.container.scan.TsFileInsertionScanDataContainer; +import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock; +import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil; import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionTestFileWriter; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.pipe.api.access.Row; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.MetaMarker; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.file.metadata.IChunkMetadata; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.utils.Binary; @@ -56,6 +67,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.Field; import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; @@ -119,6 +131,297 @@ public void testScanContainer() throws Exception { System.out.println(System.currentTimeMillis() - startTime); } + @Test + public void testScanParserSplitNonAlignedSinglePageChunkByEstimatedPageMemory() throws Exception { + final long originalPipeMaxReaderChunkSize = + CommonDescriptor.getInstance().getConfig().getPipeMaxReaderChunkSize(); + final int originalPageSizeInByte = + TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + final int originalMaxNumberOfPointsInPage = + TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage(); + + try { + TSFileDescriptor.getInstance().getConfig().setPageSizeInByte(64 * 1024); + TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(10000); + + final int measurementCount = 16; + final int rowCount = 64; + final List schemaList = new ArrayList<>(); + for (int i = 0; i < measurementCount; ++i) { + schemaList.add( + new MeasurementSchema( + "s" + i, TSDataType.STRING, TSEncoding.PLAIN, CompressionType.LZ4)); + } + + nonalignedTsFile = new File("nonaligned-single-page-high-compression.tsfile"); + final Tablet tablet = new Tablet("root.sg.d", schemaList, rowCount); + final Binary value = + new Binary(new String(new char[512]).replace('\0', 'a'), TSFileConfig.STRING_CHARSET); + for (int row = 0; row < rowCount; ++row) { + tablet.addTimestamp(row, row); + for (int measurementIndex = 0; measurementIndex < measurementCount; ++measurementIndex) { + tablet.addValue("s" + measurementIndex, row, value); + } + } + tablet.rowSize = rowCount; + + try (final TsFileWriter writer = new TsFileWriter(nonalignedTsFile)) { + writer.registerTimeseries(new Path("root.sg.d"), schemaList); + writer.write(tablet); + } + + CommonDescriptor.getInstance() + .getConfig() + .setPipeMaxReaderChunkSize( + calculatePipeMaxReaderChunkSizeForSinglePageNonAlignedChunk(nonalignedTsFile)); + + int tabletCount = 0; + int maxMeasurementCount = 0; + int pointCount = 0; + try (final TsFileInsertionScanDataContainer parser = + new TsFileInsertionScanDataContainer( + nonalignedTsFile, + new PrefixPipePattern("root"), + Long.MIN_VALUE, + Long.MAX_VALUE, + null, + null, + false)) { + for (final Pair tabletWithIsAligned : parser.toTabletWithIsAligneds()) { + Assert.assertFalse(tabletWithIsAligned.getRight()); + final Tablet parsedTablet = tabletWithIsAligned.getLeft(); + tabletCount++; + maxMeasurementCount = Math.max(maxMeasurementCount, parsedTablet.getSchemas().size()); + pointCount += getNonNullSize(parsedTablet); + } + } + + Assert.assertTrue(tabletCount > 1); + Assert.assertTrue(maxMeasurementCount < measurementCount); + Assert.assertEquals(measurementCount * rowCount, pointCount); + } finally { + CommonDescriptor.getInstance() + .getConfig() + .setPipeMaxReaderChunkSize(originalPipeMaxReaderChunkSize); + TSFileDescriptor.getInstance().getConfig().setPageSizeInByte(originalPageSizeInByte); + TSFileDescriptor.getInstance() + .getConfig() + .setMaxNumberOfPointsInPage(originalMaxNumberOfPointsInPage); + } + } + + @Test + public void testScanParserSplitAlignedSinglePageChunkByEstimatedPageMemory() throws Exception { + final long originalPipeMaxReaderChunkSize = + CommonDescriptor.getInstance().getConfig().getPipeMaxReaderChunkSize(); + final int originalPageSizeInByte = + TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + final int originalMaxNumberOfPointsInPage = + TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage(); + + try { + TSFileDescriptor.getInstance().getConfig().setPageSizeInByte(64 * 1024); + TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(10000); + + final int measurementCount = 16; + final int rowCount = 64; + final List schemaList = new ArrayList<>(); + for (int i = 0; i < measurementCount; ++i) { + schemaList.add( + new MeasurementSchema( + "s" + i, TSDataType.STRING, TSEncoding.PLAIN, CompressionType.LZ4)); + } + + alignedTsFile = new File("aligned-single-page-high-compression.tsfile"); + final Tablet tablet = new Tablet("root.sg.d", schemaList, rowCount); + final Binary value = + new Binary(new String(new char[512]).replace('\0', 'a'), TSFileConfig.STRING_CHARSET); + for (int row = 0; row < rowCount; ++row) { + tablet.addTimestamp(row, row); + for (int measurementIndex = 0; measurementIndex < measurementCount; ++measurementIndex) { + tablet.addValue("s" + measurementIndex, row, value); + } + } + tablet.rowSize = rowCount; + + try (final TsFileWriter writer = new TsFileWriter(alignedTsFile)) { + writer.registerAlignedTimeseries(new Path("root.sg.d"), schemaList); + writer.writeAligned(tablet); + } + + CommonDescriptor.getInstance() + .getConfig() + .setPipeMaxReaderChunkSize( + calculatePipeMaxReaderChunkSizeForSinglePageAlignedChunk(alignedTsFile)); + + int tabletCount = 0; + int maxMeasurementCount = 0; + int pointCount = 0; + try (final TsFileInsertionScanDataContainer parser = + new TsFileInsertionScanDataContainer( + alignedTsFile, + new PrefixPipePattern("root"), + Long.MIN_VALUE, + Long.MAX_VALUE, + null, + null, + false)) { + for (final Pair tabletWithIsAligned : parser.toTabletWithIsAligneds()) { + Assert.assertTrue(tabletWithIsAligned.getRight()); + final Tablet parsedTablet = tabletWithIsAligned.getLeft(); + tabletCount++; + maxMeasurementCount = Math.max(maxMeasurementCount, parsedTablet.getSchemas().size()); + pointCount += getNonNullSize(parsedTablet); + } + } + + Assert.assertTrue(tabletCount > 1); + Assert.assertTrue(maxMeasurementCount < measurementCount); + Assert.assertEquals(measurementCount * rowCount, pointCount); + } finally { + CommonDescriptor.getInstance() + .getConfig() + .setPipeMaxReaderChunkSize(originalPipeMaxReaderChunkSize); + TSFileDescriptor.getInstance().getConfig().setPageSizeInByte(originalPageSizeInByte); + TSFileDescriptor.getInstance() + .getConfig() + .setMaxNumberOfPointsInPage(originalMaxNumberOfPointsInPage); + } + } + + @Test + public void testScanParserSplitAlignedMultiPageChunkByEstimatedPageMemory() throws Exception { + final long originalPipeMaxReaderChunkSize = + CommonDescriptor.getInstance().getConfig().getPipeMaxReaderChunkSize(); + final int originalPageSizeInByte = + TSFileDescriptor.getInstance().getConfig().getPageSizeInByte(); + final int originalMaxNumberOfPointsInPage = + TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage(); + + try { + TSFileDescriptor.getInstance().getConfig().setPageSizeInByte(64 * 1024); + TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(32); + + final int measurementCount = 16; + final int rowCount = 64; + final List schemaList = new ArrayList<>(); + for (int i = 0; i < measurementCount; ++i) { + schemaList.add( + new MeasurementSchema( + "s" + i, TSDataType.STRING, TSEncoding.PLAIN, CompressionType.LZ4)); + } + + alignedTsFile = new File("aligned-multi-page-high-compression.tsfile"); + final Tablet tablet = new Tablet("root.sg.d", schemaList, rowCount); + final Binary value = + new Binary(new String(new char[512]).replace('\0', 'a'), TSFileConfig.STRING_CHARSET); + for (int row = 0; row < rowCount; ++row) { + tablet.addTimestamp(row, row); + for (int measurementIndex = 0; measurementIndex < measurementCount; ++measurementIndex) { + tablet.addValue("s" + measurementIndex, row, value); + } + } + tablet.rowSize = rowCount; + + try (final TsFileWriter writer = new TsFileWriter(alignedTsFile)) { + writer.registerAlignedTimeseries(new Path("root.sg.d"), schemaList); + writer.writeAligned(tablet); + } + + CommonDescriptor.getInstance() + .getConfig() + .setPipeMaxReaderChunkSize( + calculatePipeMaxReaderChunkSizeForMultiPageAlignedChunk(alignedTsFile)); + + int tabletCount = 0; + int maxMeasurementCount = 0; + int pointCount = 0; + try (final TsFileInsertionScanDataContainer parser = + new TsFileInsertionScanDataContainer( + alignedTsFile, + new PrefixPipePattern("root"), + Long.MIN_VALUE, + Long.MAX_VALUE, + null, + null, + false)) { + for (final Pair tabletWithIsAligned : parser.toTabletWithIsAligneds()) { + Assert.assertTrue(tabletWithIsAligned.getRight()); + final Tablet parsedTablet = tabletWithIsAligned.getLeft(); + tabletCount++; + maxMeasurementCount = Math.max(maxMeasurementCount, parsedTablet.getSchemas().size()); + pointCount += getNonNullSize(parsedTablet); + } + } + + Assert.assertTrue(tabletCount > 1); + Assert.assertTrue(maxMeasurementCount < measurementCount); + Assert.assertEquals(measurementCount * rowCount, pointCount); + } finally { + CommonDescriptor.getInstance() + .getConfig() + .setPipeMaxReaderChunkSize(originalPipeMaxReaderChunkSize); + TSFileDescriptor.getInstance().getConfig().setPageSizeInByte(originalPageSizeInByte); + TSFileDescriptor.getInstance() + .getConfig() + .setMaxNumberOfPointsInPage(originalMaxNumberOfPointsInPage); + } + } + + @Test + public void testScanParserResizesChunkMemoryForFirstAlignedValueChunk() throws Exception { + final long originalPipeMaxReaderChunkSize = + PipeConfig.getInstance().getPipeMaxReaderChunkSize(); + final int originalPipeDataStructureTabletSizeInBytes = + PipeConfig.getInstance().getPipeDataStructureTabletSizeInBytes(); + final int configuredBatchMemorySize = 1024 * 1024; + CommonDescriptor.getInstance().getConfig().setPipeMaxReaderChunkSize(0); + CommonDescriptor.getInstance() + .getConfig() + .setPipeDataStructureTabletSizeInBytes(configuredBatchMemorySize); + + alignedTsFile = new File("single-aligned-value-chunk.tsfile"); + final List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); + + final Tablet tablet = new Tablet("root.sg.d", schemaList, 2); + tablet.rowSize = 2; + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, 1L); + tablet.addTimestamp(1, 2); + tablet.addValue("s1", 1, 2L); + + try { + try (final TsFileWriter writer = new TsFileWriter(alignedTsFile)) { + writer.registerAlignedTimeseries(new Path("root.sg.d"), schemaList); + writer.writeAligned(tablet); + } + + try (final TsFileInsertionScanDataContainer parser = + new TsFileInsertionScanDataContainer( + alignedTsFile, + new PrefixPipePattern("root"), + Long.MIN_VALUE, + Long.MAX_VALUE, + null, + null, + false)) { + Assert.assertTrue(getAllocatedChunkMemory(parser).getMemoryUsageInBytes() > 0); + Assert.assertTrue(getAllocatedBatchDataMemory(parser).getMemoryUsageInBytes() > 0); + Assert.assertTrue( + getAllocatedBatchDataMemory(parser).getMemoryUsageInBytes() + < configuredBatchMemorySize); + } + } finally { + CommonDescriptor.getInstance() + .getConfig() + .setPipeMaxReaderChunkSize(originalPipeMaxReaderChunkSize); + CommonDescriptor.getInstance() + .getConfig() + .setPipeDataStructureTabletSizeInBytes(originalPipeDataStructureTabletSizeInBytes); + } + } + public void testToTabletInsertionEvents(final boolean isQuery) throws Exception { // Test empty chunk testMixedTsFileWithEmptyChunk(isQuery); @@ -645,4 +948,115 @@ private int getNonNullSize(final Tablet tablet) { } return count; } + + private PipeMemoryBlock getAllocatedChunkMemory(final TsFileInsertionScanDataContainer parser) + throws NoSuchFieldException, IllegalAccessException { + final Field field = + TsFileInsertionScanDataContainer.class.getDeclaredField("allocatedMemoryBlockForChunk"); + field.setAccessible(true); + return (PipeMemoryBlock) field.get(parser); + } + + private PipeMemoryBlock getAllocatedBatchDataMemory(final TsFileInsertionScanDataContainer parser) + throws NoSuchFieldException, IllegalAccessException { + final Field field = + TsFileInsertionScanDataContainer.class.getDeclaredField("allocatedMemoryBlockForBatchData"); + field.setAccessible(true); + return (PipeMemoryBlock) field.get(parser); + } + + private long calculatePipeMaxReaderChunkSizeForSinglePageAlignedChunk(final File tsFile) + throws Exception { + try (final TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { + final List deviceIDList = reader.getAllDevices(); + Assert.assertEquals(1, deviceIDList.size()); + final IDeviceID deviceID = deviceIDList.get(0); + final List alignedChunkMetadataList = + reader.getAlignedChunkMetadata(deviceID); + Assert.assertEquals(1, alignedChunkMetadataList.size()); + + final AlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(0); + final Chunk timeChunk = + reader.readMemChunk((ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata()); + Assert.assertEquals( + MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER, timeChunk.getHeader().getChunkType() & 0x3F); + + final List valueChunkList = new ArrayList<>(); + long chunkSizeLimit = PipeMemoryWeightUtil.calculateChunkRamBytesUsed(timeChunk); + for (final IChunkMetadata valueChunkMetadata : + alignedChunkMetadata.getValueChunkMetadataList()) { + final Chunk valueChunk = reader.readMemChunk((ChunkMetadata) valueChunkMetadata); + Assert.assertEquals( + MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER, valueChunk.getHeader().getChunkType() & 0x3F); + valueChunkList.add(valueChunk); + chunkSizeLimit += valueChunk.getHeader().getDataSize(); + } + + final long estimatedPageMemorySize = + AlignedSinglePageWholeChunkReader.calculatePageEstimatedMemoryUsageInBytes( + timeChunk, valueChunkList); + Assert.assertTrue(estimatedPageMemorySize > chunkSizeLimit); + return chunkSizeLimit; + } + } + + private long calculatePipeMaxReaderChunkSizeForSinglePageNonAlignedChunk(final File tsFile) + throws Exception { + try (final TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { + final IDeviceID deviceID = reader.getDeviceMeasurementsMap().keySet().iterator().next(); + final List measurements = reader.getDeviceMeasurementsMap().get(deviceID); + Assert.assertFalse(measurements.isEmpty()); + + long chunkSizeLimit = 0; + long estimatedPageMemorySize = 0; + for (final String measurement : measurements) { + final List chunkMetadataList = + reader.getChunkMetadataList(new Path(deviceID, measurement, false)); + Assert.assertEquals(1, chunkMetadataList.size()); + + final Chunk chunk = reader.readMemChunk(chunkMetadataList.get(0)); + Assert.assertEquals( + MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER, chunk.getHeader().getChunkType() & 0x3F); + chunkSizeLimit += chunk.getHeader().getDataSize(); + estimatedPageMemorySize += + SinglePageWholeChunkReader.calculateMaxPageEstimatedMemoryUsageInBytesWithBatchData( + chunk); + } + + Assert.assertTrue(estimatedPageMemorySize > chunkSizeLimit); + return chunkSizeLimit; + } + } + + private long calculatePipeMaxReaderChunkSizeForMultiPageAlignedChunk(final File tsFile) + throws Exception { + try (final TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.getAbsolutePath())) { + final List deviceIDList = reader.getAllDevices(); + Assert.assertEquals(1, deviceIDList.size()); + final IDeviceID deviceID = deviceIDList.get(0); + final List alignedChunkMetadataList = + reader.getAlignedChunkMetadata(deviceID); + Assert.assertEquals(1, alignedChunkMetadataList.size()); + + final AlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(0); + final Chunk timeChunk = + reader.readMemChunk((ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata()); + Assert.assertEquals(MetaMarker.CHUNK_HEADER, timeChunk.getHeader().getChunkType() & 0x3F); + + long chunkSizeLimit = PipeMemoryWeightUtil.calculateChunkRamBytesUsed(timeChunk); + long estimatedMaxPageMemorySize = + SinglePageWholeChunkReader.calculateMaxPageEstimatedMemoryUsageInBytes(timeChunk); + for (final IChunkMetadata valueChunkMetadata : + alignedChunkMetadata.getValueChunkMetadataList()) { + final Chunk valueChunk = reader.readMemChunk((ChunkMetadata) valueChunkMetadata); + Assert.assertEquals(MetaMarker.CHUNK_HEADER, valueChunk.getHeader().getChunkType() & 0x3F); + chunkSizeLimit += valueChunk.getHeader().getDataSize(); + estimatedMaxPageMemorySize += + SinglePageWholeChunkReader.calculateMaxPageEstimatedMemoryUsageInBytes(valueChunk); + } + + Assert.assertTrue(estimatedMaxPageMemorySize > chunkSizeLimit); + return chunkSizeLimit; + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletUtilsTest.java new file mode 100644 index 0000000000000..8bf32bd066ffb --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeTabletUtilsTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.event.common.tablet; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +public class PipeTabletUtilsTest { + + @Test + public void testPutValueUnmarksReusedNullRow() { + final List schemas = + Arrays.asList( + new MeasurementSchema("s1", TSDataType.FLOAT), + new MeasurementSchema("s2", TSDataType.FLOAT)); + final Tablet tablet = new Tablet("root.sg.d1", schemas, 2); + + PipeTabletUtils.markNullValue(tablet, 0, 0); + PipeTabletUtils.markNullValue(tablet, 0, 1); + + PipeTabletUtils.putValue(tablet, 0, 0, TSDataType.FLOAT, 1.0f); + PipeTabletUtils.putTimestamp(tablet, 0, 1L); + PipeTabletUtils.compactBitMaps(tablet); + + Assert.assertNull(tablet.bitMaps[0]); + Assert.assertTrue(tablet.bitMaps[1].isMarked(0)); + } + + @Test + public void testCopyBitMapsOrCreateEmptyWithNullBitMaps() { + final List schemas = + Arrays.asList( + new MeasurementSchema("s1", TSDataType.FLOAT), + new MeasurementSchema("s2", TSDataType.FLOAT)); + final Tablet tablet = new Tablet("root.sg.d1", schemas, 2); + tablet.addTimestamp(0, 1L); + tablet.addValue("s1", 0, 1.0f); + tablet.addValue("s2", 0, 2.0f); + + Assert.assertNull(tablet.bitMaps); + + final BitMap[] bitMaps = PipeTabletUtils.copyBitMapsOrCreateEmpty(tablet); + + Assert.assertEquals(schemas.size(), bitMaps.length); + Assert.assertNull(bitMaps[0]); + Assert.assertNull(bitMaps[1]); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessorTest.java new file mode 100644 index 0000000000000..2957ffd4e3e55 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessorTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.processor.twostage.plugin; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; + +public class TwoStageCountProcessorTest { + + @Test + public void testOutputSeriesSupportsNewAndLegacyKeys() throws Exception { + Assert.assertEquals( + "root.db.d.s1", parseOutputSeries("processor.output.series", "root.db.d.s1").getFullPath()); + Assert.assertEquals( + "root.db.d.s2", parseOutputSeries("processor.output-series", "root.db.d.s2").getFullPath()); + } + + @Test + public void testValidateOutputSeriesSupportsNewAndLegacyKeys() throws Exception { + validateOutputSeries("processor.output.series", "root.db.d.s1"); + validateOutputSeries("processor.output-series", "root.db.d.s2"); + } + + private PartialPath parseOutputSeries(final String key, final String value) throws Exception { + return TwoStageCountProcessor.parseOutputSeries( + new PipeParameters(Collections.singletonMap(key, value))); + } + + private void validateOutputSeries(final String key, final String value) throws Exception { + new TwoStageCountProcessor() + .validate( + new PipeParameterValidator(new PipeParameters(Collections.singletonMap(key, value)))); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/PipeStatementTsStatusVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/PipeStatementTsStatusVisitorTest.java index 2b20f1d91efb8..756d11818251f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/PipeStatementTsStatusVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/PipeStatementTsStatusVisitorTest.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.receiver; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.db.pipe.receiver.protocol.thrift.IoTDBDataNodeReceiver; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; @@ -62,4 +63,17 @@ public void testTTLIdempotency() { StatusUtils.OK, new TSStatus(TSStatusCode.OUT_OF_TTL.getStatusCode())))) .getCode()); } + + @Test + public void testDatabaseNotExistRuntimeExceptionClassification() { + Assert.assertEquals( + TSStatusCode.PIPE_RECEIVER_PARALLEL_OR_USER_CONFLICT_EXCEPTION.getStatusCode(), + IoTDBDataNodeReceiver.STATEMENT_EXCEPTION_VISITOR + .process( + new InsertRowsStatement(), + new IoTDBRuntimeException( + "Create DataPartition failed because the database: root.test.sg_0 is not exists", + TSStatusCode.DATABASE_NOT_EXIST.getStatusCode())) + .getCode()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiverTest.java index 19dea8140a190..e23db1f1ca8cd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/airgap/IoTDBAirGapReceiverTest.java @@ -19,18 +19,32 @@ package org.apache.iotdb.db.pipe.receiver.protocol.airgap; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.pipe.receiver.IoTDBReceiver; import org.apache.iotdb.commons.pipe.sink.payload.airgap.AirGapELanguageConstant; +import org.apache.iotdb.commons.pipe.sink.payload.airgap.AirGapOneByteResponse; +import org.apache.iotdb.commons.pipe.sink.payload.airgap.AirGapPseudoTPipeTransferRequest; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; +import org.apache.iotdb.db.pipe.receiver.protocol.thrift.IoTDBDataNodeReceiverAgent; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; import org.apache.tsfile.utils.BytesUtils; import org.junit.Assert; import org.junit.Test; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.net.Socket; +import java.nio.ByteBuffer; public class IoTDBAirGapReceiverTest { @@ -69,4 +83,93 @@ public void testRejectNestedELanguagePrefix() throws Exception { Assert.assertThrows(IOException.class, () -> receiver.readData(inputStream)); Assert.assertTrue(exception.getMessage().contains("nested E-Language prefix")); } + + @Test + public void testTemporaryUnavailableRetryTimeoutReturnsFail() throws Exception { + final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + final long originalRetryLocalIntervalMs = commonConfig.getPipeAirGapRetryLocalIntervalMs(); + final long originalRetryMaxMs = commonConfig.getPipeAirGapRetryMaxMs(); + + try { + commonConfig.setPipeAirGapRetryLocalIntervalMs(0); + commonConfig.setPipeAirGapRetryMaxMs(1); + + final RecordingSocket socket = new RecordingSocket(); + final IoTDBAirGapReceiver receiver = new IoTDBAirGapReceiver(socket, 3L); + final StubIoTDBDataNodeReceiverAgent stubAgent = new StubIoTDBDataNodeReceiverAgent(); + stubAgent.setStubReceiver( + new StubReceiver( + new TSStatus( + TSStatusCode.PIPE_RECEIVER_TEMPORARY_UNAVAILABLE_EXCEPTION.getStatusCode()))); + setField(receiver, "agent", stubAgent); + + final AirGapPseudoTPipeTransferRequest req = new AirGapPseudoTPipeTransferRequest(); + req.setVersion(IoTDBSinkRequestVersion.VERSION_1.getVersion()); + req.setType((short) 0); + req.setBody(ByteBuffer.allocate(0)); + + final Method handleReq = + IoTDBAirGapReceiver.class.getDeclaredMethod( + "handleReq", AirGapPseudoTPipeTransferRequest.class, long.class); + handleReq.setAccessible(true); + handleReq.invoke(receiver, req, System.currentTimeMillis() - 10_000L); + + Assert.assertArrayEquals(AirGapOneByteResponse.FAIL, socket.getWrittenBytes()); + } finally { + commonConfig.setPipeAirGapRetryLocalIntervalMs(originalRetryLocalIntervalMs); + commonConfig.setPipeAirGapRetryMaxMs(originalRetryMaxMs); + } + } + + private static void setField(final Object target, final String fieldName, final Object value) + throws Exception { + final Field field = IoTDBAirGapReceiver.class.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + } + + private static class RecordingSocket extends Socket { + + private final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + @Override + public OutputStream getOutputStream() { + return outputStream; + } + + byte[] getWrittenBytes() { + return outputStream.toByteArray(); + } + } + + private static class StubIoTDBDataNodeReceiverAgent extends IoTDBDataNodeReceiverAgent { + + void setStubReceiver(final IoTDBReceiver receiver) { + setReceiverWithSpecifiedClient(null, receiver); + } + } + + private static class StubReceiver implements IoTDBReceiver { + + private final TPipeTransferResp response; + + private StubReceiver(final TSStatus status) { + response = new TPipeTransferResp(status); + } + + @Override + public TPipeTransferResp receive(final TPipeTransferReq req) { + return response; + } + + @Override + public void handleExit() { + // noop for unit test + } + + @Override + public IoTDBSinkRequestVersion getVersion() { + return IoTDBSinkRequestVersion.VERSION_1; + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgentTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgentTest.java new file mode 100644 index 0000000000000..eaf21aef1fde8 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/legacy/IoTDBLegacyPipeReceiverAgentTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.receiver.protocol.legacy; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.db.pipe.sink.payload.legacy.TsFilePipeData; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TSyncIdentityInfo; +import org.apache.iotdb.service.rpc.thrift.TSyncTransportMetaInfo; + +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +public class IoTDBLegacyPipeReceiverAgentTest { + + private static final String PIPE_NAME = "poc"; + private static final long CREATE_TIME = 1700000000000L; + private static final String REMOTE_ADDRESS = "127.0.0.1"; + + private String originalSyncDir; + private Path syncDir; + private IoTDBLegacyPipeReceiverAgent agent; + + @Before + public void setUp() throws Exception { + originalSyncDir = CommonDescriptor.getInstance().getConfig().getSyncDir(); + syncDir = Files.createTempDirectory("legacy-pipe-receiver"); + CommonDescriptor.getInstance().getConfig().setSyncDir(syncDir.toString()); + + agent = new IoTDBLegacyPipeReceiverAgent(); + final TSStatus status = + agent.handshake( + new TSyncIdentityInfo(PIPE_NAME, CREATE_TIME, "UNKNOWN", ""), + REMOTE_ADDRESS, + null, + null); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + + @After + public void tearDown() throws Exception { + if (agent != null) { + agent.handleClientExit(); + } + CommonDescriptor.getInstance().getConfig().setSyncDir(originalSyncDir); + if (syncDir != null) { + FileUtils.deleteDirectory(syncDir.toFile()); + } + } + + @Test + public void testTransportFileRejectsPathTraversal() throws Exception { + final String traversal = + ".." + File.separator + ".." + File.separator + ".." + File.separator + "pwned"; + + final TSStatus status = + agent.transportFile( + new TSyncTransportMetaInfo(traversal, 0), + ByteBuffer.wrap("pwned".getBytes(StandardCharsets.UTF_8))); + + Assert.assertEquals(TSStatusCode.SYNC_FILE_ERROR.getStatusCode(), status.getCode()); + Assert.assertTrue(status.getMessage().contains("Illegal fileName")); + Assert.assertFalse(Files.exists(syncDir.resolve("pwned.patch"))); + } + + @Test + public void testTransportFileWritesPlainFileUnderFileDataDir() throws Exception { + final String fileName = "1-2-3-4.tsfile"; + final byte[] payload = "iotdb".getBytes(StandardCharsets.UTF_8); + + final TSStatus status = + agent.transportFile(new TSyncTransportMetaInfo(fileName, 0), ByteBuffer.wrap(payload)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + final Path patchFile = getFileDataDir().resolve(fileName + ".patch"); + Assert.assertArrayEquals(payload, Files.readAllBytes(patchFile)); + } + + @Test + public void testTransportPipeDataRejectsPathTraversalTsFileName() throws Exception { + final String traversal = ".." + File.separator + "evil.tsfile"; + + final TSStatus status = + agent.transportPipeData(ByteBuffer.wrap(new TsFilePipeData("", traversal, -1).serialize())); + + Assert.assertEquals(TSStatusCode.PIPESERVER_ERROR.getStatusCode(), status.getCode()); + Assert.assertTrue(status.getMessage().contains("Illegal fileName")); + } + + private Path getFileDataDir() { + return syncDir + .resolve("receiver") + .resolve(String.format("%s-%d-%s", PIPE_NAME, CREATE_TIME, REMOTE_ADDRESS)) + .resolve("file-data"); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiverTest.java new file mode 100644 index 0000000000000..1e279a18febfb --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/receiver/protocol/thrift/IoTDBDataNodeReceiverTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.receiver.protocol.thrift; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; +import org.apache.iotdb.db.storageengine.load.active.ActiveLoadPathHelper; +import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator; + +import org.junit.Assert; +import org.junit.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +public class IoTDBDataNodeReceiverTest { + + @Test + public void testLoadTsFileSyncStatementUsesTreeDatabaseLevelFromDatabaseName() throws Exception { + final Path tsFile = Files.createTempFile("pipe-load-tree-database-level", ".tsfile"); + try { + final LoadTsFileStatement statement = + IoTDBDataNodeReceiver.buildLoadTsFileStatementForSync( + "root.test.sg_0", tsFile.toString(), true, true); + + Assert.assertEquals("root.test.sg_0", statement.getDatabase()); + Assert.assertEquals(2, statement.getDatabaseLevel()); + } finally { + Files.deleteIfExists(tsFile); + } + } + + @Test + public void testLoadTsFileAsyncAttributesUseTreeDatabaseLevelFromDatabaseName() throws Exception { + final Path tsFile = Files.createTempFile("pipe-async-load-tree-database-level", ".tsfile"); + try { + final Map attributes = + IoTDBDataNodeReceiver.buildLoadTsFileAttributesForAsync( + "root.test.sg_0", true, false, true); + + Assert.assertEquals( + "root.test.sg_0", attributes.get(LoadTsFileConfigurator.DATABASE_NAME_KEY)); + Assert.assertEquals("2", attributes.get(LoadTsFileConfigurator.DATABASE_LEVEL_KEY)); + + final LoadTsFileStatement statement = LoadTsFileStatement.createUnchecked(tsFile.toString()); + ActiveLoadPathHelper.applyAttributesToStatement(attributes, statement, false); + Assert.assertEquals("root.test.sg_0", statement.getDatabase()); + Assert.assertEquals(2, statement.getDatabaseLevel()); + Assert.assertTrue(statement.isVerifySchema()); + } finally { + Files.deleteIfExists(tsFile); + } + } + + @Test + public void testLoadTsFileSyncStatementKeepsDefaultDatabaseLevelWhenDatabaseNameIsNull() + throws Exception { + final Path tsFile = Files.createTempFile("pipe-load-default-database-level", ".tsfile"); + try { + final LoadTsFileStatement statement = + IoTDBDataNodeReceiver.buildLoadTsFileStatementForSync( + null, tsFile.toString(), true, true); + + Assert.assertNull(statement.getDatabase()); + Assert.assertEquals( + IoTDBDescriptor.getInstance().getConfig().getDefaultDatabaseLevel(), + statement.getDatabaseLevel()); + } finally { + Files.deleteIfExists(tsFile); + } + } + + @Test + public void testRepeatedStatementExceptionLogIsReduced() throws Exception { + final Path tsFile = Files.createTempFile("pipe-load-log-reducer", ".tsfile"); + try { + final LoadTsFileStatement statement = + IoTDBDataNodeReceiver.buildLoadTsFileStatementForSync( + "root.test.sg_0", tsFile.toString(), true, true); + final long receiverId = System.nanoTime(); + final Exception exception = new RuntimeException("repeated receiver exception " + receiverId); + + Assert.assertTrue( + IoTDBDataNodeReceiver.shouldLogStatementException(receiverId, statement, exception)); + Assert.assertFalse( + IoTDBDataNodeReceiver.shouldLogStatementException(receiverId, statement, exception)); + Assert.assertTrue( + IoTDBDataNodeReceiver.shouldLogStatementException( + receiverId, statement, new RuntimeException("another receiver exception"))); + } finally { + Files.deleteIfExists(tsFile); + } + } + + @Test + public void testLoadTsFileSyncStatementVerifiesSchemaWhenConvertingType() throws Exception { + final Path tsFile = Files.createTempFile("pipe-load-convert-verify-schema", ".tsfile"); + try { + final LoadTsFileStatement statement = + IoTDBDataNodeReceiver.buildLoadTsFileStatementForSync( + "root.test.sg_0", tsFile.toString(), false, true); + + Assert.assertTrue(statement.isConvertOnTypeMismatch()); + Assert.assertTrue(statement.isVerifySchema()); + } finally { + Files.deleteIfExists(tsFile); + } + } + + @Test + public void testLoadTsFileSyncStatementCanSkipVerifySchemaWhenNotConvertingType() + throws Exception { + final Path tsFile = Files.createTempFile("pipe-load-no-convert-no-verify-schema", ".tsfile"); + try { + final LoadTsFileStatement statement = + IoTDBDataNodeReceiver.buildLoadTsFileStatementForSync( + "root.test.sg_0", tsFile.toString(), false, false); + + Assert.assertFalse(statement.isConvertOnTypeMismatch()); + Assert.assertFalse(statement.isVerifySchema()); + } finally { + Files.deleteIfExists(tsFile); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimatorTest.java new file mode 100644 index 0000000000000..58b2756b16f6b --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimatorTest.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.resource.memory; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertMultiTabletsNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class InsertNodeMemoryEstimatorTest { + + @Test + public void testInsertRowsNodeLaterRowSizeIsEstimated() throws IllegalPathException { + InsertRowNode firstRow = + createTextInsertRowNode("child-1", "root.sg.d1", new String[] {"s1"}, new String[] {"v1"}); + InsertRowNode smallSecondRow = + createTextInsertRowNode("child-2", "root.sg.d2", new String[] {"s1"}, new String[] {"v2"}); + InsertRowNode largeSecondRow = + createTextInsertRowNode( + "child-3", + "root.sg.device_with_a_longer_path_segment", + new String[] {"s1", "measurement_with_a_longer_name", "s3"}, + new String[] {"v2", repeatedString("payload", 32), repeatedString("payload", 48)}); + + long baselineSize = + InsertNodeMemoryEstimator.sizeOf(createInsertRowsNode("parent", firstRow, smallSecondRow)); + long largerNodeSize = + InsertNodeMemoryEstimator.sizeOf(createInsertRowsNode("parent", firstRow, largeSecondRow)); + + Assert.assertTrue(largerNodeSize > baselineSize); + } + + @Test + public void testInsertRowsNodeResultsAreEstimated() throws IllegalPathException { + InsertRowsNode node = + createInsertRowsNode( + "parent", + createTextInsertRowNode( + "child-1", "root.sg.d1", new String[] {"s1"}, new String[] {"v1"}), + createTextInsertRowNode( + "child-2", "root.sg.d2", new String[] {"s1"}, new String[] {"v2"})); + + long sizeWithoutResults = InsertNodeMemoryEstimator.sizeOf(node); + + TSStatus statusWithoutSubStatus = createStatus("outer-message"); + node.getResults().put(1, statusWithoutSubStatus); + long sizeWithResults = InsertNodeMemoryEstimator.sizeOf(node); + + TSStatus statusWithSubStatus = createStatus("outer-message"); + List subStatusList = new ArrayList<>(); + subStatusList.add(createStatus(repeatedString("inner-message", 16))); + statusWithSubStatus.setSubStatus(subStatusList); + node.getResults().put(1, statusWithSubStatus); + long sizeWithSubStatus = InsertNodeMemoryEstimator.sizeOf(node); + + Assert.assertTrue(sizeWithResults > sizeWithoutResults); + Assert.assertTrue(sizeWithSubStatus > sizeWithResults); + } + + @Test + public void testInsertRowsOfOneDeviceNodeLaterRowSizeIsEstimated() throws IllegalPathException { + InsertRowNode firstRow = + createTextInsertRowNode( + "child-1", "root.sg.d1", new String[] {"s1", "s2"}, new String[] {"v1", "v2"}); + InsertRowNode smallSecondRow = + createTextInsertRowNode( + "child-2", "root.sg.d1", new String[] {"s1", "s2"}, new String[] {"v3", "v4"}); + InsertRowNode largeSecondRow = + createTextInsertRowNode( + "child-3", + "root.sg.d1", + new String[] {"s1", "s2"}, + new String[] {repeatedString("payload", 32), repeatedString("payload", 48)}); + + long baselineSize = + InsertNodeMemoryEstimator.sizeOf( + createInsertRowsOfOneDeviceNode("parent", firstRow, smallSecondRow)); + long largerNodeSize = + InsertNodeMemoryEstimator.sizeOf( + createInsertRowsOfOneDeviceNode("parent", firstRow, largeSecondRow)); + + Assert.assertTrue(largerNodeSize > baselineSize); + } + + @Test + public void testInsertMultiTabletsNodeLaterTabletSizeIsEstimated() throws IllegalPathException { + InsertTabletNode firstTablet = createTextInsertTabletNode("child-1", "root.sg.d1", 1, 1, 2); + InsertTabletNode smallSecondTablet = + createTextInsertTabletNode("child-2", "root.sg.d2", 1, 1, 2); + InsertTabletNode largeSecondTablet = + createTextInsertTabletNode("child-3", "root.sg.d3", 3, 8, 16); + + long baselineSize = + InsertNodeMemoryEstimator.sizeOf( + createInsertMultiTabletsNode("parent", firstTablet, smallSecondTablet)); + long largerNodeSize = + InsertNodeMemoryEstimator.sizeOf( + createInsertMultiTabletsNode("parent", firstTablet, largeSecondTablet)); + + Assert.assertTrue(largerNodeSize > baselineSize); + } + + @Test + public void testPlanNodeIdIsEstimated() throws IllegalPathException { + InsertRowNode shortPlanNodeIdRow = + createTextInsertRowNode("id", "root.sg.d1", new String[] {"s1"}, new String[] {"v1"}); + InsertRowNode longPlanNodeIdRow = + createTextInsertRowNode( + repeatedString("plan-node-id", 12), + "root.sg.d1", + new String[] {"s1"}, + new String[] {"v1"}); + + Assert.assertTrue( + InsertNodeMemoryEstimator.sizeOf(longPlanNodeIdRow) + > InsertNodeMemoryEstimator.sizeOf(shortPlanNodeIdRow)); + } + + private static InsertRowsNode createInsertRowsNode( + String planNodeId, InsertRowNode... insertRowNodes) { + InsertRowsNode node = new InsertRowsNode(new PlanNodeId(planNodeId)); + for (int i = 0; i < insertRowNodes.length; i++) { + node.addOneInsertRowNode(insertRowNodes[i], i); + } + node.setDevicePath(insertRowNodes[0].getDevicePath()); + node.setMeasurementSchemas(insertRowNodes[0].getMeasurementSchemas()); + node.setMeasurements(insertRowNodes[0].getMeasurements()); + node.setDataTypes(insertRowNodes[0].getDataTypes()); + return node; + } + + private static InsertRowsOfOneDeviceNode createInsertRowsOfOneDeviceNode( + String planNodeId, InsertRowNode... insertRowNodes) { + InsertRowsOfOneDeviceNode node = new InsertRowsOfOneDeviceNode(new PlanNodeId(planNodeId)); + List rows = new ArrayList<>(Arrays.asList(insertRowNodes)); + List indexes = new ArrayList<>(); + for (int i = 0; i < insertRowNodes.length; i++) { + indexes.add(i); + } + node.setInsertRowNodeList(rows); + node.setInsertRowNodeIndexList(indexes); + return node; + } + + private static InsertMultiTabletsNode createInsertMultiTabletsNode( + String planNodeId, InsertTabletNode... insertTabletNodes) { + InsertMultiTabletsNode node = new InsertMultiTabletsNode(new PlanNodeId(planNodeId)); + for (int i = 0; i < insertTabletNodes.length; i++) { + node.addInsertTabletNode(insertTabletNodes[i], i); + } + return node; + } + + private static InsertRowNode createTextInsertRowNode( + String planNodeId, String devicePath, String[] measurements, String[] values) + throws IllegalPathException { + TSDataType[] dataTypes = new TSDataType[measurements.length]; + MeasurementSchema[] measurementSchemas = new MeasurementSchema[measurements.length]; + Object[] rowValues = new Object[measurements.length]; + for (int i = 0; i < measurements.length; i++) { + dataTypes[i] = TSDataType.TEXT; + measurementSchemas[i] = new MeasurementSchema(measurements[i], TSDataType.TEXT); + rowValues[i] = new Binary(values[i], TSFileConfig.STRING_CHARSET); + } + return new InsertRowNode( + new PlanNodeId(planNodeId), + new PartialPath(devicePath), + false, + measurements, + dataTypes, + measurementSchemas, + 1L, + rowValues, + false); + } + + private static InsertTabletNode createTextInsertTabletNode( + String planNodeId, String devicePath, int measurementCount, int rowCount, int repeatCount) + throws IllegalPathException { + String[] measurements = new String[measurementCount]; + TSDataType[] dataTypes = new TSDataType[measurementCount]; + MeasurementSchema[] measurementSchemas = new MeasurementSchema[measurementCount]; + Object[] columns = new Object[measurementCount]; + for (int measurementIndex = 0; measurementIndex < measurementCount; measurementIndex++) { + measurements[measurementIndex] = "s" + measurementIndex; + dataTypes[measurementIndex] = TSDataType.TEXT; + measurementSchemas[measurementIndex] = + new MeasurementSchema(measurements[measurementIndex], TSDataType.TEXT); + Binary[] values = new Binary[rowCount]; + for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { + values[rowIndex] = + new Binary( + repeatedString("value-" + measurementIndex + "-" + rowIndex, repeatCount), + TSFileConfig.STRING_CHARSET); + } + columns[measurementIndex] = values; + } + + long[] times = new long[rowCount]; + for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { + times[rowIndex] = rowIndex; + } + + return new InsertTabletNode( + new PlanNodeId(planNodeId), + new PartialPath(devicePath), + false, + measurements, + dataTypes, + measurementSchemas, + times, + null, + columns, + rowCount); + } + + private static TSStatus createStatus(String message) { + TSStatus status = new TSStatus(); + status.setCode(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + status.setMessage(message); + return status; + } + + private static String repeatedString(String unit, int repeatCount) { + StringBuilder builder = new StringBuilder(unit.length() * repeatCount); + for (int i = 0; i < repeatCount; i++) { + builder.append(unit); + } + return builder.toString(); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeDataNodeThriftRequestTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeDataNodeThriftRequestTest.java index ee9b7218dab35..4e4d11aacbf53 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeDataNodeThriftRequestTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeDataNodeThriftRequestTest.java @@ -321,6 +321,46 @@ public void testPipeTransferTabletBatchReq() throws IOException { Assert.assertFalse(deserializedReq.getTabletReqs().get(0).getIsAligned()); } + @Test + public void testPipeTransferTabletBatchReqInternsRepeatedMeasurementNames() throws IOException { + final List tabletBuffers = new ArrayList<>(); + tabletBuffers.add( + serializeTablet(createSingleValueTablet(new String("root.sg.d"), new String("s1")), false)); + tabletBuffers.add( + serializeTablet(createSingleValueTablet(new String("root.sg.d"), new String("s1")), false)); + + final PipeTransferTabletBatchReq deserializedReq = + PipeTransferTabletBatchReq.fromTPipeTransferReq( + PipeTransferTabletBatchReq.toTPipeTransferReq(Collections.emptyList(), tabletBuffers)); + final Tablet firstTablet = deserializedReq.getTabletReqs().get(0).getTablet(); + final Tablet secondTablet = deserializedReq.getTabletReqs().get(1).getTablet(); + + Assert.assertSame(firstTablet.deviceId, secondTablet.deviceId); + Assert.assertSame( + firstTablet.getSchemas().get(0).getMeasurementId(), + secondTablet.getSchemas().get(0).getMeasurementId()); + } + + private static Tablet createSingleValueTablet(final String deviceId, final String measurement) { + final List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema(measurement, TSDataType.INT32)); + final Tablet tablet = new Tablet(deviceId, schemaList, 1); + tablet.addTimestamp(0, 1); + tablet.addValue(measurement, 0, 1); + tablet.rowSize = 1; + return tablet; + } + + private static ByteBuffer serializeTablet(final Tablet tablet, final boolean isAligned) + throws IOException { + try (final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + tablet.serialize(outputStream); + ReadWriteIOUtils.write(isAligned, outputStream); + return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } + } + @Test public void testPipeTransferFilePieceReq() throws IOException { final byte[] body = "testPipeTransferFilePieceReq".getBytes(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSinkTest.java index fae6430876273..537ae6648b2b1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSinkTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/PipeSinkTest.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.pipe.sink; import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; import org.apache.iotdb.commons.pipe.config.plugin.configuraion.PipeTaskRuntimeConfiguration; import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskSinkRuntimeEnvironment; @@ -28,14 +29,18 @@ import org.apache.iotdb.db.pipe.sink.protocol.opcua.OpcUaSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink; import org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBDataRegionSyncSink; +import org.apache.iotdb.db.pipe.sink.protocol.websocket.WebSocketConnectorServer; +import org.apache.iotdb.db.pipe.sink.protocol.websocket.WebSocketSink; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.exception.PipeException; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.security.SecureRandom; import java.util.Arrays; @@ -103,6 +108,85 @@ public void testIoTDBThriftAsyncConnectorToOthers() { } } + @Test + public void testAsyncSinkDropDoesNotRequeueDroppedPipeEvents() throws Exception { + try (final IoTDBDataRegionAsyncSink connector = new IoTDBDataRegionAsyncSink()) { + final PipeParameters parameters = + new PipeParameters( + new HashMap() { + { + put( + PipeSinkConstant.CONNECTOR_KEY, + BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName()); + put(PipeSinkConstant.CONNECTOR_IOTDB_NODE_URLS_KEY, "127.0.0.1:6668"); + } + }); + connector.validate(new PipeParameterValidator(parameters)); + connector.customize( + parameters, + new PipeTaskRuntimeConfiguration(new PipeTaskSinkRuntimeEnvironment("pipe", 1L, 1))); + + final PipeRawTabletInsertionEvent droppedEvent = + createPipeRawTabletInsertionEvent("pipe", 1L, 1); + droppedEvent.increaseReferenceCount("test"); + droppedEvent.setCommitterKeyAndCommitId(new CommitterKey("pipe", 1L, 1, -1), 1L); + + connector.discardEventsOfPipe("pipe", 1L, 1); + connector.addFailureEventToRetryQueue(droppedEvent, new PipeException("test")); + + Assert.assertEquals(0, connector.getRetryEventQueueSize()); + Assert.assertTrue(droppedEvent.isReleased()); + + final PipeRawTabletInsertionEvent recreatedPipeEvent = + createPipeRawTabletInsertionEvent("pipe", 2L, 1); + recreatedPipeEvent.increaseReferenceCount("test"); + recreatedPipeEvent.setCommitterKeyAndCommitId(new CommitterKey("pipe", 2L, 1, -1), 1L); + + connector.addFailureEventToRetryQueue(recreatedPipeEvent, new PipeException("test")); + + Assert.assertEquals(1, connector.getRetryEventQueueSize()); + } + } + + @Test + public void testWebSocketSinkDropDoesNotRequeueDroppedPipeEvents() { + final String pipeName = "pipe_" + System.nanoTime(); + final WebSocketConnectorServer server = WebSocketConnectorServer.getOrCreateInstance(0); + final WebSocketSink connector = Mockito.mock(WebSocketSink.class); + Mockito.when(connector.getPipeName()).thenReturn(pipeName); + + server.register(connector); + try { + final PipeRawTabletInsertionEvent droppedEvent = + createPipeRawTabletInsertionEvent(pipeName, 1L, 1); + droppedEvent.increaseReferenceCount(WebSocketSink.class.getName()); + droppedEvent.setCommitterKeyAndCommitId(new CommitterKey(pipeName, 1L, 1, -1), 1L); + server.addEvent(droppedEvent, connector); + + server.discardEventsOfPipe(pipeName, 1L, 1); + Assert.assertTrue(droppedEvent.isReleased()); + + final PipeRawTabletInsertionEvent recreatedDroppedPipeEvent = + createPipeRawTabletInsertionEvent(pipeName, 1L, 1); + recreatedDroppedPipeEvent.increaseReferenceCount(WebSocketSink.class.getName()); + recreatedDroppedPipeEvent.setCommitterKeyAndCommitId( + new CommitterKey(pipeName, 1L, 1, -1), 2L); + server.addEvent(recreatedDroppedPipeEvent, connector); + + Assert.assertTrue(recreatedDroppedPipeEvent.isReleased()); + + final PipeRawTabletInsertionEvent recreatedPipeEvent = + createPipeRawTabletInsertionEvent(pipeName, 2L, 1); + recreatedPipeEvent.increaseReferenceCount(WebSocketSink.class.getName()); + recreatedPipeEvent.setCommitterKeyAndCommitId(new CommitterKey(pipeName, 2L, 1, -1), 3L); + server.addEvent(recreatedPipeEvent, connector); + + Assert.assertFalse(recreatedPipeEvent.isReleased()); + } finally { + server.unregister(connector); + } + } + @Test public void testOpcUaSink() { final List schemaList = @@ -181,4 +265,15 @@ public void testOpcUaSink() { Assert.fail(); } } + + private PipeRawTabletInsertionEvent createPipeRawTabletInsertionEvent( + final String pipeName, final long creationTime, final int regionId) { + final List schemaList = + Arrays.asList(new MeasurementSchema("s1", TSDataType.INT64)); + final Tablet tablet = new Tablet("root.db.d" + regionId, schemaList, 1); + tablet.addTimestamp(0, 1L); + tablet.addValue("s1", 0, 1L); + return new PipeRawTabletInsertionEvent( + tablet, false, pipeName, creationTime, null, null, false); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSinkTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSinkTest.java new file mode 100644 index 0000000000000..ef947f13c281b --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataRegionAirGapSinkTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.sink.protocol.airgap; + +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; +import org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant; +import org.apache.iotdb.commons.pipe.config.plugin.configuraion.PipeTaskRuntimeConfiguration; +import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskSinkRuntimeEnvironment; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent; +import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent; +import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferTabletBatchReq; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class IoTDBDataRegionAirGapSinkTest { + + @Test + public void testTransferTabletBatchOverAirGap() throws Exception { + try (final RecordingIoTDBDataRegionAirGapSink sink = new RecordingIoTDBDataRegionAirGapSink()) { + final PipeParameters parameters = buildParameters(false); + sink.validate(new PipeParameterValidator(parameters)); + sink.customize( + parameters, + new PipeTaskRuntimeConfiguration(new PipeTaskSinkRuntimeEnvironment("pipe", 1L, 1))); + sink.prepareSocket(); + + sink.transfer(createPipeRawTabletInsertionEvent("pipe", 1L, 1L)); + sink.transfer(createPipeRawTabletInsertionEvent("pipe", 1L, 2L)); + + Thread.sleep(300L); + sink.transfer(new PipeHeartbeatEvent("1", false)); + + Assert.assertEquals(1, sink.sentRequests.size()); + + final TPipeTransferReq req = toTPipeTransferReq(sink.sentRequests.get(0)); + Assert.assertEquals(PipeRequestType.TRANSFER_TABLET_BATCH.getType(), req.type); + + final PipeTransferTabletBatchReq batchReq = + PipeTransferTabletBatchReq.fromTPipeTransferReq(req); + Assert.assertEquals(2, batchReq.getTabletReqs().size()); + } + } + + @Test + public void testTransferTsFileBatchOverAirGap() throws Exception { + try (final RecordingIoTDBDataRegionAirGapSink sink = new RecordingIoTDBDataRegionAirGapSink()) { + final PipeParameters parameters = buildParameters(true); + sink.validate(new PipeParameterValidator(parameters)); + sink.customize( + parameters, + new PipeTaskRuntimeConfiguration(new PipeTaskSinkRuntimeEnvironment("pipe", 1L, 1))); + sink.prepareSocket(); + + sink.transfer(createPipeRawTabletInsertionEvent("pipe", 1L, 1L)); + sink.transfer(createPipeRawTabletInsertionEvent("pipe", 1L, 2L)); + + Thread.sleep(300L); + sink.transfer(new PipeHeartbeatEvent("1", false)); + + final List requestTypes = new ArrayList<>(); + for (final byte[] requestBytes : sink.sentRequests) { + requestTypes.add(toTPipeTransferReq(requestBytes).type); + } + + Assert.assertTrue(requestTypes.contains(PipeRequestType.TRANSFER_TS_FILE_PIECE.getType())); + Assert.assertTrue(requestTypes.contains(PipeRequestType.TRANSFER_TS_FILE_SEAL.getType())); + Assert.assertFalse(requestTypes.contains(PipeRequestType.TRANSFER_TABLET_RAW.getType())); + Assert.assertFalse(requestTypes.contains(PipeRequestType.TRANSFER_TABLET_BATCH.getType())); + } + } + + private PipeParameters buildParameters(final boolean useTsFileBatch) { + final Map attributes = new HashMap<>(); + attributes.put( + PipeSinkConstant.CONNECTOR_KEY, + BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName()); + attributes.put(PipeSinkConstant.CONNECTOR_IOTDB_NODE_URLS_KEY, "127.0.0.1:6668"); + attributes.put(PipeSinkConstant.CONNECTOR_IOTDB_BATCH_DELAY_MS_KEY, "200"); + attributes.put(PipeSinkConstant.CONNECTOR_IOTDB_BATCH_SIZE_KEY, "1048576"); + if (useTsFileBatch) { + attributes.put(PipeSinkConstant.CONNECTOR_FORMAT_KEY, "tsfile"); + } + return new PipeParameters(attributes); + } + + private PipeRawTabletInsertionEvent createPipeRawTabletInsertionEvent( + final String pipeName, final long creationTime, final long value) { + final List schemaList = + Arrays.asList(new MeasurementSchema("s1", TSDataType.INT64)); + final Tablet tablet = new Tablet("root.db.d1", schemaList, 1); + tablet.addTimestamp(0, value); + tablet.addValue("s1", 0, value); + return new PipeRawTabletInsertionEvent( + tablet, false, pipeName, creationTime, null, null, false); + } + + private static TPipeTransferReq toTPipeTransferReq(final byte[] requestBytes) { + final ByteBuffer buffer = ByteBuffer.wrap(requestBytes); + + final TPipeTransferReq req = new TPipeTransferReq(); + req.version = ReadWriteIOUtils.readByte(buffer); + req.type = ReadWriteIOUtils.readShort(buffer); + req.body = buffer.slice(); + return req; + } + + private static class RecordingIoTDBDataRegionAirGapSink extends IoTDBDataRegionAirGapSink { + + private final List sentRequests = new ArrayList<>(); + + private void prepareSocket() { + sockets.set(0, new TestingAirGapSocket()); + } + + @Override + protected int nextSocketIndex() { + return 0; + } + + @Override + protected boolean sendBytes(final AirGapSocket socket, final byte[] bytes) { + sentRequests.add(Arrays.copyOf(bytes, bytes.length)); + return true; + } + + private static class TestingAirGapSocket extends AirGapSocket { + + private TestingAirGapSocket() { + super("127.0.0.1", 6668); + } + + @Override + public synchronized void setSoTimeout(final int timeout) { + // No-op for unit test. + } + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTrackableHandlerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTrackableHandlerTest.java new file mode 100644 index 0000000000000..60b6923508548 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/handler/PipeTransferTrackableHandlerTest.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.sink.protocol.thrift.async.handler; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.async.AsyncPipeDataTransferServiceClient; +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferSliceReq; +import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; + +import org.apache.thrift.TException; +import org.apache.thrift.async.AsyncMethodCallback; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class PipeTransferTrackableHandlerTest { + + private final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + + private int originalRequestSliceThresholdBytes; + + @Before + public void setUp() { + originalRequestSliceThresholdBytes = commonConfig.getPipeSinkRequestSliceThresholdBytes(); + commonConfig.setPipeSinkRequestSliceThresholdBytes(4); + } + + @After + public void tearDown() { + commonConfig.setPipeSinkRequestSliceThresholdBytes(originalRequestSliceThresholdBytes); + } + + @Test + public void testLargeRequestWillBeSlicedForAsyncTransfer() throws Exception { + final IoTDBDataRegionAsyncSink sink = Mockito.mock(IoTDBDataRegionAsyncSink.class); + final AsyncPipeDataTransferServiceClient client = + Mockito.mock(AsyncPipeDataTransferServiceClient.class); + Mockito.when(client.shouldReturnSelf()).thenReturn(true); + + final List transferredRequests = new ArrayList<>(); + Mockito.doAnswer( + invocation -> { + final TPipeTransferReq req = invocation.getArgument(0); + final AsyncMethodCallback callback = invocation.getArgument(1); + transferredRequests.add(req); + callback.onComplete(successResp()); + return null; + }) + .when(client) + .pipeTransfer(Mockito.any(TPipeTransferReq.class), Mockito.any()); + + final TestPipeTransferTrackableHandler handler = new TestPipeTransferTrackableHandler(sink); + final TPipeTransferReq originalReq = createReq(10); + + handler.transfer(client, originalReq); + + Assert.assertEquals(3, transferredRequests.size()); + Assert.assertEquals(1, handler.completeCount); + Assert.assertEquals(0, handler.errorCount); + + final PipeTransferSliceReq firstSlice = + PipeTransferSliceReq.fromTPipeTransferReq(transferredRequests.get(0)); + final PipeTransferSliceReq secondSlice = + PipeTransferSliceReq.fromTPipeTransferReq(transferredRequests.get(1)); + final PipeTransferSliceReq thirdSlice = + PipeTransferSliceReq.fromTPipeTransferReq(transferredRequests.get(2)); + + Assert.assertEquals( + PipeRequestType.TRANSFER_SLICE.getType(), transferredRequests.get(0).getType()); + Assert.assertEquals(firstSlice.getOrderId(), secondSlice.getOrderId()); + Assert.assertEquals(firstSlice.getOrderId(), thirdSlice.getOrderId()); + Assert.assertEquals(originalReq.getType(), firstSlice.getOriginReqType()); + Assert.assertEquals(10, firstSlice.getOriginBodySize()); + Assert.assertEquals(3, firstSlice.getSliceCount()); + Assert.assertEquals(0, firstSlice.getSliceIndex()); + Assert.assertEquals(1, secondSlice.getSliceIndex()); + Assert.assertEquals(2, thirdSlice.getSliceIndex()); + Assert.assertEquals(4, firstSlice.getSliceBody().length); + Assert.assertEquals(4, secondSlice.getSliceBody().length); + Assert.assertEquals(2, thirdSlice.getSliceBody().length); + + final ArgumentCaptor shouldReturnSelfCaptor = ArgumentCaptor.forClass(Boolean.class); + Mockito.verify(client, Mockito.times(3)).setShouldReturnSelf(shouldReturnSelfCaptor.capture()); + Assert.assertEquals(Arrays.asList(false, false, true), shouldReturnSelfCaptor.getAllValues()); + } + + @Test + public void testLargeRequestFallsBackToWholeRequestWhenSliceTransferFails() throws Exception { + final IoTDBDataRegionAsyncSink sink = Mockito.mock(IoTDBDataRegionAsyncSink.class); + final AsyncPipeDataTransferServiceClient client = + Mockito.mock(AsyncPipeDataTransferServiceClient.class); + Mockito.when(client.shouldReturnSelf()).thenReturn(true); + + final List transferredRequests = new ArrayList<>(); + Mockito.doAnswer( + invocation -> { + final TPipeTransferReq req = invocation.getArgument(0); + final AsyncMethodCallback callback = invocation.getArgument(1); + transferredRequests.add(req); + if (req.getType() == PipeRequestType.TRANSFER_SLICE.getType()) { + callback.onComplete(failedResp()); + } else { + callback.onComplete(successResp()); + } + return null; + }) + .when(client) + .pipeTransfer(Mockito.any(TPipeTransferReq.class), Mockito.any()); + + final TestPipeTransferTrackableHandler handler = new TestPipeTransferTrackableHandler(sink); + final TPipeTransferReq originalReq = createReq(10); + + handler.transfer(client, originalReq); + + Assert.assertEquals(2, transferredRequests.size()); + Assert.assertEquals( + PipeRequestType.TRANSFER_SLICE.getType(), transferredRequests.get(0).getType()); + Assert.assertEquals(originalReq.getType(), transferredRequests.get(1).getType()); + Assert.assertEquals(originalReq.getVersion(), transferredRequests.get(1).getVersion()); + Assert.assertArrayEquals(originalReq.getBody(), transferredRequests.get(1).getBody()); + Assert.assertEquals(1, handler.completeCount); + Assert.assertEquals(0, handler.errorCount); + } + + private static TPipeTransferReq createReq(final int bodySize) { + final byte[] body = new byte[bodySize]; + for (int i = 0; i < body.length; ++i) { + body[i] = (byte) i; + } + + final TPipeTransferReq req = new TPipeTransferReq(); + req.version = IoTDBSinkRequestVersion.VERSION_1.getVersion(); + req.type = (short) 123; + req.body = ByteBuffer.wrap(body); + return req; + } + + private static TPipeTransferResp successResp() { + final TPipeTransferResp resp = new TPipeTransferResp(); + resp.setStatus(new TSStatus().setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode())); + return resp; + } + + private static TPipeTransferResp failedResp() { + final TPipeTransferResp resp = new TPipeTransferResp(); + resp.setStatus( + new TSStatus().setCode(TSStatusCode.PIPE_TRANSFER_SLICE_OUT_OF_ORDER.getStatusCode())); + return resp; + } + + private static class TestPipeTransferTrackableHandler extends PipeTransferTrackableHandler { + + private int completeCount; + private int errorCount; + + private TestPipeTransferTrackableHandler(final IoTDBDataRegionAsyncSink sink) { + super(sink); + } + + private void transfer( + final AsyncPipeDataTransferServiceClient client, final TPipeTransferReq req) + throws TException { + tryTransfer(client, req); + } + + @Override + protected boolean onCompleteInternal(final TPipeTransferResp response) { + completeCount++; + return true; + } + + @Override + protected void onErrorInternal(final Exception exception) { + errorCount++; + } + + @Override + protected void doTransfer( + final AsyncPipeDataTransferServiceClient client, final TPipeTransferReq req) + throws TException { + transferWithOptionalRequestSlicing(client, req); + } + + @Override + public void clearEventsReferenceCount() { + // Do nothing + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java index 9e5b42bb93c4d..2cba4c8eebcaa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.pipe.config.plugin.configuraion.PipeTaskRuntimeConfiguration; import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskSourceRuntimeEnvironment; import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent; import org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionHybridSource; import org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionLogSource; import org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionSource; @@ -39,6 +40,7 @@ import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.iotdb.pipe.api.event.Event; import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; +import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; @@ -63,6 +65,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; public class PipeRealtimeExtractTest { @@ -268,6 +271,52 @@ public void testRealtimeExtractProcess() { } } + @Test + public void testListenToTsFileSkipsAssignerWithoutTsFileSource() throws Exception { + try (final NoTsFileRealtimeDataRegionSource extractor = + new NoTsFileRealtimeDataRegionSource()) { + final PipeParameters parameters = + new PipeParameters( + new HashMap() { + { + put(PipeSourceConstant.EXTRACTOR_PATTERN_KEY, pattern1); + } + }); + final PipeTaskRuntimeConfiguration configuration = + new PipeTaskRuntimeConfiguration( + new PipeTaskSourceRuntimeEnvironment( + "1", + 1, + Integer.parseInt(dataRegion1), + new PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1))); + + extractor.validate(new PipeParameterValidator(parameters)); + extractor.customize(parameters, configuration); + extractor.start(); + + final File dataRegionDir = + new File(tsFileDir.getPath() + File.separator + dataRegion1 + File.separator + "0"); + final boolean ignored = dataRegionDir.mkdirs(); + final File tsFile = new File(dataRegionDir, "0-0-0-0.tsfile"); + Assert.assertTrue(tsFile.createNewFile()); + + final TsFileResource resource = new TsFileResource(tsFile); + resource.updateStartTime( + new PlainDeviceID(String.join(TsFileConstant.PATH_SEPARATOR, device)), 0); + resource.close(); + + PipeInsertionDataNodeListener.getInstance().listenToTsFile(dataRegion1, resource, false); + + final long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(1); + while (System.currentTimeMillis() < deadline + && extractor.getObservedTsFileEventCount() == 0) { + TimeUnit.MILLISECONDS.sleep(10); + } + + Assert.assertEquals(0, extractor.getObservedTsFileEventCount()); + } + } + private Future write2DataRegion( final int writeNum, final String dataRegionId, final int startNum) { final File dataRegionDir = @@ -351,4 +400,36 @@ private Future listen( } }); } + + private static class NoTsFileRealtimeDataRegionSource extends PipeRealtimeDataRegionSource { + + private final AtomicInteger observedTsFileEventCount = new AtomicInteger(0); + + @Override + public Event supply() { + return null; + } + + @Override + protected void doExtract(final PipeRealtimeEvent event) { + if (event.getEvent() instanceof TsFileInsertionEvent) { + observedTsFileEventCount.incrementAndGet(); + } + event.decreaseReferenceCount(NoTsFileRealtimeDataRegionSource.class.getName(), false); + } + + @Override + public boolean isNeedListenToTsFile() { + return false; + } + + @Override + public boolean isNeedListenToInsertNode() { + return false; + } + + private int getObservedTsFileEventCount() { + return observedTsFileEventCount.get(); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/DisruptorShutdownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/DisruptorShutdownTest.java new file mode 100644 index 0000000000000..ef57ea625aef6 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/disruptor/DisruptorShutdownTest.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.pipe.source.dataregion.realtime.disruptor; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class DisruptorShutdownTest { + + @Test + public void testBatchEventProcessorDrainsPublishedEventsOnShutdownInterrupt() throws Exception { + final RingBuffer ringBuffer = RingBuffer.createMultiProducer(TestEvent::new, 32); + ringBuffer.publishEvent((event, sequence, value) -> event.value = value, 1); + + final TestSequenceBarrier barrier = new TestSequenceBarrier(0L); + final AtomicInteger handledEventCount = new AtomicInteger(); + final BatchEventProcessor processor = + new BatchEventProcessor<>( + ringBuffer, + barrier, + (event, sequence, endOfBatch) -> handledEventCount.incrementAndGet()); + + final Thread processorThread = new Thread(processor, "pipe-batch-event-processor-test"); + processorThread.start(); + + Assert.assertTrue(barrier.awaitWaitForCall()); + processor.halt(); + barrier.interruptWait(); + + processorThread.join(TimeUnit.SECONDS.toMillis(5)); + + Assert.assertFalse(processorThread.isAlive()); + Assert.assertEquals(1, handledEventCount.get()); + Assert.assertEquals(0L, processor.getSequence().get()); + } + + @Test + public void testBatchEventProcessorDrainsEventsPublishedAfterCurrentBatchWhenHalting() + throws Exception { + final RingBuffer ringBuffer = RingBuffer.createMultiProducer(TestEvent::new, 32); + ringBuffer.publishEvent((event, sequence, value) -> event.value = value, 1); + ringBuffer.publishEvent((event, sequence, value) -> event.value = value, 2); + + final SnapshotSequenceBarrier barrier = new SnapshotSequenceBarrier(0L, 1L); + final AtomicInteger handledEventCount = new AtomicInteger(); + final AtomicReference> processorReference = + new AtomicReference<>(); + final BatchEventProcessor processor = + new BatchEventProcessor<>( + ringBuffer, + barrier, + (event, sequence, endOfBatch) -> { + handledEventCount.incrementAndGet(); + if (event.value == 1) { + processorReference.get().halt(); + } + }); + processorReference.set(processor); + + final Thread processorThread = + new Thread(processor, "pipe-batch-event-processor-snapshot-test"); + processorThread.start(); + processorThread.join(TimeUnit.SECONDS.toMillis(5)); + + Assert.assertFalse(processorThread.isAlive()); + Assert.assertEquals(2, handledEventCount.get()); + Assert.assertEquals(1L, processor.getSequence().get()); + } + + @Test + public void testDisruptorShutdownInterruptsWaitingProcessor() throws Exception { + final AtomicReference processorThreadReference = new AtomicReference<>(); + final ThreadFactory threadFactory = + runnable -> { + final Thread thread = new Thread(runnable, "pipe-disruptor-shutdown-test"); + processorThreadReference.set(thread); + return thread; + }; + + final Disruptor disruptor = new Disruptor<>(TestEvent::new, 32, threadFactory); + disruptor.handleEventsWith((event, sequence, endOfBatch) -> {}); + disruptor.start(); + + final Thread processorThread = processorThreadReference.get(); + Assert.assertNotNull(processorThread); + + TimeUnit.MILLISECONDS.sleep(50); + disruptor.shutdown(); + + Assert.assertFalse(processorThread.isAlive()); + } + + @Test + public void testUnexpectedInterruptDoesNotStopProcessor() throws Exception { + final AtomicReference processorThreadReference = new AtomicReference<>(); + final ThreadFactory threadFactory = + runnable -> { + final Thread thread = new Thread(runnable, "pipe-disruptor-unexpected-interrupt-test"); + processorThreadReference.set(thread); + return thread; + }; + + final CountDownLatch handled = new CountDownLatch(1); + final Disruptor disruptor = new Disruptor<>(TestEvent::new, 32, threadFactory); + final RingBuffer ringBuffer = + disruptor.handleEventsWith((event, sequence, endOfBatch) -> handled.countDown()).start(); + + final Thread processorThread = processorThreadReference.get(); + Assert.assertNotNull(processorThread); + + TimeUnit.MILLISECONDS.sleep(50); + processorThread.interrupt(); + + ringBuffer.publishEvent((event, sequence, value) -> event.value = value, 1); + Assert.assertTrue(handled.await(5, TimeUnit.SECONDS)); + Assert.assertTrue(processorThread.isAlive()); + + disruptor.shutdown(); + Assert.assertFalse(processorThread.isAlive()); + } + + @Test + public void testPublishEventCanAbortWhenClosingWhileBufferIsFull() throws Exception { + final RingBuffer ringBuffer = RingBuffer.createMultiProducer(TestEvent::new, 1); + final Sequence gatingSequence = new Sequence(); + ringBuffer.addGatingSequences(gatingSequence); + ringBuffer.publishEvent((event, sequence, value) -> event.value = value, 1); + + final AtomicBoolean isClosed = new AtomicBoolean(false); + final AtomicBoolean published = new AtomicBoolean(true); + final Thread publisherThread = + new Thread( + () -> + published.set( + ringBuffer.publishEvent( + (event, sequence, value) -> event.value = value, 2, isClosed::get)), + "pipe-disruptor-publish-abort-test"); + + publisherThread.start(); + TimeUnit.MILLISECONDS.sleep(50); + isClosed.set(true); + publisherThread.join(TimeUnit.SECONDS.toMillis(5)); + + Assert.assertFalse(publisherThread.isAlive()); + Assert.assertFalse(published.get()); + } + + private static class TestEvent { + private int value; + } + + private static class TestSequenceBarrier extends SequenceBarrier { + + private final long cursor; + private final CountDownLatch waitForCalled = new CountDownLatch(1); + private final CountDownLatch interruptWait = new CountDownLatch(1); + + private TestSequenceBarrier(final long cursor) { + super(new MultiProducerSequencer(32, new Sequence[0]), new Sequence[0]); + this.cursor = cursor; + } + + @Override + public long waitFor(final long sequence) throws InterruptedException { + waitForCalled.countDown(); + interruptWait.await(); + throw new InterruptedException(); + } + + @Override + public long getCursor() { + return cursor; + } + + @Override + public long getHighestPublishedSequence(final long lowerBound, final long availableSequence) { + return availableSequence; + } + + private boolean awaitWaitForCall() throws InterruptedException { + return waitForCalled.await(5, TimeUnit.SECONDS); + } + + private void interruptWait() { + interruptWait.countDown(); + } + } + + private static class SnapshotSequenceBarrier extends SequenceBarrier { + + private final long waitForResult; + private final long cursor; + + private SnapshotSequenceBarrier(final long waitForResult, final long cursor) { + super(new MultiProducerSequencer(32, new Sequence[0]), new Sequence[0]); + this.waitForResult = waitForResult; + this.cursor = cursor; + } + + @Override + public long waitFor(final long sequence) { + return waitForResult; + } + + @Override + public long getCursor() { + return cursor; + } + + @Override + public long getHighestPublishedSequence(final long lowerBound, final long availableSequence) { + return availableSequence; + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/LocalSinkChannelTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/LocalSinkChannelTest.java index 8d9c4b9cd6569..b3c50aaa517c5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/LocalSinkChannelTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/LocalSinkChannelTest.java @@ -92,13 +92,14 @@ public void testSend() { Assert.assertFalse(localSinkChannel.isFinished()); Assert.assertEquals(11 * mockTsBlockSize, localSinkChannel.getBufferRetainedSizeInBytes()); Mockito.verify(spyMemoryPool, Mockito.times(11)) - .reserve( + .reserveWithPriority( queryId, FragmentInstanceId.createFragmentInstanceIdFromTFragmentInstanceId( remoteFragmentInstanceId), remotePlanNodeId, mockTsBlockSize, - Long.MAX_VALUE); + Long.MAX_VALUE, + false); // Receive TsBlocks. int numOfReceivedTsblocks = 0; @@ -184,13 +185,14 @@ public void testAbort() { Assert.assertFalse(localSinkChannel.isFinished()); Assert.assertEquals(11 * mockTsBlockSize, localSinkChannel.getBufferRetainedSizeInBytes()); Mockito.verify(spyMemoryPool, Mockito.times(11)) - .reserve( + .reserveWithPriority( queryId, FragmentInstanceId.createFragmentInstanceIdFromTFragmentInstanceId( remoteFragmentInstanceId), remotePlanNodeId, mockTsBlockSize, - Long.MAX_VALUE); + Long.MAX_VALUE, + false); // Abort. localSinkChannel.abort(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueueTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueueTest.java index 46196d1c990d1..a95fd35ba76c9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueueTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueueTest.java @@ -21,13 +21,13 @@ import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool; +import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool.MemoryReservationResult; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import org.apache.commons.lang3.Validate; import org.apache.tsfile.read.common.block.TsBlock; -import org.apache.tsfile.utils.Pair; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -62,15 +62,16 @@ public void testAsyncListenerAfterAbortDoesNotAddTsBlock() { MemoryPool mockMemoryPool = Mockito.mock(MemoryPool.class); Mockito.when(mockLocalMemoryManager.getQueryPool()).thenReturn(mockMemoryPool); - // reserve() returns (manualFuture, false) — simulating memory blocked + // reserveWithPriority() returns blocked future and reserve failure. Mockito.when( - mockMemoryPool.reserve( + mockMemoryPool.reserveWithPriority( Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyLong(), - Mockito.anyLong())) - .thenReturn(new Pair<>(manualFuture, Boolean.FALSE)); + Mockito.anyLong(), + Mockito.anyBoolean())) + .thenReturn(new MemoryReservationResult(manualFuture, false, 1024L)); // tryCancel returns 0 — simulating future already completed (can't cancel) Mockito.when(mockMemoryPool.tryCancel(Mockito.any())).thenReturn(0L); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/ShuffleSinkHandleTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/ShuffleSinkHandleTest.java index 489ec5c9ed607..4190c2fa61abe 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/ShuffleSinkHandleTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/ShuffleSinkHandleTest.java @@ -104,13 +104,14 @@ public void testAbort() { Assert.assertFalse(localSinkChannel.isFinished()); Assert.assertEquals(11 * mockTsBlockSize, localSinkChannel.getBufferRetainedSizeInBytes()); Mockito.verify(spyMemoryPool, Mockito.times(11)) - .reserve( + .reserveWithPriority( queryId, FragmentInstanceId.createFragmentInstanceIdFromTFragmentInstanceId( remoteFragmentInstanceId), remotePlanNodeId, mockTsBlockSize, - Long.MAX_VALUE); + Long.MAX_VALUE, + false); // Abort. shuffleSinkHandle.abort(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java index ee8493372065a..40721c93db785 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SourceHandleTest.java @@ -254,13 +254,14 @@ public void testBlockedOneTimeReceive() { .collect(Collectors.toList())); try { Mockito.verify(spyMemoryPool, Mockito.timeout(10_000).times(6)) - .reserve( + .reserveWithPriority( queryId, FragmentInstanceId.createFragmentInstanceIdFromTFragmentInstanceId( localFragmentInstanceId), localPlanNodeId, MOCK_TSBLOCK_SIZE, - maxBytesCanReserve); + maxBytesCanReserve, + false); Mockito.verify(mockClient, Mockito.timeout(10_0000).times(1)) .getDataBlock( Mockito.argThat( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java index 327d4a34c39e7..b09498ad949dd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java @@ -20,11 +20,11 @@ package org.apache.iotdb.db.queryengine.execution.exchange; import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool; +import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool.MemoryReservationResult; import com.google.common.util.concurrent.SettableFuture; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.column.TsBlockSerde; -import org.apache.tsfile.utils.Pair; import org.mockito.Mockito; import org.mockito.stubbing.Answer; @@ -68,21 +68,25 @@ public static MemoryPool createMockBlockedMemoryPool( settableFuture.get().set(null); AtomicReference reservedBytes = new AtomicReference<>(0L); Mockito.when( - mockMemoryPool.reserve( + mockMemoryPool.reserveWithPriority( Mockito.eq(queryId), Mockito.eq(fragmentInstanceId), Mockito.eq(planNodeId), Mockito.anyLong(), - Mockito.anyLong())) + Mockito.anyLong(), + Mockito.anyBoolean())) .thenAnswer( invocation -> { long bytesToReserve = invocation.getArgument(3); if (reservedBytes.get() + bytesToReserve <= capacityInBytes) { - reservedBytes.updateAndGet(v -> v + (long) invocation.getArgument(3)); - return new Pair<>(settableFuture.get(), true); + reservedBytes.updateAndGet(v -> v + bytesToReserve); + return new MemoryReservationResult(settableFuture.get(), true, bytesToReserve); } else { + if (invocation.getArgument(5)) { + return new MemoryReservationResult(settableFuture.get(), true, 0L); + } settableFuture.set(SettableFuture.create()); - return new Pair<>(settableFuture.get(), false); + return new MemoryReservationResult(settableFuture.get(), false, bytesToReserve); } }); Mockito.doAnswer( @@ -124,13 +128,17 @@ public static MemoryPool createMockBlockedMemoryPool( public static MemoryPool createMockNonBlockedMemoryPool() { MemoryPool mockMemoryPool = Mockito.mock(MemoryPool.class); Mockito.when( - mockMemoryPool.reserve( + mockMemoryPool.reserveWithPriority( Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyLong(), - Mockito.anyLong())) - .thenReturn(new Pair<>(immediateFuture(null), true)); + Mockito.anyLong(), + Mockito.anyBoolean())) + .thenAnswer( + invocation -> + new MemoryReservationResult( + immediateFuture(null), true, invocation.getArgument(3))); Mockito.when( mockMemoryPool.tryReserve( Mockito.anyString(), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java index 30b9df14b3acb..cfc7f887dcfd3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java @@ -44,7 +44,7 @@ import org.apache.iotdb.db.utils.datastructure.AlignedTVList; import org.apache.iotdb.db.utils.datastructure.TVList; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -133,19 +133,19 @@ public void testTVListOwnerTransfer() throws InterruptedException { FragmentInstanceExecution execution1 = createFragmentInstanceExecution(1, instanceNotificationExecutor); FragmentInstanceContext fragmentInstanceContext1 = execution1.getFragmentInstanceContext(); - fragmentInstanceContext1.addTVListToSet(ImmutableMap.of(tvList, 0)); + fragmentInstanceContext1.addTVListToSet(ImmutableSet.of(tvList)); tvList.getQueryContextSet().add(fragmentInstanceContext1); FragmentInstanceExecution execution2 = createFragmentInstanceExecution(2, instanceNotificationExecutor); FragmentInstanceContext fragmentInstanceContext2 = execution2.getFragmentInstanceContext(); - fragmentInstanceContext2.addTVListToSet(ImmutableMap.of(tvList, 0)); + fragmentInstanceContext2.addTVListToSet(ImmutableSet.of(tvList)); tvList.getQueryContextSet().add(fragmentInstanceContext2); // mock flush's behavior fragmentInstanceContext1 .getMemoryReservationContext() - .reserveMemoryCumulatively(tvList.calculateRamSize()); + .reserveMemoryCumulatively(tvList.calculateRamSize().getRamSize()); tvList.setOwnerQuery(fragmentInstanceContext1); fragmentInstanceContext1.decrementNumOfUnClosedDriver(); @@ -226,7 +226,7 @@ public void testTVListCloneForQuery() { pointReader.nextTimeValuePair(); } assertTrue(tvList.isSorted()); - assertEquals(tvList.calculateRamSize(), tvList.getReservedMemoryBytes()); + assertEquals(tvList.calculateRamSize().getRamSize(), tvList.getReservedMemoryBytes()); } catch (QueryProcessException | IOException | MetadataException diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java index bae3c0643f45c..cf7e15dc71f46 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/memory/MemoryPoolTest.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.queryengine.execution.memory; +import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool.MemoryReservationResult; + import com.google.common.util.concurrent.ListenableFuture; import org.junit.Assert; import org.junit.Before; @@ -94,7 +96,9 @@ public void testOverTryReserve() { public void testReserve() { ListenableFuture future = - pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256L, Long.MAX_VALUE).left; + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256L, Long.MAX_VALUE, false) + .getFuture(); Assert.assertTrue(future.isDone()); Assert.assertEquals(256L, pool.getQueryMemoryReservedBytes(QUERY_ID)); Assert.assertEquals(256L, pool.getReservedBytes()); @@ -104,7 +108,8 @@ public void testReserve() { public void tesReserveZero() { try { - pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 0L, Long.MAX_VALUE); + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 0L, Long.MAX_VALUE, false); Assert.fail("Expect IllegalArgumentException"); } catch (IllegalArgumentException ignore) { } @@ -114,7 +119,8 @@ public void tesReserveZero() { public void testReserveNegative() { try { - pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, -1L, Long.MAX_VALUE); + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, -1L, Long.MAX_VALUE, false); Assert.fail("Expect IllegalArgumentException"); } catch (IllegalArgumentException ignore) { } @@ -124,7 +130,9 @@ public void testReserveNegative() { public void testReserveAll() { ListenableFuture future = - pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, Long.MAX_VALUE).left; + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, Long.MAX_VALUE, false) + .getFuture(); Assert.assertTrue(future.isDone()); Assert.assertEquals(512L, pool.getQueryMemoryReservedBytes(QUERY_ID)); Assert.assertEquals(512L, pool.getReservedBytes()); @@ -134,11 +142,15 @@ public void testReserveAll() { public void testOverReserve() { ListenableFuture future = - pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256L, Long.MAX_VALUE).left; + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256L, Long.MAX_VALUE, false) + .getFuture(); Assert.assertTrue(future.isDone()); Assert.assertEquals(256L, pool.getQueryMemoryReservedBytes(QUERY_ID)); Assert.assertEquals(256L, pool.getReservedBytes()); - future = pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, 513L).left; + future = + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, 513L, false) + .getFuture(); Assert.assertFalse(future.isDone()); Assert.assertEquals(256L, pool.getQueryMemoryReservedBytes(QUERY_ID)); Assert.assertEquals(256L, pool.getReservedBytes()); @@ -148,11 +160,13 @@ public void testOverReserve() { public void testReserveAndFree() { Assert.assertTrue( - pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, Long.MAX_VALUE) - .left + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, Long.MAX_VALUE, false) + .getFuture() .isDone()); ListenableFuture future = - pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, 513L).left; + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, 513L, false) + .getFuture(); Assert.assertFalse(future.isDone()); Assert.assertEquals(512L, pool.getQueryMemoryReservedBytes(QUERY_ID)); Assert.assertEquals(512L, pool.getReservedBytes()); @@ -166,18 +180,22 @@ public void testReserveAndFree() { public void testMultiReserveAndFree() { Assert.assertTrue( - pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256L, Long.MAX_VALUE) - .left + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256L, Long.MAX_VALUE, false) + .getFuture() .isDone()); Assert.assertEquals(256L, pool.getQueryMemoryReservedBytes(QUERY_ID)); Assert.assertEquals(256L, pool.getReservedBytes()); ListenableFuture future1 = - pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, 513L).left; + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, 513L, false) + .getFuture(); ListenableFuture future2 = - pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, 513L).left; + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, 513L, false) + .getFuture(); ListenableFuture future3 = - pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, 513L).left; + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, 513L, false) + .getFuture(); Assert.assertFalse(future1.isDone()); Assert.assertFalse(future2.isDone()); Assert.assertFalse(future3.isDone()); @@ -284,7 +302,8 @@ public void testTryCancelBlockedReservation() { pool.tryReserveForTest(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, Long.MAX_VALUE)); ListenableFuture f = - pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256L, 512L).left; + pool.reserveWithPriority(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256L, 512L, false) + .getFuture(); Assert.assertFalse(f.isDone()); // Cancel the reservation. Assert.assertEquals(256L, pool.tryCancel(f)); @@ -296,11 +315,76 @@ public void testTryCancelBlockedReservation() { public void testTryCancelCompletedReservation() { ListenableFuture f = - pool.reserve(QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256L, Long.MAX_VALUE).left; + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256L, Long.MAX_VALUE, false) + .getFuture(); Assert.assertTrue(f.isDone()); // Cancel the reservation. Assert.assertEquals(0L, pool.tryCancel(f)); Assert.assertTrue(f.isDone()); Assert.assertFalse(f.isCancelled()); } + + /** + * Normal query: requested bytes exceed what the pool can still provide — reserve fails (blocked + * future, not immediate success). + */ + @Test + public void testReserveWithPriorityNormalQueryExceedsAvailable() { + MemoryReservationResult r1 = + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, Long.MAX_VALUE, false); + Assert.assertTrue(r1.isReserveSuccess()); + Assert.assertEquals(512L, r1.getReservedBytes()); + Assert.assertTrue(r1.getFuture().isDone()); + + MemoryReservationResult r2 = + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, Long.MAX_VALUE, false); + Assert.assertTrue(r2.isReserveSuccess()); + Assert.assertEquals(512L, r2.getReservedBytes()); + Assert.assertEquals(1024L, pool.getReservedBytes()); + + MemoryReservationResult r3 = + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256L, Long.MAX_VALUE, false); + Assert.assertFalse(r3.isReserveSuccess()); + Assert.assertEquals(256L, r3.getReservedBytes()); + Assert.assertFalse(r3.getFuture().isDone()); + Assert.assertEquals(1024L, pool.getReservedBytes()); + } + + /** SHOW QUERIES path: exceeds pool capacity — treated as success with zero bytes from pool. */ + @Test + public void testReserveWithPriorityShowQueriesExceedsAvailable() { + Assert.assertTrue( + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, Long.MAX_VALUE, false) + .isReserveSuccess()); + Assert.assertTrue( + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 512L, Long.MAX_VALUE, false) + .isReserveSuccess()); + Assert.assertEquals(1024L, pool.getReservedBytes()); + + MemoryReservationResult r = + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256L, Long.MAX_VALUE, true); + Assert.assertTrue(r.isReserveSuccess()); + Assert.assertEquals(0L, r.getReservedBytes()); + Assert.assertTrue(r.getFuture().isDone()); + Assert.assertEquals(1024L, pool.getReservedBytes()); + } + + /** SHOW QUERIES path: pool has room — same as normal successful reserve. */ + @Test + public void testReserveWithPriorityShowQueriesWithinAvailable() { + MemoryReservationResult r = + pool.reserveWithPriority( + QUERY_ID, FRAGMENT_INSTANCE_ID, PLAN_NODE_ID, 256L, Long.MAX_VALUE, true); + Assert.assertTrue(r.isReserveSuccess()); + Assert.assertEquals(256L, r.getReservedBytes()); + Assert.assertTrue(r.getFuture().isDone()); + Assert.assertEquals(256L, pool.getReservedBytes()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/AlignedSeriesAggregationScanOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/AlignedSeriesAggregationScanOperatorTest.java index 2a5c5c13d0b37..bc81047b17a36 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/AlignedSeriesAggregationScanOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/AlignedSeriesAggregationScanOperatorTest.java @@ -69,6 +69,7 @@ import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext.createFragmentInstanceContext; import static org.apache.iotdb.db.queryengine.execution.operator.AggregationOperatorTest.TEST_TIME_SLICE; import static org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil.initTimeRangeIterator; +import static org.apache.iotdb.db.queryengine.execution.operator.OperatorTestUtils.nextNonNullOrEmpty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -122,7 +123,7 @@ public void testAggregationWithoutTimeFilter() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, null, true, null); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); for (int i = 0; i < measurementSchemas.size(); i++) { assertEquals(500, resultTsBlock.getColumn(i).getLong(0)); } @@ -153,7 +154,7 @@ public void testAggregationWithoutTimeFilterOrderByTimeDesc() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, null, false, null); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); for (int i = 0; i < measurementSchemas.size(); i++) { assertEquals(500, resultTsBlock.getColumn(i).getLong(0)); } @@ -189,7 +190,7 @@ public void testMultiAggregationFuncWithoutTimeFilter1() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, null, true, null); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); assertEquals(500, resultTsBlock.getColumn(0).getLong(0)); assertEquals(6524750.0, resultTsBlock.getColumn(1).getDouble(0), 0.0001); count++; @@ -227,7 +228,7 @@ public void testMultiAggregationFuncWithoutTimeFilter2() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, null, true, null); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); assertTrue(resultTsBlock.getColumn(0).getBoolean(0)); assertEquals(10499, resultTsBlock.getColumn(1).getInt(0)); assertEquals(20199, resultTsBlock.getColumn(2).getLong(0)); @@ -269,7 +270,7 @@ public void testMultiAggregationFuncWithoutTimeFilterOrderByTimeDesc() throws Ex initAlignedSeriesAggregationScanOperator(aggregators, null, false, null); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); assertTrue(resultTsBlock.getColumn(0).getBoolean(0)); assertEquals(10499, resultTsBlock.getColumn(1).getInt(0)); assertEquals(20199, resultTsBlock.getColumn(2).getLong(0)); @@ -304,7 +305,7 @@ public void testAggregationWithTimeFilter1() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, timeFilter, true, null); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); for (int i = 0; i < measurementSchemas.size(); i++) { assertEquals(resultTsBlock.getColumn(i).getLong(0), 380); } @@ -337,7 +338,7 @@ public void testAggregationWithTimeFilter2() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, timeFilter, true, null); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); for (int i = 0; i < measurementSchemas.size(); i++) { assertEquals(resultTsBlock.getColumn(i).getLong(0), 380); } @@ -370,7 +371,7 @@ public void testAggregationWithTimeFilter3() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, timeFilter, true, null); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); for (int i = 0; i < measurementSchemas.size(); i++) { assertEquals(resultTsBlock.getColumn(i).getLong(0), 300); } @@ -410,7 +411,7 @@ public void testMultiAggregationWithTimeFilter() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, timeFilter, true, null); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); assertTrue(resultTsBlock.getColumn(0).getBoolean(0)); assertEquals(399, resultTsBlock.getColumn(1).getInt(0)); assertEquals(20199, resultTsBlock.getColumn(2).getLong(0)); @@ -448,7 +449,7 @@ public void testGroupByWithoutGlobalTimeFilter() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, null, true, groupByTimeParameter); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(100 * count, resultTsBlock.getTimeColumn().getLong(pos)); @@ -489,7 +490,7 @@ public void testGroupByWithGlobalTimeFilter() throws Exception { aggregators, timeFilter, true, groupByTimeParameter); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(100 * count, resultTsBlock.getTimeColumn().getLong(pos)); @@ -533,7 +534,7 @@ public void testGroupByWithMultiFunction() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, null, true, groupByTimeParameter); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(100 * count, resultTsBlock.getTimeColumn().getLong(pos)); @@ -578,7 +579,7 @@ public void testGroupByWithMultiFunctionOrderByTimeDesc() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, null, false, groupByTimeParameter); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(100 * (3 - count), resultTsBlock.getTimeColumn().getLong(pos)); @@ -613,7 +614,7 @@ public void testGroupBySlidingTimeWindow() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, null, true, groupByTimeParameter); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(50 * count, resultTsBlock.getTimeColumn().getLong(pos)); @@ -645,7 +646,7 @@ public void testGroupBySlidingTimeWindow2() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, null, true, groupByTimeParameter); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(timeColumn[count], resultTsBlock.getTimeColumn().getLong(pos)); @@ -688,7 +689,7 @@ public void testGroupBySlidingWindowWithMultiFunction() throws Exception { initAlignedSeriesAggregationScanOperator(aggregators, null, true, groupByTimeParameter); int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(timeColumn[count], resultTsBlock.getTimeColumn().getLong(pos)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/FillOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/FillOperatorTest.java index e2bb5a655ec13..ac3c83a8364de 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/FillOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/FillOperatorTest.java @@ -42,6 +42,7 @@ import java.util.concurrent.ExecutorService; import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext.createFragmentInstanceContext; +import static org.apache.iotdb.db.queryengine.execution.operator.OperatorTestUtils.nextNonNullOrEmpty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -209,7 +210,7 @@ public long ramBytesUsed() { } }; while (fillOperator.hasNext()) { - TsBlock block = fillOperator.next(); + TsBlock block = nextNonNullOrEmpty(fillOperator); for (int i = 0; i < block.getPositionCount(); i++) { long expectedTime = i + 1 + count * 10000L; assertEquals(expectedTime, block.getTimeByIndex(i)); @@ -386,7 +387,7 @@ public long ramBytesUsed() { } }; while (fillOperator.hasNext()) { - TsBlock block = fillOperator.next(); + TsBlock block = nextNonNullOrEmpty(fillOperator); for (int i = 0; i < block.getPositionCount(); i++) { long expectedTime = i + 1 + count * 10000L; assertEquals(expectedTime, block.getTimeByIndex(i)); @@ -563,7 +564,7 @@ public long ramBytesUsed() { } }; while (fillOperator.hasNext()) { - TsBlock block = fillOperator.next(); + TsBlock block = nextNonNullOrEmpty(fillOperator); for (int i = 0; i < block.getPositionCount(); i++) { long expectedTime = i + 1 + count * 10000L; assertEquals(expectedTime, block.getTimeByIndex(i)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/HorizontallyConcatOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/HorizontallyConcatOperatorTest.java index bad50a7e1c9cf..ec258a36c0e94 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/HorizontallyConcatOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/HorizontallyConcatOperatorTest.java @@ -66,6 +66,7 @@ import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext.createFragmentInstanceContext; import static org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil.initTimeRangeIterator; +import static org.apache.iotdb.db.queryengine.execution.operator.OperatorTestUtils.nextNonNullOrEmpty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -197,7 +198,7 @@ public void batchTest1() throws Exception { int count = 0; while (horizontallyConcatOperator.isBlocked().isDone() && horizontallyConcatOperator.hasNext()) { - TsBlock tsBlock = horizontallyConcatOperator.next(); + TsBlock tsBlock = nextNonNullOrEmpty(horizontallyConcatOperator); assertEquals(6, tsBlock.getValueColumnCount()); for (int i = 0; i < tsBlock.getPositionCount(); i++, count++) { assertEquals(count, tsBlock.getTimeByIndex(i)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LimitOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LimitOperatorTest.java index 31dda8933581d..4b4a5329f8a43 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LimitOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LimitOperatorTest.java @@ -157,6 +157,9 @@ public void batchTest() throws Exception { int count = 0; while (limitOperator.isBlocked().isDone() && limitOperator.hasNext()) { TsBlock tsBlock = limitOperator.next(); + if (tsBlock == null) { + continue; + } assertEquals(2, tsBlock.getValueColumnCount()); assertTrue(tsBlock.getColumn(0) instanceof IntColumn); assertTrue(tsBlock.getColumn(1) instanceof IntColumn); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LinearFillOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LinearFillOperatorTest.java index 08859dc127be6..8ea63f132e9ad 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LinearFillOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/LinearFillOperatorTest.java @@ -41,8 +41,8 @@ import java.util.concurrent.ExecutorService; import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext.createFragmentInstanceContext; +import static org.apache.iotdb.db.queryengine.execution.operator.OperatorTestUtils.nextNonNullOrEmpty; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; public class LinearFillOperatorTest { @@ -1301,8 +1301,7 @@ public long ramBytesUsed() { }; while (fillOperator.hasNext()) { - TsBlock block = fillOperator.next(); - assertNotNull(block); + TsBlock block = nextNonNullOrEmpty(fillOperator); for (int i = 0; i < block.getPositionCount(); i++) { long expectedTime = i + count; assertEquals(expectedTime, block.getTimeByIndex(i)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeSortOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeSortOperatorTest.java index 76ec370ad98be..9afc9c7c5409e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeSortOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeSortOperatorTest.java @@ -88,6 +88,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext.createFragmentInstanceContext; +import static org.apache.iotdb.db.queryengine.execution.operator.OperatorTestUtils.nextNonNullOrEmpty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -1765,7 +1766,7 @@ public void mergeSortTest() throws Exception { int index = 0; while (mergeSortOperator.isBlocked().isDone() && mergeSortOperator.hasNext()) { - TsBlock result = mergeSortOperator.next(); + TsBlock result = nextNonNullOrEmpty(mergeSortOperator); for (int i = 0; i < result.getPositionCount(); i++) { long time = result.getTimeByIndex(i); assertEquals(time, ans[index++]); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OffsetOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OffsetOperatorTest.java index a1188becf1096..6602c65ff981a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OffsetOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OffsetOperatorTest.java @@ -62,6 +62,7 @@ import java.util.concurrent.TimeUnit; import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext.createFragmentInstanceContext; +import static org.apache.iotdb.db.queryengine.execution.operator.OperatorTestUtils.nextNonNullOrEmpty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -161,7 +162,7 @@ public void batchTest1() throws Exception { new LimitOperator(driverContext.getOperatorContexts().get(4), 250, offsetOperator); int count = 100; while (limitOperator.isBlocked().isDone() && limitOperator.hasNext()) { - TsBlock tsBlock = limitOperator.next(); + TsBlock tsBlock = nextNonNullOrEmpty(limitOperator); assertEquals(2, tsBlock.getValueColumnCount()); assertTrue(tsBlock.getColumn(0) instanceof IntColumn); assertTrue(tsBlock.getColumn(1) instanceof IntColumn); @@ -265,7 +266,7 @@ public void batchTest2() throws Exception { int count = 0; while (offsetOperator.isBlocked().isDone() && offsetOperator.hasNext()) { - TsBlock tsBlock = offsetOperator.next(); + TsBlock tsBlock = nextNonNullOrEmpty(offsetOperator); assertEquals(2, tsBlock.getValueColumnCount()); assertTrue(tsBlock.getColumn(0) instanceof IntColumn); assertTrue(tsBlock.getColumn(1) instanceof IntColumn); @@ -367,7 +368,7 @@ public void batchTest3() throws Exception { new OffsetOperator(driverContext.getOperatorContexts().get(3), 500, timeJoinOperator); while (offsetOperator.isBlocked().isDone() && offsetOperator.hasNext()) { - TsBlock tsBlock = offsetOperator.next(); + TsBlock tsBlock = nextNonNull(offsetOperator); assertEquals(2, tsBlock.getValueColumnCount()); assertTrue(tsBlock.getColumn(0) instanceof IntColumn); assertTrue(tsBlock.getColumn(1) instanceof IntColumn); @@ -455,7 +456,7 @@ public void batchTest4() throws Exception { driverContext.getOperatorContexts().get(3), 98_784_247_808L, timeJoinOperator); while (offsetOperator.isBlocked().isDone() && offsetOperator.hasNext()) { - TsBlock tsBlock = offsetOperator.next(); + TsBlock tsBlock = nextNonNull(offsetOperator); assertEquals(2, tsBlock.getValueColumnCount()); assertTrue(tsBlock.getColumn(0) instanceof IntColumn); assertTrue(tsBlock.getColumn(1) instanceof IntColumn); @@ -468,4 +469,14 @@ public void batchTest4() throws Exception { instanceNotificationExecutor.shutdown(); } } + + private static TsBlock nextNonNull(Operator operator) throws Exception { + while (operator.hasNext()) { + TsBlock result = operator.next(); + if (result != null) { + return result; + } + } + throw new AssertionError("Expected a non-null TsBlock from operator"); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorTestUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorTestUtils.java new file mode 100644 index 0000000000000..37b979e87bec1 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorTestUtils.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.queryengine.execution.operator; + +import org.apache.tsfile.read.common.block.TsBlock; + +public final class OperatorTestUtils { + + private OperatorTestUtils() { + // Utility class. + } + + public static TsBlock nextNonNullOrEmpty(Operator operator) throws Exception { + while (operator.hasNext()) { + TsBlock result = operator.next(); + if (!isNullOrEmpty(result)) { + return result; + } + } + throw new AssertionError("Expected a non-null and non-empty TsBlock from operator"); + } + + public static TsBlock lastNonNullOrEmpty(Operator operator) throws Exception { + TsBlock result = null; + while (operator.isBlocked().isDone() && operator.hasNext()) { + TsBlock nextResult = operator.next(); + if (!isNullOrEmpty(nextResult)) { + result = nextResult; + } + } + return result; + } + + private static boolean isNullOrEmpty(TsBlock tsBlock) { + return tsBlock == null || tsBlock.getPositionCount() == 0; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/SeriesAggregationScanOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/SeriesAggregationScanOperatorTest.java index bbf96e89fb4fb..afa08223d6463 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/SeriesAggregationScanOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/SeriesAggregationScanOperatorTest.java @@ -66,6 +66,7 @@ import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext.createFragmentInstanceContext; import static org.apache.iotdb.db.queryengine.execution.operator.AggregationOperatorTest.TEST_TIME_SLICE; import static org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil.initTimeRangeIterator; +import static org.apache.iotdb.db.queryengine.execution.operator.OperatorTestUtils.nextNonNullOrEmpty; import static org.junit.Assert.assertEquals; public class SeriesAggregationScanOperatorTest { @@ -111,7 +112,7 @@ public void testAggregationWithoutTimeFilter() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); assertEquals(500, resultTsBlock.getColumn(0).getLong(0)); count++; } @@ -135,7 +136,7 @@ public void testAggregationWithoutTimeFilterOrderByTimeDesc() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); assertEquals(500, resultTsBlock.getColumn(0).getLong(0)); count++; } @@ -161,7 +162,7 @@ public void testMultiAggregationFuncWithoutTimeFilter1() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); assertEquals(500, resultTsBlock.getColumn(0).getLong(0)); assertEquals(6524750.0, resultTsBlock.getColumn(1).getDouble(0), 0.0001); count++; @@ -192,7 +193,7 @@ public void testMultiAggregationFuncWithoutTimeFilter2() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); assertEquals(20000, resultTsBlock.getColumn(0).getInt(0)); assertEquals(10499, resultTsBlock.getColumn(1).getInt(0)); assertEquals(0, resultTsBlock.getColumn(2).getLong(0)); @@ -228,7 +229,7 @@ public void testMultiAggregationFuncWithoutTimeFilterOrderByTimeDesc() throws Ex int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); assertEquals(20000, resultTsBlock.getColumn(0).getInt(0)); assertEquals(10499, resultTsBlock.getColumn(1).getInt(0)); assertEquals(0, resultTsBlock.getColumn(2).getLong(0)); @@ -259,7 +260,7 @@ public void testAggregationWithTimeFilter1() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); assertEquals(resultTsBlock.getColumn(0).getLong(0), 380); count++; } @@ -285,7 +286,7 @@ public void testAggregationWithTimeFilter2() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); assertEquals(resultTsBlock.getColumn(0).getLong(0), 380); count++; } @@ -311,7 +312,7 @@ public void testAggregationWithTimeFilter3() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); assertEquals(resultTsBlock.getColumn(0).getLong(0), 300); count++; } @@ -343,7 +344,7 @@ public void testMultiAggregationWithTimeFilter() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); assertEquals(20100, resultTsBlock.getColumn(0).getInt(0)); assertEquals(399, resultTsBlock.getColumn(1).getInt(0)); assertEquals(100, resultTsBlock.getColumn(2).getLong(0)); @@ -375,7 +376,7 @@ public void testGroupByWithoutGlobalTimeFilter() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(100 * count, resultTsBlock.getTimeColumn().getLong(pos)); @@ -408,7 +409,7 @@ public void testGroupByWithGlobalTimeFilter() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(100 * count, resultTsBlock.getTimeColumn().getLong(pos)); @@ -450,7 +451,7 @@ public void testGroupByWithMultiFunction() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(100 * count, resultTsBlock.getTimeColumn().getLong(pos)); @@ -495,7 +496,7 @@ public void testGroupByWithMultiFunctionOrderByTimeDesc() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(100 * (3 - count), resultTsBlock.getTimeColumn().getLong(pos)); @@ -530,7 +531,7 @@ public void testGroupBySlidingTimeWindow() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(50 * count, resultTsBlock.getTimeColumn().getLong(pos)); @@ -563,7 +564,7 @@ public void testGroupBySlidingTimeWindow2() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(timeColumn[count], resultTsBlock.getTimeColumn().getLong(pos)); @@ -606,7 +607,7 @@ public void testGroupBySlidingWindowWithMultiFunction() throws Exception { int count = 0; while (seriesAggregationScanOperator.hasNext()) { - TsBlock resultTsBlock = seriesAggregationScanOperator.next(); + TsBlock resultTsBlock = nextNonNullOrEmpty(seriesAggregationScanOperator); int positionCount = resultTsBlock.getPositionCount(); for (int pos = 0; pos < positionCount; pos++) { assertEquals(timeColumn[count], resultTsBlock.getTimeColumn().getLong(pos)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/SeriesScanOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/SeriesScanOperatorTest.java index 11daae6f1c547..39455ab420895 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/SeriesScanOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/SeriesScanOperatorTest.java @@ -55,6 +55,7 @@ import java.util.concurrent.TimeUnit; import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext.createFragmentInstanceContext; +import static org.apache.iotdb.db.queryengine.execution.operator.OperatorTestUtils.nextNonNullOrEmpty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -114,7 +115,7 @@ public void batchTest() throws Exception { int count = 0; while (seriesScanOperator.hasNext()) { - TsBlock tsBlock = seriesScanOperator.next(); + TsBlock tsBlock = nextNonNullOrEmpty(seriesScanOperator); assertEquals(1, tsBlock.getValueColumnCount()); assertTrue(tsBlock.getColumn(0) instanceof IntColumn); for (int i = 0; i < tsBlock.getPositionCount(); i++, count++) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/UpdateLastCacheOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/UpdateLastCacheOperatorTest.java index 0c9c47815aebf..30604831e2680 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/UpdateLastCacheOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/UpdateLastCacheOperatorTest.java @@ -62,6 +62,7 @@ import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext.createFragmentInstanceContext; import static org.apache.iotdb.db.queryengine.execution.operator.AggregationOperatorTest.TEST_TIME_SLICE; import static org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil.initTimeRangeIterator; +import static org.apache.iotdb.db.queryengine.execution.operator.OperatorTestUtils.nextNonNullOrEmpty; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -103,7 +104,7 @@ public void testUpdateLastCacheOperatorTestWithoutTimeFilter() { assertTrue(updateLastCacheOperator.isBlocked().isDone()); assertTrue(updateLastCacheOperator.hasNext()); - TsBlock result = updateLastCacheOperator.next(); + TsBlock result = nextNonNullOrEmpty(updateLastCacheOperator); assertEquals(1, result.getPositionCount()); assertEquals(3, result.getValueColumnCount()); @@ -133,7 +134,7 @@ public void testUpdateLastCacheOperatorTestWithTimeFilter1() { assertTrue(updateLastCacheOperator.isBlocked().isDone()); assertTrue(updateLastCacheOperator.hasNext()); - TsBlock result = updateLastCacheOperator.next(); + TsBlock result = nextNonNullOrEmpty(updateLastCacheOperator); assertEquals(1, result.getPositionCount()); assertEquals(3, result.getValueColumnCount()); @@ -163,7 +164,7 @@ public void testUpdateLastCacheOperatorTestWithTimeFilter2() { assertTrue(updateLastCacheOperator.isBlocked().isDone()); assertTrue(updateLastCacheOperator.hasNext()); - TsBlock result = updateLastCacheOperator.next(); + TsBlock result = nextNonNullOrEmpty(updateLastCacheOperator); assertEquals(1, result.getPositionCount()); assertEquals(3, result.getValueColumnCount()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperatorTest.java index 3bdd80a3c1b53..c5e8335ebf566 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaCountOperatorTest.java @@ -114,6 +114,82 @@ public void testSchemaCountOperator() throws Exception { } } + @Test + public void testSchemaCountOperatorSkipSchemaRegion() throws Exception { + ExecutorService instanceNotificationExecutor = + IoTDBThreadPoolFactory.newFixedThreadPool(1, "test-instance-notification"); + try { + QueryId queryId = new QueryId("stub_query"); + FragmentInstanceId instanceId = + new FragmentInstanceId(new PlanFragmentId(queryId, 0), "stub-instance"); + FragmentInstanceStateMachine stateMachine = + new FragmentInstanceStateMachine(instanceId, instanceNotificationExecutor); + FragmentInstanceContext fragmentInstanceContext = + createFragmentInstanceContext(instanceId, stateMachine); + DriverContext driverContext = new DriverContext(fragmentInstanceContext, 0); + PlanNodeId planNodeId = queryId.genPlanNodeId(); + ISchemaRegion schemaRegion = Mockito.mock(ISchemaRegion.class); + OperatorContext operatorContext = + driverContext.addOperatorContext( + 1, planNodeId, SchemaCountOperator.class.getSimpleName()); + operatorContext.setDriverContext( + new SchemaDriverContext(fragmentInstanceContext, schemaRegion, 0)); + ISchemaSource schemaSource = Mockito.mock(ISchemaSource.class); + Mockito.when(schemaSource.shouldSkipSchemaRegion(schemaRegion)).thenReturn(true); + + SchemaCountOperator schemaCountOperator = + new SchemaCountOperator<>( + planNodeId, driverContext.getOperatorContexts().get(0), schemaSource); + + assertTrue(schemaCountOperator.hasNext()); + TsBlock tsBlock = schemaCountOperator.next(); + assertEquals(0, tsBlock.getColumn(0).getLong(0)); + assertTrue(schemaCountOperator.isFinished()); + Mockito.verify(schemaSource, Mockito.never()).getSchemaReader(schemaRegion); + } finally { + instanceNotificationExecutor.shutdown(); + } + } + + @Test + public void testSchemaCountOperatorUseSchemaStatistic() throws Exception { + ExecutorService instanceNotificationExecutor = + IoTDBThreadPoolFactory.newFixedThreadPool(1, "test-instance-notification"); + try { + QueryId queryId = new QueryId("stub_query"); + FragmentInstanceId instanceId = + new FragmentInstanceId(new PlanFragmentId(queryId, 0), "stub-instance"); + FragmentInstanceStateMachine stateMachine = + new FragmentInstanceStateMachine(instanceId, instanceNotificationExecutor); + FragmentInstanceContext fragmentInstanceContext = + createFragmentInstanceContext(instanceId, stateMachine); + DriverContext driverContext = new DriverContext(fragmentInstanceContext, 0); + PlanNodeId planNodeId = queryId.genPlanNodeId(); + ISchemaRegion schemaRegion = Mockito.mock(ISchemaRegion.class); + OperatorContext operatorContext = + driverContext.addOperatorContext( + 1, planNodeId, SchemaCountOperator.class.getSimpleName()); + operatorContext.setDriverContext( + new SchemaDriverContext(fragmentInstanceContext, schemaRegion, 0)); + ISchemaSource schemaSource = Mockito.mock(ISchemaSource.class); + Mockito.when(schemaSource.hasSchemaStatistic(schemaRegion)).thenReturn(true); + Mockito.when(schemaSource.getSchemaStatistic(schemaRegion)).thenReturn(7L); + Mockito.when(schemaSource.checkRegionDatabaseIncluded(schemaRegion)).thenReturn(true); + + SchemaCountOperator schemaCountOperator = + new SchemaCountOperator<>( + planNodeId, driverContext.getOperatorContexts().get(0), schemaSource); + + assertTrue(schemaCountOperator.hasNext()); + TsBlock tsBlock = schemaCountOperator.next(); + assertEquals(7, tsBlock.getColumn(0).getLong(0)); + Mockito.verify(schemaSource).getSchemaStatistic(schemaRegion); + Mockito.verify(schemaSource, Mockito.never()).getSchemaReader(schemaRegion); + } finally { + instanceNotificationExecutor.shutdown(); + } + } + @Test public void testLevelTimeSeriesCountOperator() { ExecutorService instanceNotificationExecutor = @@ -185,6 +261,43 @@ public void testLevelTimeSeriesCountOperator() { } } + @Test + public void testLevelTimeSeriesCountOperatorSkipSchemaRegion() { + ExecutorService instanceNotificationExecutor = + IoTDBThreadPoolFactory.newFixedThreadPool(1, "test-instance-notification"); + try { + QueryId queryId = new QueryId("stub_query"); + FragmentInstanceId instanceId = + new FragmentInstanceId(new PlanFragmentId(queryId, 0), "stub-instance"); + FragmentInstanceStateMachine stateMachine = + new FragmentInstanceStateMachine(instanceId, instanceNotificationExecutor); + FragmentInstanceContext fragmentInstanceContext = + createFragmentInstanceContext(instanceId, stateMachine); + DriverContext driverContext = new DriverContext(fragmentInstanceContext, 0); + PlanNodeId planNodeId = queryId.genPlanNodeId(); + OperatorContext operatorContext = + driverContext.addOperatorContext( + 1, planNodeId, CountGroupByLevelScanOperator.class.getSimpleName()); + ISchemaRegion schemaRegion = Mockito.mock(ISchemaRegion.class); + operatorContext.setDriverContext( + new SchemaDriverContext(fragmentInstanceContext, schemaRegion, 0)); + ISchemaSource schemaSource = Mockito.mock(ISchemaSource.class); + Mockito.when(schemaSource.shouldSkipSchemaRegion(schemaRegion)).thenReturn(true); + + CountGroupByLevelScanOperator timeSeriesCountOperator = + new CountGroupByLevelScanOperator<>( + planNodeId, driverContext.getOperatorContexts().get(0), 1, schemaSource); + + assertTrue(collectResult(timeSeriesCountOperator).isEmpty()); + Mockito.verify(schemaSource, Mockito.never()).getSchemaReader(schemaRegion); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } finally { + instanceNotificationExecutor.shutdown(); + } + } + private List collectResult(CountGroupByLevelScanOperator operator) throws Exception { List tsBlocks = new ArrayList<>(); while (operator.hasNext()) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSourceTest.java new file mode 100644 index 0000000000000..84acab189f43b --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSourceTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.schema.source; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.db.schemaengine.rescon.ISchemaRegionStatistics; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo; + +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TimeSeriesSchemaSourceTest { + + @Test + public void testCountSourceSkipsUnauthorizedInternalDatabases() throws Exception { + final ISchemaSource countSource = + SchemaSourceFactory.getTimeSeriesSchemaCountSource( + new PartialPath("root.**"), + false, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE, + false); + + assertTrue( + countSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.SYSTEM_DATABASE))); + assertFalse(countSource.shouldSkipSchemaRegion(mockSchemaRegion("root.sg"))); + } + + @Test + public void testCountSourceKeepsAuthorizedInternalDatabases() throws Exception { + final ISchemaSource systemCountSource = + SchemaSourceFactory.getTimeSeriesSchemaCountSource( + new PartialPath("root.**"), + false, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE, + true); + assertFalse( + systemCountSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.SYSTEM_DATABASE))); + } + + @Test + public void testCountSourceSkipsUnauthorizedInternalDatabasesWithWildcardSecondNode() + throws Exception { + final ISchemaSource countSource = + SchemaSourceFactory.getTimeSeriesSchemaCountSource( + new PartialPath("root.*.**"), + false, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE, + false); + + assertTrue( + countSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.SYSTEM_DATABASE))); + assertFalse(countSource.shouldSkipSchemaRegion(mockSchemaRegion("root.sg"))); + } + + @Test + public void testCountSourceKeepsExactInternalDatabaseQueriesWithPrivilege() throws Exception { + final ISchemaSource systemCountSource = + SchemaSourceFactory.getTimeSeriesSchemaCountSource( + new PartialPath("root.__system"), + false, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE, + true); + assertFalse( + systemCountSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.SYSTEM_DATABASE))); + } + + @Test + public void testShowSourceDoesNotSkipInternalDatabases() throws Exception { + final ISchemaSource showSource = + SchemaSourceFactory.getTimeSeriesSchemaScanSource( + new PartialPath("root.**"), + false, + 0, + 0, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE); + + assertFalse( + showSource.shouldSkipSchemaRegion(mockSchemaRegion(SchemaConstant.SYSTEM_DATABASE))); + } + + @Test + public void testCountStatisticIncludesView() throws Exception { + final ISchemaSource countSource = + SchemaSourceFactory.getTimeSeriesSchemaCountSource( + new PartialPath("root.sg.**"), + false, + null, + Collections.emptyMap(), + SchemaConstant.ALL_MATCH_SCOPE, + false); + final ISchemaRegion schemaRegion = mockSchemaRegion("root.sg"); + final ISchemaRegionStatistics schemaRegionStatistics = + Mockito.mock(ISchemaRegionStatistics.class); + + Mockito.when(schemaRegion.getSchemaRegionStatistics()).thenReturn(schemaRegionStatistics); + Mockito.when(schemaRegionStatistics.getSeriesNumber(true)).thenReturn(5L); + + assertEquals(5L, countSource.getSchemaStatistic(schemaRegion)); + Mockito.verify(schemaRegionStatistics).getSeriesNumber(true); + Mockito.verify(schemaRegionStatistics, Mockito.never()).getSeriesNumber(false); + } + + private ISchemaRegion mockSchemaRegion(final String database) { + final ISchemaRegion schemaRegion = Mockito.mock(ISchemaRegion.class); + Mockito.when(schemaRegion.getDatabaseFullPath()).thenReturn(database); + return schemaRegion; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/schedule/DefaultDriverSchedulerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/schedule/DefaultDriverSchedulerTest.java index 0cc0e5edd71d5..595d5febe4c6e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/schedule/DefaultDriverSchedulerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/schedule/DefaultDriverSchedulerTest.java @@ -197,6 +197,47 @@ public void testReadyToRunning() { clear(); } + @Test + public void testAbortReadyTaskAfterPollReleasesReadyQueueReservation() throws Exception { + IMPPDataExchangeManager mockMPPDataExchangeManager = + Mockito.mock(IMPPDataExchangeManager.class); + manager.setBlockManager(mockMPPDataExchangeManager); + ITaskScheduler defaultScheduler = manager.getScheduler(); + IDriver mockDriver = Mockito.mock(IDriver.class); + DriverTaskHandle driverTaskHandle = + new DriverTaskHandle( + 1, + (MultilevelPriorityQueue) manager.getReadyQueue(), + OptionalInt.of(Integer.MAX_VALUE)); + + QueryId queryId = new QueryId("test"); + FragmentInstanceId instanceId = + new FragmentInstanceId(new PlanFragmentId(queryId, 0), "inst-0"); + DriverTaskId driverTaskID = new DriverTaskId(instanceId, 0); + Mockito.when(mockDriver.getDriverTaskId()).thenReturn(driverTaskID); + DriverTask testTask = + new DriverTask(mockDriver, 100L, DriverTaskStatus.READY, driverTaskHandle, 0, false); + + try { + manager.registerTaskToQueryMap(queryId, testTask); + manager.submitTaskToReadyQueue(testTask); + + DriverTask polledTask = manager.getReadyQueue().poll(); + Assert.assertSame(testTask, polledTask); + Assert.assertEquals(1, manager.getReadyQueueReservedTaskCount()); + + manager.abortFragmentInstance(instanceId); + + Assert.assertEquals(DriverTaskStatus.ABORTED, testTask.getStatus()); + Assert.assertEquals(0, manager.getReadyQueue().size()); + Assert.assertEquals(0, manager.getReadyQueueReservedTaskCount()); + Assert.assertFalse(defaultScheduler.readyToRunning(polledTask)); + Assert.assertEquals(0, manager.getReadyQueueReservedTaskCount()); + } finally { + clear(); + } + } + @Test public void testRunningToReady() { IMPPDataExchangeManager mockMPPDataExchangeManager = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/RegionScanPlanningTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/RegionScanPlanningTest.java index 7a5eccd87a599..fba26ceea8057 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/RegionScanPlanningTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/RegionScanPlanningTest.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.DistributedQueryPlan; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; @@ -37,10 +38,15 @@ import org.apache.tsfile.file.metadata.PlainDeviceID; import org.junit.Test; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Set; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class RegionScanPlanningTest { @@ -146,4 +152,79 @@ public void testShowTimeseriesWithTimeCondition() throws IllegalPathException { assertEquals(devicePaths, targetDevicePaths); assertEquals(path, targetMeasurementPaths); } + + @Test + public void testCountTimeseriesWithLogicalViewUsesMergeBeforeCount() throws IllegalPathException { + QueryId queryId = new QueryId("test"); + MPPQueryContext context = + new MPPQueryContext("", queryId, null, new TEndPoint(), new TEndPoint()); + + Map logicalViewContextMap = + Collections.singletonMap( + "root.sg.view.v1", + new TimeseriesContext( + "INT32", + null, + null, + null, + null, + null, + null, + null, + 1, + true, + "root.sg", + Collections.emptyMap())); + Map>> deviceToTimeseriesSchemaInfo = + new HashMap<>(); + deviceToTimeseriesSchemaInfo.put( + new PartialPath("root.sg.d22"), + Collections.singletonMap( + new MeasurementPath("root.sg.d22.s1"), + Collections.singletonList( + new TimeseriesContext( + "INT32", + null, + "PLAIN", + "LZ4", + null, + null, + null, + null, + 0, + logicalViewContextMap)))); + deviceToTimeseriesSchemaInfo.put( + new PartialPath("root.sg.d55555"), + Collections.singletonMap( + new MeasurementPath("root.sg.d55555.s1"), + Collections.singletonList( + new TimeseriesContext( + "INT32", + null, + "PLAIN", + "LZ4", + null, + null, + null, + null, + 0, + logicalViewContextMap)))); + + TimeseriesRegionScanNode regionScanNode = + new TimeseriesRegionScanNode( + queryId.genPlanNodeId(), deviceToTimeseriesSchemaInfo, true, null); + PlanNode rewrittenRoot = + new DistributionPlanner(Util.ANALYSIS, new LogicalQueryPlan(context, regionScanNode)) + .rewriteSource(); + + assertTrue(rewrittenRoot instanceof ActiveRegionScanMergeNode); + ActiveRegionScanMergeNode mergeNode = (ActiveRegionScanMergeNode) rewrittenRoot; + assertTrue(mergeNode.isOutputCount()); + assertTrue(mergeNode.isNeedMerge()); + assertEquals(2, mergeNode.getChildren().size()); + for (PlanNode child : mergeNode.getChildren()) { + assertTrue(child instanceof TimeseriesRegionScanNode); + assertFalse(((TimeseriesRegionScanNode) child).isOutputCount()); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java index 437cad010e9c0..ed9b57a5b9d80 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java @@ -224,13 +224,13 @@ public void testShowDevicesWithTimeConditionWithLimitOffset() throws IllegalPath DeviceRegionScanNode regionScanNode = new DeviceRegionScanNode(queryId.genPlanNodeId(), deviceContextMap, false, null); - LimitNode limitNode = new LimitNode(queryId.genPlanNodeId(), 20); - limitNode.addChild(regionScanNode); OffsetNode offsetNode = new OffsetNode(queryId.genPlanNodeId(), 10); - offsetNode.addChild(limitNode); + offsetNode.addChild(regionScanNode); + LimitNode limitNode = new LimitNode(queryId.genPlanNodeId(), 20); + limitNode.addChild(offsetNode); PlanNode actualPlan = parseSQLToPlanNode(sql); - Assert.assertEquals(actualPlan, offsetNode); + Assert.assertEquals(actualPlan, limitNode); } @Test @@ -287,4 +287,51 @@ public void serializeDeserializeTest() throws IllegalPathException { buffer.flip(); Assert.assertEquals(timeseriesRegionScanNode, PlanNodeType.deserialize(buffer)); } + + @Test + public void serializeDeserializeLogicalViewContextTest() throws IllegalPathException { + Map logicalViewContextMap = + Collections.singletonMap( + "root.sg.view.v1", + new TimeseriesContext( + "INT32", + null, + null, + null, + null, + null, + null, + null, + 1, + true, + "root.sg", + new HashMap<>())); + Map> timeseriesSchemaInfoMap = new HashMap<>(); + timeseriesSchemaInfoMap.put( + new MeasurementPath("root.sg.d3.s1", TSDataType.INT32), + Collections.singletonList( + new TimeseriesContext( + "INT32", + null, + TSFileDescriptor.getInstance().getConfig().getValueEncoder(TSDataType.INT32), + "LZ4", + null, + null, + null, + null, + 0, + logicalViewContextMap))); + Map>> deviceToTimeseriesSchemaInfo = + new HashMap<>(); + deviceToTimeseriesSchemaInfo.put(new PartialPath("root.sg.d3"), timeseriesSchemaInfoMap); + + TimeseriesRegionScanNode timeseriesRegionScanNode = + new TimeseriesRegionScanNode( + new PlanNodeId("timeseries_test_id"), deviceToTimeseriesSchemaInfo, false, null); + + ByteBuffer buffer = ByteBuffer.allocate(10240); + timeseriesRegionScanNode.serialize(buffer); + buffer.flip(); + Assert.assertEquals(timeseriesRegionScanNode, PlanNodeType.deserialize(buffer)); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/SchemaQueryLogicalPlannerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/SchemaQueryLogicalPlannerTest.java index 60a0d4835624e..9274e4d04404b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/SchemaQueryLogicalPlannerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/SchemaQueryLogicalPlannerTest.java @@ -47,6 +47,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.LimitNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.OffsetNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.RawDataAggregationNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.DeviceRegionScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.AggregationStep; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.CrossSeriesAggregationDescriptor; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; @@ -648,6 +649,22 @@ public void testShowDevicesWherePathContains() { } } + @Test + public void testShowDevicesWithTimeConditionPaginationOrder() { + String sql = "SHOW DEVICES root.ln.wf01.wt01 WHERE time >= 10 limit 20 offset 10"; + try { + LimitNode limitNode = (LimitNode) parseSQLToPlanNode(sql); + Assert.assertEquals(20, limitNode.getLimit()); + + OffsetNode offsetNode = (OffsetNode) limitNode.getChild(); + Assert.assertEquals(10, offsetNode.getOffset()); + Assert.assertTrue(offsetNode.getChild() instanceof DeviceRegionScanNode); + } catch (Exception e) { + e.printStackTrace(); + fail(); + } + } + @Test public void testCountNodes() { String sql = "COUNT NODES root.ln LEVEL=1"; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaCountNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaCountNodeSerdeTest.java index 0139371e26bd8..fb78ea46a2bb5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaCountNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaCountNodeSerdeTest.java @@ -87,7 +87,8 @@ public void testLevelTimeSeriesCountSerializeAndDeserialize() throws IllegalPath 10, null, Collections.emptyMap(), - SchemaConstant.ALL_MATCH_SCOPE); + SchemaConstant.ALL_MATCH_SCOPE, + true); IdentitySinkNode sinkNode = new IdentitySinkNode( new PlanNodeId("sink"), @@ -122,7 +123,8 @@ public void testTimeSeriesCountSerializeAndDeserialize() throws IllegalPathExcep true, null, Collections.emptyMap(), - SchemaConstant.ALL_MATCH_SCOPE); + SchemaConstant.ALL_MATCH_SCOPE, + true); IdentitySinkNode sinkNode = new IdentitySinkNode( new PlanNodeId("sink"), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/DeleteDataNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/DeleteDataNodeSerdeTest.java index cbfba2a687914..3087085538326 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/DeleteDataNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/DeleteDataNodeSerdeTest.java @@ -25,10 +25,14 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -64,4 +68,28 @@ public void testSerializeAndDeserialize() throws IllegalPathException { Assert.assertEquals(pathList.get(i), deserializedPathList.get(i)); } } + + @Test + public void testSerializeAndDeserializeForWAL() throws IllegalPathException, IOException { + long startTime = 1; + long endTime = 10; + List pathList = new ArrayList<>(); + pathList.add(new PartialPath("root.\u6570\u636e\u5e93.d1.\u6e29\u5ea6")); + pathList.add(new PartialPath("root.\u6570\u636e\u5e93.d2.*")); + DeleteDataNode deleteDataNode = + new DeleteDataNode(new PlanNodeId("DeleteDataNode"), pathList, startTime, endTime); + + ByteBuffer byteBuffer = ByteBuffer.allocate(deleteDataNode.serializedSize()); + deleteDataNode.serializeToWAL(new WALByteBufferForTest(byteBuffer)); + Assert.assertEquals(deleteDataNode.serializedSize(), byteBuffer.position()); + + DataInputStream dataInputStream = + new DataInputStream(new ByteArrayInputStream(byteBuffer.array())); + Assert.assertEquals(PlanNodeType.DELETE_DATA.getNodeType(), dataInputStream.readShort()); + + DeleteDataNode deserializedNode = DeleteDataNode.deserializeFromWAL(dataInputStream); + Assert.assertEquals(startTime, deserializedNode.getDeleteStartTime()); + Assert.assertEquals(endTime, deserializedNode.getDeleteEndTime()); + Assert.assertEquals(pathList, deserializedNode.getPathList()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java index 523b061428241..a4e19d9717629 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertRowNodeSerdeTest.java @@ -89,8 +89,8 @@ public void TestSerializeAndDeserializeForWAL() throws IllegalPathException, IOE tmpNode.setPlanNodeId(insertRowNode.getPlanNodeId()); tmpNode.setMeasurementSchemas( new MeasurementSchema[] { - new MeasurementSchema("s1", TSDataType.DOUBLE), - new MeasurementSchema("s2", TSDataType.FLOAT), + new MeasurementSchema("\u6e29\u5ea6", TSDataType.DOUBLE), + new MeasurementSchema("\u6e7f\u5ea6", TSDataType.FLOAT), new MeasurementSchema("s3", TSDataType.INT64), new MeasurementSchema("s4", TSDataType.INT32), new MeasurementSchema("s5", TSDataType.BOOLEAN) @@ -148,9 +148,9 @@ private InsertRowNode getInsertRowNodeWithMeasurementSchemas() throws IllegalPat InsertRowNode insertRowNode = new InsertRowNode( new PlanNodeId("plannode 2"), - new PartialPath("root.isp.d2"), + new PartialPath("root.\u6570\u636e\u5e93.d2"), false, - new String[] {"s1", "s2", "s3", "s4", "s5"}, + new String[] {"\u6e29\u5ea6", "\u6e7f\u5ea6", "s3", "s4", "s5"}, dataTypes, time, columns, @@ -158,8 +158,8 @@ private InsertRowNode getInsertRowNodeWithMeasurementSchemas() throws IllegalPat insertRowNode.setMeasurementSchemas( new MeasurementSchema[] { - new MeasurementSchema("s1", TSDataType.DOUBLE), - new MeasurementSchema("s2", TSDataType.FLOAT), + new MeasurementSchema("\u6e29\u5ea6", TSDataType.DOUBLE), + new MeasurementSchema("\u6e7f\u5ea6", TSDataType.FLOAT), new MeasurementSchema("s3", TSDataType.INT64), new MeasurementSchema("s4", TSDataType.INT32), new MeasurementSchema("s5", TSDataType.BOOLEAN) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java index 754e7a6e1386d..a883b7d24a87e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java @@ -84,8 +84,8 @@ public void testSerializeAndDeserializeForWAL() throws IllegalPathException, IOE tmpNode.setMeasurementSchemas( new MeasurementSchema[] { - new MeasurementSchema("s1", TSDataType.DOUBLE), - new MeasurementSchema("s2", TSDataType.FLOAT), + new MeasurementSchema("\u6e29\u5ea6", TSDataType.DOUBLE), + new MeasurementSchema("\u6e7f\u5ea6", TSDataType.FLOAT), new MeasurementSchema("s3", TSDataType.INT64), new MeasurementSchema("s4", TSDataType.INT32), new MeasurementSchema("s5", TSDataType.BOOLEAN) @@ -191,9 +191,9 @@ private InsertTabletNode getInsertTabletNodeWithSchema() throws IllegalPathExcep InsertTabletNode insertTabletNode = new InsertTabletNode( new PlanNodeId("plannode 1"), - new PartialPath("root.isp.d1"), + new PartialPath("root.\u6570\u636e\u5e93.d1"), false, - new String[] {"s1", "s2", "s3", "s4", "s5"}, + new String[] {"\u6e29\u5ea6", "\u6e7f\u5ea6", "s3", "s4", "s5"}, dataTypes, times, null, @@ -201,8 +201,8 @@ private InsertTabletNode getInsertTabletNodeWithSchema() throws IllegalPathExcep times.length); insertTabletNode.setMeasurementSchemas( new MeasurementSchema[] { - new MeasurementSchema("s1", TSDataType.DOUBLE), - new MeasurementSchema("s2", TSDataType.FLOAT), + new MeasurementSchema("\u6e29\u5ea6", TSDataType.DOUBLE), + new MeasurementSchema("\u6e7f\u5ea6", TSDataType.FLOAT), new MeasurementSchema("s3", TSDataType.INT64), new MeasurementSchema("s4", TSDataType.INT32), new MeasurementSchema("s5", TSDataType.BOOLEAN) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java index a93a22b6e9f4a..305a3197cfebf 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/WritePlanNodeSplitTest.java @@ -44,6 +44,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BitMap; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -201,6 +202,9 @@ public void testSplitInsertTablet() throws IllegalPathException { insertTabletNode.setColumns( new Object[] {new int[] {-20, -10, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}}); insertTabletNode.setRowCount(insertTabletNode.getTimes().length); + final BitMap[] bitMaps = new BitMap[] {new BitMap(insertTabletNode.getRowCount())}; + bitMaps[0].mark(2); + insertTabletNode.setBitMaps(bitMaps); DataPartitionQueryParam dataPartitionQueryParam = new DataPartitionQueryParam(); dataPartitionQueryParam.setDevicePath(insertTabletNode.getDevicePath().getFullPath()); @@ -219,6 +223,12 @@ public void testSplitInsertTablet() throws IllegalPathException { Assert.assertEquals(tabletNode.getTimes().length, 2); TConsensusGroupId regionId = tabletNode.getDataRegionReplicaSet().getRegionId(); Assert.assertEquals(getRegionIdByTime(tabletNode.getMinTime()), regionId.getId()); + if (tabletNode.getTimes()[0] == 1) { + Assert.assertNotNull(tabletNode.getBitMaps()); + Assert.assertTrue(tabletNode.getBitMaps()[0].isMarked(0)); + } else { + Assert.assertNull(tabletNode.getBitMaps()); + } } insertTabletNode = new InsertTabletNode(new PlanNodeId("plan node 2")); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImplTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImplTest.java new file mode 100644 index 0000000000000..3d496470597ca --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileDispatcherImplTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.scheduler.load; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.consensus.DataRegionId; +import org.apache.iotdb.commons.partition.StorageExecutor; +import org.apache.iotdb.db.queryengine.common.PlanFragmentId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; +import org.apache.iotdb.db.queryengine.plan.planner.plan.PlanFragment; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; +import org.apache.iotdb.db.storageengine.StorageEngine; +import org.apache.iotdb.rpc.RpcUtils; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.io.File; +import java.util.Collections; + +@PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*", "javax.management.*"}) +@RunWith(PowerMockRunner.class) +@PrepareForTest(StorageEngine.class) +public class LoadTsFileDispatcherImplTest { + + @Test + public void testDispatchLocallyPieceNodeSkipsSerdeRoundTrip() throws Exception { + final StorageEngine storageEngine = Mockito.mock(StorageEngine.class); + PowerMockito.mockStatic(StorageEngine.class); + PowerMockito.when(StorageEngine.getInstance()).thenReturn(storageEngine); + + final LoadTsFileDispatcherImpl dispatcher = new LoadTsFileDispatcherImpl(null, false); + dispatcher.setUuid("test-uuid"); + + final LoadTsFilePieceNode pieceNode = + new LoadTsFilePieceNode(new PlanNodeId("piece"), new File("test.tsfile")); + final FragmentInstance instance = createFragmentInstance(pieceNode); + + Mockito.when( + storageEngine.writeLoadTsFileNode( + Mockito.eq(new DataRegionId(1)), Mockito.same(pieceNode), Mockito.eq("test-uuid"))) + .thenReturn(RpcUtils.SUCCESS_STATUS); + + dispatcher.dispatchLocally(instance); + + Mockito.verify(storageEngine) + .writeLoadTsFileNode( + Mockito.eq(new DataRegionId(1)), Mockito.same(pieceNode), Mockito.eq("test-uuid")); + } + + private static FragmentInstance createFragmentInstance(final LoadTsFilePieceNode pieceNode) { + final PlanFragmentId fragmentId = new PlanFragmentId("test", 0); + final FragmentInstance instance = + new FragmentInstance( + new PlanFragment(fragmentId, pieceNode), + fragmentId.genFragmentInstanceId(), + null, + null, + 0, + null, + false, + false); + final TConsensusGroupId consensusGroupId = new DataRegionId(1).convertToTConsensusGroupId(); + instance.setExecutorAndHost( + new StorageExecutor( + new TRegionReplicaSet( + consensusGroupId, + Collections.singletonList( + new TDataNodeLocation().setInternalEndPoint(new TEndPoint("127.0.0.1", 1)))))); + return instance; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java new file mode 100644 index 0000000000000..acf608abf806a --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.crud; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.stream.Stream; + +public class LoadTsFileStatementTest { + + @Test + public void testLoadSourcePathMustBeInAllowedDirs() throws Exception { + final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + final String[] originalAllowedDirs = config.getLoadTsFileAllowedDirs().clone(); + final boolean originalCheckEnabled = config.isLoadTsFileSourcePathCheckEnabled(); + final Path allowedDir = Files.createTempDirectory("load-tsfile-allowed"); + final Path deniedDir = Files.createTempDirectory("load-tsfile-denied"); + + try { + config.setLoadTsFileSourcePathCheckEnabled(true); + config.setLoadTsFileAllowedDirs(new String[] {allowedDir.toString()}); + final Path deniedTsFile = Files.createFile(deniedDir.resolve("denied.tsfile")); + final Path traversalTsFile = + allowedDir.resolve("..").resolve(deniedDir.getFileName()).resolve("denied.tsfile"); + + assertLoadSourcePathRejected(deniedTsFile); + assertLoadSourcePathRejected(traversalTsFile); + } finally { + config.setLoadTsFileAllowedDirs(originalAllowedDirs); + config.setLoadTsFileSourcePathCheckEnabled(originalCheckEnabled); + deleteRecursively(allowedDir); + deleteRecursively(deniedDir); + } + } + + @Test + public void testLoadSourcePathCheckCanBeDisabled() throws Exception { + final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + final String[] originalAllowedDirs = config.getLoadTsFileAllowedDirs().clone(); + final boolean originalCheckEnabled = config.isLoadTsFileSourcePathCheckEnabled(); + final Path allowedDir = Files.createTempDirectory("load-tsfile-allowed"); + final Path deniedDir = Files.createTempDirectory("load-tsfile-denied"); + + try { + config.setLoadTsFileSourcePathCheckEnabled(false); + config.setLoadTsFileAllowedDirs(new String[] {allowedDir.toString()}); + final Path deniedTsFile = Files.createFile(deniedDir.resolve("denied.tsfile")); + + new LoadTsFileStatement(deniedTsFile.toString()); + } finally { + config.setLoadTsFileAllowedDirs(originalAllowedDirs); + config.setLoadTsFileSourcePathCheckEnabled(originalCheckEnabled); + deleteRecursively(allowedDir); + deleteRecursively(deniedDir); + } + } + + private static void assertLoadSourcePathRejected(final Path sourcePath) { + try { + new LoadTsFileStatement(sourcePath.toString()); + Assert.fail("Expected disallowed LOAD TSFILE source path to be rejected."); + } catch (final FileNotFoundException e) { + Assert.assertTrue(e.getMessage().contains("outside allowed directories")); + } + } + + private static void deleteRecursively(final Path path) throws IOException { + if (path == null || !Files.exists(path)) { + return; + } + + try (final Stream pathStream = Files.walk(path)) { + pathStream + .sorted(Comparator.reverseOrder()) + .forEach( + currentPath -> { + try { + Files.deleteIfExists(currentPath); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImplTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImplTest.java new file mode 100644 index 0000000000000..1dfa4a056bbb1 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImplTest.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; +import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; +import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.schemaengine.metric.SchemaRegionMemMetric; +import org.apache.iotdb.db.schemaengine.rescon.MemSchemaEngineStatistics; +import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.MeasurementCollector; +import org.apache.iotdb.db.schemaengine.template.Template; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class MTreeBelowSGMemoryImplTest { + + @Test + public void testDeviceDescendantFlagIsMaintainedAcrossCreateDeleteAndRebuild() throws Exception { + final MTreeBelowSGMemoryImpl mtree = newMTree(); + + mtree.createTimeSeries( + new MeasurementPath("root.sg.a.b.s1"), + TSDataType.BOOLEAN, + TSEncoding.PLAIN, + CompressionType.SNAPPY, + null, + null, + false, + null); + + IMemMNode database = mtree.getNodeByPath(new PartialPath("root.sg")); + IMemMNode aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + IMemMNode bNode = mtree.getNodeByPath(new PartialPath("root.sg.a.b")); + + Assert.assertTrue(database.hasDeviceDescendant()); + Assert.assertFalse(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(bNode.isDevice()); + Assert.assertFalse(bNode.hasDeviceDescendant()); + + mtree.createTimeSeries( + new MeasurementPath("root.sg.a.s0"), + TSDataType.BOOLEAN, + TSEncoding.PLAIN, + CompressionType.SNAPPY, + null, + null, + false, + null); + mtree.createTimeSeries( + new MeasurementPath("root.sg.c.d.s2"), + TSDataType.BOOLEAN, + TSEncoding.PLAIN, + CompressionType.SNAPPY, + null, + null, + false, + null); + + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + IMemMNode cNode = mtree.getNodeByPath(new PartialPath("root.sg.c")); + Assert.assertTrue(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(cNode.hasDeviceDescendant()); + + database.setHasDeviceDescendant(false); + aNode.setHasDeviceDescendant(false); + bNode.setHasDeviceDescendant(true); + cNode.setHasDeviceDescendant(false); + mtree.rebuildSubtreeMeasurementCount(); + + database = mtree.getNodeByPath(new PartialPath("root.sg")); + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + bNode = mtree.getNodeByPath(new PartialPath("root.sg.a.b")); + cNode = mtree.getNodeByPath(new PartialPath("root.sg.c")); + Assert.assertTrue(database.hasDeviceDescendant()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertFalse(bNode.hasDeviceDescendant()); + Assert.assertTrue(cNode.hasDeviceDescendant()); + + mtree.deleteTimeseries(new PartialPath("root.sg.a.s0")); + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + Assert.assertFalse(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + + mtree.deleteTimeseries(new PartialPath("root.sg.a.b.s1")); + database = mtree.getNodeByPath(new PartialPath("root.sg")); + Assert.assertTrue(database.hasDeviceDescendant()); + assertPathNotExist(mtree, new PartialPath("root.sg.a")); + + mtree.deleteTimeseries(new PartialPath("root.sg.c.d.s2")); + database = mtree.getNodeByPath(new PartialPath("root.sg")); + Assert.assertFalse(database.hasDeviceDescendant()); + assertPathNotExist(mtree, new PartialPath("root.sg.c")); + } + + @Test + public void testLeafDeviceWildcardSuffixUsesDirectMeasurementLookup() throws Exception { + final PartialPath databasePath = new PartialPath("root.sg"); + final MemSchemaRegionStatistics regionStatistics = newRegionStatistics(); + final CountingMemMTreeStore store = + new CountingMemMTreeStore( + databasePath, regionStatistics, new SchemaRegionMemMetric(regionStatistics, "root.sg")); + final IMNodeFactory nodeFactory = + MNodeFactoryLoader.getInstance().getMemMNodeIMNodeFactory(); + + final IMemMNode databaseMNode = store.getRoot(); + final IMemMNode rootNode = store.generatePrefix(databasePath); + final IMemMNode deviceNode = + store.addChild(databaseMNode, "d1", nodeFactory.createInternalMNode(databaseMNode, "d1")); + store.setToEntity(deviceNode); + + for (int i = 0; i < 64; i++) { + final String measurement = i == 0 ? "target" : "s" + i; + final IMeasurementMNode measurementMNode = + nodeFactory.createMeasurementMNode( + deviceNode.getAsDeviceMNode(), + measurement, + new MeasurementSchema( + measurement, TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY), + null); + store.addChild(deviceNode, measurement, measurementMNode.getAsMNode()); + } + + store.watchParent(deviceNode); + final List matchedPaths = new ArrayList<>(); + try (final MeasurementCollector collector = + new MeasurementCollector( + rootNode, + new PartialPath("root.sg.**.target"), + store, + false, + SchemaConstant.ALL_MATCH_SCOPE) { + @Override + protected Void collectMeasurement(final IMeasurementMNode node) { + matchedPaths.add(getCurrentMeasurementPathInTraverse(node).getFullPath()); + return null; + } + }) { + collector.traverse(); + } + + Assert.assertEquals(Collections.singletonList("root.sg.d1.target"), matchedPaths); + Assert.assertEquals(0, store.getWatchedTraverserIteratorCount()); + Assert.assertEquals(1, store.getWatchedChildLookupCount()); + } + + private static MTreeBelowSGMemoryImpl newMTree() throws Exception { + final MemSchemaRegionStatistics regionStatistics = newRegionStatistics(); + return new MTreeBelowSGMemoryImpl( + new PartialPath("root.sg"), + node -> Collections.emptyMap(), + node -> Collections.emptyMap(), + regionStatistics, + new SchemaRegionMemMetric(regionStatistics, "root.sg")); + } + + private static MemSchemaRegionStatistics newRegionStatistics() { + return new MemSchemaRegionStatistics(0, new MemSchemaEngineStatistics()); + } + + private static void assertPathNotExist(final MTreeBelowSGMemoryImpl mtree, final PartialPath path) + throws MetadataException { + try { + mtree.getNodeByPath(path); + Assert.fail("Expected path not exist: " + path.getFullPath()); + } catch (final PathNotExistException ignored) { + // expected + } + } + + private static class CountingMemMTreeStore extends MemMTreeStore { + private IMemMNode watchedParent; + private int watchedTraverserIteratorCount; + private int watchedChildLookupCount; + + private CountingMemMTreeStore( + final PartialPath rootPath, + final MemSchemaRegionStatistics regionStatistics, + final SchemaRegionMemMetric metric) { + super(rootPath, regionStatistics, metric); + } + + private void watchParent(final IMemMNode parent) { + this.watchedParent = parent; + this.watchedTraverserIteratorCount = 0; + this.watchedChildLookupCount = 0; + } + + private int getWatchedTraverserIteratorCount() { + return watchedTraverserIteratorCount; + } + + private int getWatchedChildLookupCount() { + return watchedChildLookupCount; + } + + @Override + public IMemMNode getChild(final IMemMNode parent, final String name) { + if (parent == watchedParent) { + watchedChildLookupCount++; + } + return super.getChild(parent, name); + } + + @Override + public IMNodeIterator getTraverserIterator( + final IMemMNode parent, + final Map templateMap, + final boolean skipPreDeletedSchema) + throws MetadataException { + if (parent == watchedParent) { + watchedTraverserIteratorCount++; + } + return super.getTraverserIterator(parent, templateMap, skipPreDeletedSchema); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImplTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImplTest.java new file mode 100644 index 0000000000000..65a6f31a75da7 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImplTest.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.schemaengine.metric.SchemaRegionCachedMetric; +import org.apache.iotdb.db.schemaengine.rescon.CachedSchemaEngineStatistics; +import org.apache.iotdb.db.schemaengine.rescon.CachedSchemaRegionStatistics; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.ReleaseFlushMonitor; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.ICachedMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.MeasurementCollector; + +import org.apache.commons.io.FileUtils; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class MTreeBelowSGCachedImplTest { + + private static final Field STORE_FIELD; + private static final Field ROOT_NODE_FIELD; + + static { + try { + STORE_FIELD = MTreeBelowSGCachedImpl.class.getDeclaredField("store"); + STORE_FIELD.setAccessible(true); + ROOT_NODE_FIELD = MTreeBelowSGCachedImpl.class.getDeclaredField("rootNode"); + ROOT_NODE_FIELD.setAccessible(true); + } catch (final NoSuchFieldException e) { + throw new ExceptionInInitializerError(e); + } + } + + private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private int rawCachedMNodeSize; + private MTreeBelowSGCachedImpl mtree; + + @Before + public void setUp() { + rawCachedMNodeSize = config.getCachedMNodeSizeInPBTreeMode(); + config.setCachedMNodeSizeInPBTreeMode(10000); + ReleaseFlushMonitor.getInstance().clear(); + ReleaseFlushMonitor.getInstance().init(new CachedSchemaEngineStatistics()); + } + + @After + public void tearDown() throws Exception { + if (mtree != null) { + mtree.clear(); + mtree = null; + } + ReleaseFlushMonitor.getInstance().clear(); + FileUtils.deleteDirectory(new File(config.getSchemaDir())); + config.setCachedMNodeSizeInPBTreeMode(rawCachedMNodeSize); + } + + @Test + public void testDeviceDescendantFlagIsMaintainedAcrossCreateDeleteAndLazyRecompute() + throws Exception { + mtree = newMTree(); + + createBooleanTimeSeries("root.sg.a.b.s1"); + + ICachedMNode database = mtree.getNodeByPath(new PartialPath("root.sg")); + ICachedMNode aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + ICachedMNode bNode = mtree.getNodeByPath(new PartialPath("root.sg.a.b")); + + Assert.assertTrue(database.hasDeviceDescendant()); + Assert.assertTrue(database.isDeviceDescendantComputed()); + Assert.assertFalse(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(aNode.isDeviceDescendantComputed()); + Assert.assertTrue(bNode.isDevice()); + Assert.assertFalse(bNode.hasDeviceDescendant()); + Assert.assertTrue(bNode.isDeviceDescendantComputed()); + mtree.unPinMNode(database); + mtree.unPinMNode(aNode); + mtree.unPinMNode(bNode); + + createBooleanTimeSeries("root.sg.a.s0"); + createBooleanTimeSeries("root.sg.c.d.s2"); + + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + ICachedMNode cNode = mtree.getNodeByPath(new PartialPath("root.sg.c")); + Assert.assertTrue(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(aNode.isDeviceDescendantComputed()); + Assert.assertTrue(cNode.hasDeviceDescendant()); + Assert.assertTrue(cNode.isDeviceDescendantComputed()); + + aNode.setHasDeviceDescendant(false); + aNode.setDeviceDescendantComputed(false); + mtree.unPinMNode(aNode); + cNode.setHasDeviceDescendant(false); + cNode.setDeviceDescendantComputed(false); + mtree.unPinMNode(cNode); + + Assert.assertEquals( + Collections.singletonList("root.sg.a.b.s1"), + collectMeasurementPaths(new PartialPath("root.sg.a.**.s1"))); + + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(aNode.isDeviceDescendantComputed()); + mtree.unPinMNode(aNode); + cNode = mtree.getNodeByPath(new PartialPath("root.sg.c")); + Assert.assertFalse(cNode.isDeviceDescendantComputed()); + mtree.unPinMNode(cNode); + + mtree.deleteTimeseries(new PartialPath("root.sg.a.s0")); + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + Assert.assertFalse(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(aNode.isDeviceDescendantComputed()); + mtree.unPinMNode(aNode); + + mtree.deleteTimeseries(new PartialPath("root.sg.a.b.s1")); + database = mtree.getNodeByPath(new PartialPath("root.sg")); + Assert.assertTrue(database.hasDeviceDescendant()); + Assert.assertTrue(database.isDeviceDescendantComputed()); + mtree.unPinMNode(database); + assertPathNotExist(new PartialPath("root.sg.a")); + + mtree.deleteTimeseries(new PartialPath("root.sg.c.d.s2")); + database = mtree.getNodeByPath(new PartialPath("root.sg")); + Assert.assertFalse(database.hasDeviceDescendant()); + Assert.assertTrue(database.isDeviceDescendantComputed()); + mtree.unPinMNode(database); + assertPathNotExist(new PartialPath("root.sg.c")); + } + + private MTreeBelowSGCachedImpl newMTree() throws Exception { + final CachedSchemaRegionStatistics regionStatistics = + new CachedSchemaRegionStatistics(0, new CachedSchemaEngineStatistics()); + return new MTreeBelowSGCachedImpl( + new PartialPath("root.sg"), + node -> Collections.emptyMap(), + node -> Collections.emptyMap(), + () -> {}, + node -> {}, + node -> {}, + 0, + regionStatistics, + new SchemaRegionCachedMetric(regionStatistics, "root.sg")); + } + + private void createBooleanTimeSeries(final String path) throws MetadataException { + mtree.createTimeSeries( + new MeasurementPath(path), + TSDataType.BOOLEAN, + TSEncoding.PLAIN, + CompressionType.SNAPPY, + null, + null); + } + + private List collectMeasurementPaths(final PartialPath pattern) throws Exception { + final ICachedMNode rootNode = (ICachedMNode) ROOT_NODE_FIELD.get(mtree); + final CachedMTreeStore store = (CachedMTreeStore) STORE_FIELD.get(mtree); + final List matchedPaths = new ArrayList<>(); + try (MeasurementCollector collector = + new MeasurementCollector( + rootNode, pattern, store, false, SchemaConstant.ALL_MATCH_SCOPE) { + @Override + protected Void collectMeasurement(final IMeasurementMNode node) { + matchedPaths.add(getCurrentMeasurementPathInTraverse(node).getFullPath()); + return null; + } + }) { + collector.traverse(); + } + return matchedPaths; + } + + private void assertPathNotExist(final PartialPath path) throws MetadataException { + try { + mtree.getNodeByPath(path); + Assert.fail("Expected path not exist: " + path.getFullPath()); + } catch (final PathNotExistException ignored) { + // expected + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index 5082b8fc4f09a..f7a79eb8bf8e5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.DataRegionId; @@ -27,8 +28,10 @@ import org.apache.iotdb.commons.exception.ShutdownException; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.BatchProcessException; import org.apache.iotdb.db.exception.DataRegionException; import org.apache.iotdb.db.exception.TsFileProcessorException; import org.apache.iotdb.db.exception.WriteProcessException; @@ -36,6 +39,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeSchemaCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; @@ -62,6 +66,8 @@ import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.iotdb.db.utils.constant.TestConstant; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.PlainDeviceID; @@ -78,6 +84,7 @@ import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,6 +95,9 @@ import java.util.Collections; import java.util.List; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; + public class DataRegionTest { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig(); @@ -126,6 +136,7 @@ public void tearDown() throws Exception { dataRegion.syncDeleteDataFiles(); StorageEngine.getInstance().deleteDataRegion(new DataRegionId(0)); } + DataNodeSchemaCache.getInstance().cleanUp(); EnvironmentUtils.cleanDir(TestConstant.OUTPUT_DATA_DIR); CompactionTaskManager.getInstance().stop(); EnvironmentUtils.cleanEnv(); @@ -975,6 +986,189 @@ public void testInsertUnSequenceRows() dataRegion1.syncDeleteDataFiles(); } + @Test + public void testInsertRowPropagatesTsFileProcessorCreationFailure() + throws IllegalPathException, DataRegionException, TsFileProcessorException { + final HookedDataRegion dataRegion1 = new HookedDataRegion(systemDir, "root.fail_row"); + dataRegion1.setTsFileProcessorSupplier( + (timePartitionId, sequence) -> { + throw new WriteProcessRejectException("mock creation failure"); + }); + + final TSRecord record = new TSRecord(1, "root.fail_row"); + record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(1))); + final InsertRowNode insertRowNode = buildInsertRowNodeByTSRecord(record); + + try { + dataRegion1.insert(insertRowNode); + Assert.fail("Expected WriteProcessRejectException"); + } catch (WriteProcessRejectException e) { + Assert.assertTrue(e.getMessage().contains("mock creation failure")); + } catch (WriteProcessException e) { + Assert.fail("Expected WriteProcessRejectException but got " + e.getClass().getSimpleName()); + } finally { + dataRegion1.syncDeleteDataFiles(); + } + } + + @Test + public void testInsertRowsMarkAllFailedRowsForSameProcessor() throws Exception { + final HookedDataRegion dataRegion1 = new HookedDataRegion(systemDir, "root.fail_rows"); + final TsFileProcessor processor = Mockito.mock(TsFileProcessor.class); + Mockito.doThrow(new WriteProcessException("mock insert rows failure")) + .when(processor) + .insert(any(InsertRowsNode.class), any(long[].class)); + Mockito.when(processor.shouldFlush()).thenReturn(false); + Mockito.when(processor.isSequence()).thenReturn(true); + dataRegion1.setTsFileProcessorSupplier((timePartitionId, sequence) -> processor); + + final List indexList = Arrays.asList(0, 1); + final List nodes = new ArrayList<>(); + for (long time : new long[] {1, 2}) { + final TSRecord record = new TSRecord(time, "root.fail_rows"); + record.addTuple( + DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(time))); + nodes.add(buildInsertRowNodeByTSRecord(record)); + } + final InsertRowsNode insertRowsNode = new InsertRowsNode(new PlanNodeId(""), indexList, nodes); + + try { + dataRegion1.insert(insertRowsNode); + Assert.fail("Expected BatchProcessException"); + } catch (BatchProcessException e) { + Assert.assertEquals(2, insertRowsNode.getResults().size()); + Assert.assertEquals( + TSStatusCode.WRITE_PROCESS_ERROR.getStatusCode(), + insertRowsNode.getResults().get(0).getCode()); + Assert.assertEquals( + TSStatusCode.WRITE_PROCESS_ERROR.getStatusCode(), + insertRowsNode.getResults().get(1).getCode()); + } finally { + dataRegion1.syncDeleteDataFiles(); + } + } + + @Test + public void testInsertRowsLastCacheSkipsFailedRows() throws Exception { + final boolean originalLastCacheEnable = COMMON_CONFIG.isLastCacheEnable(); + COMMON_CONFIG.setLastCacheEnable(true); + + final HookedDataRegion dataRegion1 = new HookedDataRegion(systemDir, "root.cache_rows"); + final TsFileProcessor successProcessor = Mockito.mock(TsFileProcessor.class); + Mockito.when(successProcessor.shouldFlush()).thenReturn(false); + Mockito.when(successProcessor.isSequence()).thenReturn(true); + final long failingTime = TimePartitionUtils.getTimePartitionInterval() + 1; + final long failingPartitionId = TimePartitionUtils.getTimePartitionId(failingTime); + dataRegion1.setTsFileProcessorSupplier( + (timePartitionId, sequence) -> { + if (timePartitionId == failingPartitionId) { + throw new WriteProcessException("mock row failure"); + } + return successProcessor; + }); + + final MeasurementPath lastCachePath = + new MeasurementPath( + "root.cache_rows", + measurementId, + new MeasurementSchema( + measurementId, TSDataType.INT32, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED)); + DataNodeSchemaCache.getInstance() + .declareLastCache(dataRegion1.getDatabaseName(), lastCachePath); + + final List indexList = Arrays.asList(0, 1); + final List nodes = new ArrayList<>(); + final long[] times = new long[] {1, failingTime}; + final int[] values = new int[] {10, 20}; + for (int i = 0; i < times.length; i++) { + final long time = times[i]; + final TSRecord record = new TSRecord(time, "root.cache_rows"); + record.addTuple( + DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(values[i]))); + nodes.add(buildInsertRowNodeByTSRecord(record)); + } + final InsertRowsNode insertRowsNode = new InsertRowsNode(new PlanNodeId(""), indexList, nodes); + + try { + dataRegion1.insert(insertRowsNode); + Assert.fail("Expected BatchProcessException"); + } catch (BatchProcessException e) { + final TimeValuePair lastCache = DataNodeSchemaCache.getInstance().getLastCache(lastCachePath); + Assert.assertNotNull(lastCache); + Assert.assertEquals(1, lastCache.getTimestamp()); + Assert.assertEquals(10, lastCache.getValue().getInt()); + } finally { + dataRegion1.syncDeleteDataFiles(); + COMMON_CONFIG.setLastCacheEnable(originalLastCacheEnable); + } + } + + @Test + public void testInsertTabletLastCacheSkipsFailedRows() throws Exception { + final boolean originalLastCacheEnable = COMMON_CONFIG.isLastCacheEnable(); + COMMON_CONFIG.setLastCacheEnable(true); + + final HookedDataRegion dataRegion1 = new HookedDataRegion(systemDir, "root.cache_tablet"); + final TsFileProcessor processor = Mockito.mock(TsFileProcessor.class); + Mockito.doAnswer( + invocation -> { + TSStatus[] results = invocation.getArgument(3); + results[0] = RpcUtils.SUCCESS_STATUS; + results[1] = + RpcUtils.getStatus( + TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode(), "mock row failure"); + throw new WriteProcessException("mock tablet failure"); + }) + .when(processor) + .insertTablet(any(InsertTabletNode.class), anyInt(), anyInt(), any(TSStatus[].class)); + Mockito.when(processor.shouldFlush()).thenReturn(false); + Mockito.when(processor.isSequence()).thenReturn(true); + dataRegion1.setTsFileProcessorSupplier((timePartitionId, sequence) -> processor); + + final MeasurementPath lastCachePath = + new MeasurementPath( + "root.cache_tablet", + measurementId, + new MeasurementSchema( + measurementId, TSDataType.INT32, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED)); + DataNodeSchemaCache.getInstance() + .declareLastCache(dataRegion1.getDatabaseName(), lastCachePath); + + final String[] measurements = new String[] {measurementId}; + final TSDataType[] dataTypes = new TSDataType[] {TSDataType.INT32}; + final MeasurementSchema[] measurementSchemas = + new MeasurementSchema[] { + new MeasurementSchema(measurementId, TSDataType.INT32, TSEncoding.PLAIN) + }; + final long[] times = new long[] {1, 2}; + final Object[] columns = new Object[] {new int[] {10, 20}}; + final InsertTabletNode insertTabletNode = + new InsertTabletNode( + new QueryId("test_write").genPlanNodeId(), + new PartialPath("root.cache_tablet"), + false, + measurements, + dataTypes, + measurementSchemas, + times, + null, + columns, + times.length); + + try { + dataRegion1.insertTablet(insertTabletNode); + Assert.fail("Expected BatchProcessException"); + } catch (BatchProcessException e) { + final TimeValuePair lastCache = DataNodeSchemaCache.getInstance().getLastCache(lastCachePath); + Assert.assertNotNull(lastCache); + Assert.assertEquals(1, lastCache.getTimestamp()); + Assert.assertEquals(10, lastCache.getValue().getInt()); + } finally { + dataRegion1.syncDeleteDataFiles(); + COMMON_CONFIG.setLastCacheEnable(originalLastCacheEnable); + } + } + @Test public void testSmallReportProportionInsertRow() throws WriteProcessException, @@ -1490,6 +1684,32 @@ public DummyDataRegion(String systemInfoDir, String storageGroupName) } } + private interface TsFileProcessorSupplier { + TsFileProcessor get(long timePartitionId, boolean sequence) throws WriteProcessException; + } + + private static class HookedDataRegion extends DummyDataRegion { + private TsFileProcessorSupplier tsFileProcessorSupplier; + + private HookedDataRegion(String systemInfoDir, String storageGroupName) + throws DataRegionException { + super(systemInfoDir, storageGroupName); + } + + private void setTsFileProcessorSupplier(TsFileProcessorSupplier tsFileProcessorSupplier) { + this.tsFileProcessorSupplier = tsFileProcessorSupplier; + } + + @Override + protected TsFileProcessor getOrCreateTsFileProcessor(long timeRangeId, boolean sequence) + throws WriteProcessException { + if (tsFileProcessorSupplier != null) { + return tsFileProcessorSupplier.get(timeRangeId, sequence); + } + return super.getOrCreateTsFileProcessor(timeRangeId, sequence); + } + } + // -- test for deleting data directly // -- delete data and file only when: // 1. tsfile is closed diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java index 37d38411ff671..1a407648a51f2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/BatchedCompactionWithTsFileSplitterTest.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.load.LoadFileException; @@ -30,10 +31,15 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskSummary; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionCheckerUtils; +import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionTestFileWriter; +import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion; +import org.apache.iotdb.db.storageengine.dataregion.modification.Modification; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; import org.apache.iotdb.db.storageengine.load.splitter.AlignedChunkData; +import org.apache.iotdb.db.storageengine.load.splitter.DeletionData; import org.apache.iotdb.db.storageengine.load.splitter.TsFileData; import org.apache.iotdb.db.storageengine.load.splitter.TsFileSplitter; @@ -54,7 +60,9 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; +import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -233,6 +241,84 @@ public void testCompactionFlushPageAndSplitByTimePartition() consumeChunkDataAndValidate(targetResource); } + @Test + public void testDeletionDataShouldOnlyBeGeneratedOnceAtEnd() + throws IOException, MetadataException, LoadFileException, IllegalPathException { + TsFileResource resource = createAlignedMultiDeviceFile(); + try (ModificationFile modificationFile = ModificationFile.getNormalMods(resource)) { + modificationFile.write( + new Deletion(new MeasurementPath("root.testsg.d0.s0"), Long.MAX_VALUE, 100)); + modificationFile.write( + new Deletion(new MeasurementPath("root.testsg.d0.s1"), Long.MAX_VALUE, 200)); + modificationFile.write( + new Deletion(new MeasurementPath("root.testsg.d1.s0"), Long.MAX_VALUE, 300)); + modificationFile.write( + new Deletion(new MeasurementPath("root.testsg.d1.s1"), Long.MAX_VALUE, 400)); + } + + List expectedMods = + new ArrayList<>(ModificationFile.getNormalMods(resource).getModifications()); + List deletionMods = new ArrayList<>(); + File actualModsFile = new File(resource.getTsFilePath() + ".mods"); + try (ModificationFile actualModificationFile = + new ModificationFile(actualModsFile.getAbsolutePath())) { + TsFileSplitter splitter = + new TsFileSplitter( + resource.getTsFile(), + tsFileData -> { + if (tsFileData instanceof DeletionData) { + deletionMods.add(((DeletionData) tsFileData).getModEntry()); + } + return true; + }); + splitter.splitTsFileByDataPartition(); + } + + List actualMods; + try (ModificationFile actualModificationFile = + new ModificationFile(actualModsFile.getAbsolutePath())) { + actualMods = new ArrayList<>(actualModificationFile.getModifications()); + } + Assert.assertEquals(expectedMods, actualMods); + Files.deleteIfExists(actualModsFile.toPath()); + } + + private TsFileResource createAlignedMultiDeviceFile() throws IOException { + TsFileResource resource = createEmptyFileAndResource(true); + try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) { + writer.startChunkGroup("d0"); + writer.generateSimpleAlignedSeriesToCurrentDeviceWithNullValue( + Arrays.asList("s0", "s1"), + new TimeRange[][] { + new TimeRange[] {new TimeRange(1, 100), new TimeRange(200, 300)}, + new TimeRange[] { + new TimeRange(604799900, 604800020), new TimeRange(604810020, 604820020) + } + }, + TSEncoding.PLAIN, + CompressionType.LZ4, + Arrays.asList(false, false)); + writer.endChunkGroup(); + + writer.startChunkGroup("d1"); + writer.generateSimpleAlignedSeriesToCurrentDeviceWithNullValue( + Arrays.asList("s0", "s1"), + new TimeRange[][] { + new TimeRange[] {new TimeRange(1, 100), new TimeRange(200, 300)}, + new TimeRange[] { + new TimeRange(604799900, 604800020), new TimeRange(604810020, 604820020) + } + }, + TSEncoding.PLAIN, + CompressionType.LZ4, + Arrays.asList(false, false)); + writer.endChunkGroup(); + + writer.endFile(); + } + return resource; + } + private TsFileResource performCompaction() throws StorageEngineException, IOException, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java index f9186b774e1f3..b9259db0a0fed 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java @@ -177,6 +177,108 @@ public void testWriteDuringPrepareTVListAndActualQueryExecution() } } + @Test + public void testWriteAndFlushSortDuringQuerySortTVListAndActualQueryExecution() + throws QueryProcessException, IOException, IllegalPathException { + + PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0"); + List measurementSchemas = + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32), + new MeasurementSchema("s3", TSDataType.INT32)); + for (int i = 1000; i < 2000; i++) { + memTable.writeAlignedRow( + new PlainDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + for (int i = 100; i < 200; i++) { + memTable.writeAlignedRow( + new PlainDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + MeasurementPath path = new MeasurementPath("root.test.d1.s1", TSDataType.INT32); + memTable.delete(path, path.getDevicePath(), 150, 160); + path = new MeasurementPath("root.test.d1.s2", TSDataType.INT32); + memTable.delete(path, path.getDevicePath(), 150, 160); + path = new MeasurementPath("root.test.d1.s3", TSDataType.INT32); + memTable.delete(path, path.getDevicePath(), 150, 160); + ResourceByPathUtils resourcesByPathUtils = + ResourceByPathUtils.getResourceInstance( + new AlignedPath("root.test.d1", Arrays.asList("s1", "s2", "s3"), measurementSchemas)); + ReadOnlyMemChunk readOnlyMemChunk = + resourcesByPathUtils.getReadOnlyMemChunkFromMemTable( + new QueryContext(), memTable, null, Long.MAX_VALUE, null); + + for (int i = 1; i <= 50; i++) { + memTable.writeAlignedRow( + new PlainDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); + } + memTable.getWritableMemChunk(new PlainDeviceID("root.test.d1"), "").sortTvListForFlush(); + + readOnlyMemChunk.sortTvLists(); + + MemPointIterator memPointIterator = readOnlyMemChunk.createMemPointIterator(Ordering.ASC, null); + while (memPointIterator.hasNextBatch()) { + memPointIterator.nextBatch(); + } + } + + @Test + public void testFlushingQueryDoesNotSortWorkingTVListUsedByPreviousQuery() + throws QueryProcessException, IOException, IllegalPathException { + + PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0"); + List measurementSchemas = + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32), + new MeasurementSchema("s3", TSDataType.INT32)); + PlainDeviceID deviceID = new PlainDeviceID("root.test.d1"); + for (int i = 1000; i < 2000; i++) { + memTable.writeAlignedRow(deviceID, measurementSchemas, i, new Object[] {i, i, i}); + } + + ResourceByPathUtils resourcesByPathUtils = + ResourceByPathUtils.getResourceInstance( + new AlignedPath("root.test.d1", Arrays.asList("s1", "s2", "s3"), measurementSchemas)); + AlignedReadOnlyMemChunk firstQueryMemChunk = + (AlignedReadOnlyMemChunk) + resourcesByPathUtils.getReadOnlyMemChunkFromMemTable( + new QueryContext(1), memTable, null, Long.MAX_VALUE, null); + TVList originalWorkingList = memTable.getWritableMemChunk(deviceID, "").getWorkingTVList(); + Assert.assertSame( + originalWorkingList, + firstQueryMemChunk.getAligendTvListQueryMap().keySet().iterator().next()); + + for (int i = 1; i <= 50; i++) { + memTable.writeAlignedRow(deviceID, measurementSchemas, i, new Object[] {i, i, i}); + } + MeasurementPath path = new MeasurementPath("root.test.d1.s1", TSDataType.INT32); + memTable.delete(path, path.getDevicePath(), 1, 10); + path = new MeasurementPath("root.test.d1.s2", TSDataType.INT32); + memTable.delete(path, path.getDevicePath(), 1, 10); + path = new MeasurementPath("root.test.d1.s3", TSDataType.INT32); + memTable.delete(path, path.getDevicePath(), 1, 10); + Assert.assertFalse(originalWorkingList.isSorted()); + + AlignedReadOnlyMemChunk flushingQueryMemChunk = + (AlignedReadOnlyMemChunk) + resourcesByPathUtils.getReadOnlyMemChunkFromMemTable( + new QueryContext(2), memTable, new ArrayList<>(), Long.MAX_VALUE, null); + TVList flushingQueryList = + flushingQueryMemChunk.getAligendTvListQueryMap().keySet().iterator().next(); + Assert.assertNotSame(originalWorkingList, flushingQueryList); + + flushingQueryMemChunk.sortTvLists(); + Assert.assertFalse(originalWorkingList.isSorted()); + + firstQueryMemChunk.sortTvLists(); + MemPointIterator memPointIterator = + firstQueryMemChunk.createMemPointIterator(Ordering.ASC, null); + while (memPointIterator.hasNextBatch()) { + memPointIterator.nextBatch(); + } + } + @Test public void memSeriesToStringTest() throws IOException { TSDataType dataType = TSDataType.INT32; @@ -679,7 +781,7 @@ public void testReleaseWithNotEnoughMemory() throws CpuNotEnoughException { Mockito.mock(MemoryReservationManager.class); Mockito.doThrow(new MemoryNotEnoughException("")) .when(memoryReservationManager) - .reserveMemoryCumulatively(list.calculateRamSize()); + .reserveMemoryCumulatively(list.calculateRamSize().getRamSize()); // create FragmentInstanceId QueryId queryId = new QueryId("stub_query"); @@ -698,7 +800,7 @@ public void testReleaseWithNotEnoughMemory() throws CpuNotEnoughException { list.getQueryContextSet().add(queryContext); Map tvlistMap = new HashMap<>(); tvlistMap.put(list, 100); - queryContext.addTVListToSet(tvlistMap); + queryContext.addTVListToSet(tvlistMap.keySet()); // fragment instance execution IDriverScheduler scheduler = Mockito.mock(IDriverScheduler.class); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionTest.java new file mode 100644 index 0000000000000..c8c1a7fd0ca8d --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.modification; + +import org.apache.iotdb.commons.path.PartialPath; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; + +import static org.junit.Assert.assertEquals; + +public class DeletionTest { + + @Test + public void testSerializedSize() throws Exception { + Deletion deletion = + new Deletion(new PartialPath("root.\u6570\u636e\u5e93.d1.\u6e29\u5ea6"), 0, 1, 5); + + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + long serializedSize = + deletion.serializeWithoutFileOffset(new DataOutputStream(byteArrayOutputStream)); + byte[] bytes = byteArrayOutputStream.toByteArray(); + + assertEquals(deletion.getSerializedSize(), serializedSize); + assertEquals(deletion.getSerializedSize(), bytes.length); + assertEquals( + deletion, + Deletion.deserializeWithoutFileOffset( + new DataInputStream(new ByteArrayInputStream(bytes)))); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java index f483bcfd49265..8e2f45ce5564c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALFileTest.java @@ -61,7 +61,7 @@ public class WALFileTest { new File( TestConstant.BASE_OUTPUT_PATH.concat( WALFileUtils.getLogFileName(0, 0, WALFileStatus.CONTAINS_SEARCH_INDEX))); - private final String devicePath = "root.test_sg.test_d"; + private final String devicePath = "root.\u6570\u636e\u5e93.test_d"; @Before public void setUp() throws Exception { @@ -171,6 +171,16 @@ public void testReadMetadataFromBrokenFile() throws IOException { assertTrue(walMetaData.getMemTablesId().isEmpty()); } + @Test + public void testInsertTabletEntrySerializedSizeWithRange() + throws IOException, IllegalPathException { + WALEntry walEntry = new WALInfoEntry(1, getInsertTabletNode(devicePath), 1, 3); + ByteBuffer byteBuffer = ByteBuffer.allocate(walEntry.serializedSize()); + WALByteBufferForTest buffer = new WALByteBufferForTest(byteBuffer); + walEntry.serialize(buffer); + assertEquals(walEntry.serializedSize(), byteBuffer.position()); + } + public static InsertRowNode getInsertRowNode(String devicePath) throws IllegalPathException { long time = 110L; TSDataType[] dataTypes = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeWaitForRollFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeWaitForRollFileTest.java new file mode 100644 index 0000000000000..b75efaefca4fa --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeWaitForRollFileTest.java @@ -0,0 +1,406 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.storageengine.dataregion.wal.node; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.consensus.iot.log.ConsensusReqReader; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ContinuousSameSearchIndexSeparatorNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.storageengine.StorageEngine; +import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; +import org.apache.iotdb.db.storageengine.dataregion.memtable.PrimitiveMemTable; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALMode; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.db.utils.constant.TestConstant; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BitMap; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.awaitility.Awaitility; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class WALNodeWaitForRollFileTest { + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private static final String identifier = String.valueOf(Integer.MAX_VALUE); + private static final String logDirectory = + TestConstant.BASE_OUTPUT_PATH.concat("wal-roll-file-test"); + private static final String databasePath = "root.test_sg"; + private static final String devicePath = databasePath + ".test_d"; + private static final String dataRegionId = "1"; + private WALMode prevMode; + private String prevConsensus; + private WALNode walNode; + private long originWALThreshold; + + @Before + public void setUp() throws Exception { + originWALThreshold = config.getWalFileSizeThresholdInByte(); + EnvironmentUtils.cleanDir(logDirectory); + prevMode = config.getWalMode(); + prevConsensus = config.getDataRegionConsensusProtocolClass(); + config.setWalMode(WALMode.SYNC); + config.setDataRegionConsensusProtocolClass(ConsensusFactory.IOT_CONSENSUS); + config.setWalFileSizeThresholdInByte(2 * 1024 * 1024); + walNode = new WALNode(identifier, logDirectory); + } + + @After + public void tearDown() throws Exception { + walNode.close(); + config.setWalMode(prevMode); + config.setDataRegionConsensusProtocolClass(prevConsensus); + config.setWalFileSizeThresholdInByte(originWALThreshold); + EnvironmentUtils.cleanDir(logDirectory); + StorageEngine.getInstance().reset(); + } + + /** + * Verifies that waitForNextReady(time, unit) throws TimeoutException when no WAL data is + * available at the requested search index. This uses waitForRollFile internally. + */ + @Test + public void testWaitForNextReadyTimesOutWhenNoData() throws Exception { + ConsensusReqReader.ReqIterator iterator = walNode.getReqIterator(1); + assertFalse(iterator.hasNext()); + try { + iterator.waitForNextReady(1, TimeUnit.SECONDS); + fail("Expected TimeoutException"); + } catch (TimeoutException e) { + // expected + } + } + + /** + * Verifies that waitForNextReady(time, unit) does NOT wake up from a buffer flush alone — it + * requires a WAL file roll. This is the core behavioral change: the old waitForFlush would return + * on any buffer sync, but waitForRollFile only returns when a new WAL file is created. + */ + @Test + public void testWaitForNextReadyNotWokenByFlushWithoutRoll() throws Exception { + IMemTable memTable = new PrimitiveMemTable(databasePath, dataRegionId); + walNode.onMemTableCreated(memTable, logDirectory + File.separator + "test.tsfile"); + + // write a small amount of data (not enough to trigger roll) + InsertTabletNode insertTabletNode = getInsertTabletNode(devicePath, new long[] {1}); + insertTabletNode.setSearchIndex(1); + walNode.log(memTable.getMemTableId(), insertTabletNode, 0, insertTabletNode.getRowCount()); + + Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> walNode.isAllWALEntriesConsumed()); + + // data is flushed to buffer but no WAL file roll happened yet, iterator at search index 1 + // should not find data (because the current-writing WAL file is not readable by the iterator) + ConsensusReqReader.ReqIterator iterator = walNode.getReqIterator(1); + + try { + long start = System.currentTimeMillis(); + iterator.waitForNextReady(2, TimeUnit.SECONDS); + if (System.currentTimeMillis() - start + < WALNode.WAIT_FOR_NEXT_WAL_ENTRY_TIMEOUT_IN_SEC * 1000) { + fail("The data should not be found before timeout"); + } + } catch (TimeoutException e) { + // expected: flush happened but no roll, so waitForRollFile timed out + } + } + + /** + * Verifies that waitForNextReady succeeds after a WAL file roll makes data readable. The iterator + * should wake up when rollLogWriter signals the rollLogWriterCondition. + */ + @Test + public void testWaitForNextReadySucceedsAfterRollFile() throws Exception { + IMemTable memTable = new PrimitiveMemTable(databasePath, dataRegionId); + walNode.onMemTableCreated(memTable, logDirectory + File.separator + "test.tsfile"); + + // write data with search index + for (int i = 1; i <= 5; i++) { + InsertTabletNode insertTabletNode = getInsertTabletNode(devicePath, new long[] {i}); + insertTabletNode.setSearchIndex(i); + walNode.log(memTable.getMemTableId(), insertTabletNode, 0, insertTabletNode.getRowCount()); + } + + Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> walNode.isAllWALEntriesConsumed()); + + // roll the WAL file so the data is in a closed file readable by the iterator + walNode.rollWALFile(); + Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> walNode.isAllWALEntriesConsumed()); + + // iterator at search index 1 should find the data after roll + ConsensusReqReader.ReqIterator iterator = walNode.getReqIterator(1); + assertTrue(iterator.hasNext()); + assertNotNull(iterator.next()); + } + + /** + * Verifies that waitForNextReady wakes up when a WAL file roll is triggered concurrently. A + * background thread rolls the WAL file while the main thread waits on the iterator. + */ + @Test(timeout = 30000) + public void testWaitForNextReadyWakesUpOnConcurrentRoll() throws Exception { + IMemTable memTable = new PrimitiveMemTable(databasePath, dataRegionId); + walNode.onMemTableCreated(memTable, logDirectory + File.separator + "test.tsfile"); + + // write data with search index + InsertTabletNode insertTabletNode = getInsertTabletNode(devicePath, new long[] {1}); + insertTabletNode.setSearchIndex(1); + walNode.log(memTable.getMemTableId(), insertTabletNode, 0, insertTabletNode.getRowCount()); + walNode.log( + memTable.getMemTableId(), new ContinuousSameSearchIndexSeparatorNode(new PlanNodeId(""))); + + Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> walNode.isAllWALEntriesConsumed()); + + ConsensusReqReader.ReqIterator iterator = walNode.getReqIterator(1); + + AtomicBoolean found = new AtomicBoolean(false); + AtomicReference error = new AtomicReference<>(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + + // background: wait for data to become available via waitForNextReady + Future waitFuture = + executor.submit( + () -> { + try { + iterator.waitForNextReady(15, TimeUnit.SECONDS); + if (iterator.hasNext()) { + found.set(true); + } + } catch (Exception e) { + error.set(e); + } + }); + + // give the waiter thread time to start blocking + Thread.sleep(500); + + // trigger WAL file roll — this should signal rollLogWriterCondition and wake up the iterator + walNode.rollWALFile(); + Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> walNode.isAllWALEntriesConsumed()); + + waitFuture.get(20, TimeUnit.SECONDS); + executor.shutdown(); + + if (error.get() != null) { + throw error.get(); + } + assertTrue("Iterator should have found data after WAL file roll", found.get()); + } + + /** + * Verifies that the no-arg waitForNextReady eventually proceeds when enough data is written to + * trigger an automatic WAL file roll (file size exceeds threshold). Uses a small WAL file size + * threshold to trigger the roll quickly. + */ + @Test(timeout = 60000) + public void testWaitForNextReadyWithAutoRollOnSizeThreshold() throws Exception { + // use small WAL file size to trigger auto-roll + config.setWalFileSizeThresholdInByte(1024); + + try { + IMemTable memTable = new PrimitiveMemTable(databasePath, dataRegionId); + walNode.onMemTableCreated(memTable, logDirectory + File.separator + "test.tsfile"); + + // write initial data with search index + InsertTabletNode first = getInsertTabletNode(devicePath, new long[] {1}); + first.setSearchIndex(1); + walNode.log(memTable.getMemTableId(), first, 0, first.getRowCount()); + + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .until(() -> walNode.isAllWALEntriesConsumed()); + + ConsensusReqReader.ReqIterator iterator = walNode.getReqIterator(1); + + AtomicBoolean found = new AtomicBoolean(false); + AtomicReference error = new AtomicReference<>(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + + Future waitFuture = + executor.submit( + () -> { + try { + iterator.waitForNextReady(30, TimeUnit.SECONDS); + if (iterator.hasNext()) { + found.set(true); + } + } catch (Exception e) { + error.set(e); + } + }); + + Thread.sleep(500); + + // write more data to exceed the small threshold and trigger auto-roll + for (int i = 2; i <= 50; i++) { + InsertTabletNode node = getInsertTabletNode(devicePath, new long[] {i}); + node.setSearchIndex(i); + walNode.log(memTable.getMemTableId(), node, 0, node.getRowCount()); + } + + waitFuture.get(40, TimeUnit.SECONDS); + executor.shutdown(); + + if (error.get() != null) { + fail("waitForNextReady threw unexpected exception: " + error.get().getMessage()); + } + assertTrue("Iterator should have found data after auto WAL file roll", found.get()); + } finally { + config.setWalFileSizeThresholdInByte(2 * 1024 * 1024); + } + } + + /** + * Verifies that the no-arg waitForNextReady() automatically triggers a WAL file roll after the + * timeout expires (WAIT_FOR_NEXT_WAL_ENTRY_TIMEOUT_IN_SEC = 30s). The flow is: data written to + * buffer → waitForRollFile(30s) times out → rollWALFile() called → data moves to closed file → + * hasNext() returns true → method returns. + */ + @Test(timeout = 120000) + public void testWaitForNextReadyAutoTriggersRollOnTimeout() throws Exception { + IMemTable memTable = new PrimitiveMemTable(databasePath, dataRegionId); + walNode.onMemTableCreated(memTable, logDirectory + File.separator + "test.tsfile"); + + // write data with search index — stays in the current (active) WAL file + InsertTabletNode insertTabletNode = getInsertTabletNode(devicePath, new long[] {1}); + insertTabletNode.setSearchIndex(1); + walNode.log(memTable.getMemTableId(), insertTabletNode, 0, insertTabletNode.getRowCount()); + walNode.log( + memTable.getMemTableId(), new ContinuousSameSearchIndexSeparatorNode(new PlanNodeId(""))); + + Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() -> walNode.isAllWALEntriesConsumed()); + + // iterator cannot read the active WAL file, so hasNext() should be false + ConsensusReqReader.ReqIterator iterator = walNode.getReqIterator(1); + assertFalse("Data should not be visible before WAL file roll", iterator.hasNext()); + + AtomicBoolean found = new AtomicBoolean(false); + AtomicReference error = new AtomicReference<>(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + + long startTime = System.currentTimeMillis(); + + // call the no-arg waitForNextReady() — it should: + // 1) wait 30s for rollLogWriterCondition (timeout) + // 2) auto-call rollWALFile() + // 3) data becomes readable, hasNext() returns true, method returns + Future waitFuture = + executor.submit( + () -> { + try { + iterator.waitForNextReady(); + if (iterator.hasNext()) { + found.set(true); + } + } catch (Exception e) { + error.set(e); + } + }); + + waitFuture.get(90, TimeUnit.SECONDS); + executor.shutdown(); + + long elapsed = System.currentTimeMillis() - startTime; + + if (error.get() != null) { + fail("waitForNextReady() threw unexpected exception: " + error.get().getMessage()); + } + assertTrue("Iterator should have found data after auto-triggered WAL file roll", found.get()); + assertTrue( + "Should have waited at least 30s for the timeout to trigger auto-roll, but only waited " + + elapsed + + "ms", + elapsed >= TimeUnit.SECONDS.toMillis(WALNode.WAIT_FOR_NEXT_WAL_ENTRY_TIMEOUT_IN_SEC - 1)); + } + + private InsertTabletNode getInsertTabletNode(String devicePath, long[] times) + throws IllegalPathException { + String[] measurements = new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}; + TSDataType[] dataTypes = new TSDataType[6]; + dataTypes[0] = TSDataType.DOUBLE; + dataTypes[1] = TSDataType.FLOAT; + dataTypes[2] = TSDataType.INT64; + dataTypes[3] = TSDataType.INT32; + dataTypes[4] = TSDataType.BOOLEAN; + dataTypes[5] = TSDataType.TEXT; + + Object[] columns = new Object[6]; + columns[0] = new double[times.length]; + columns[1] = new float[times.length]; + columns[2] = new long[times.length]; + columns[3] = new int[times.length]; + columns[4] = new boolean[times.length]; + columns[5] = new Binary[times.length]; + + for (int r = 0; r < times.length; r++) { + ((double[]) columns[0])[r] = 1.0d + r; + ((float[]) columns[1])[r] = 2.0f + r; + ((long[]) columns[2])[r] = 10000L + r; + ((int[]) columns[3])[r] = 100 + r; + ((boolean[]) columns[4])[r] = (r % 2 == 0); + ((Binary[]) columns[5])[r] = new Binary("hh" + r, TSFileConfig.STRING_CHARSET); + } + + BitMap[] bitMaps = new BitMap[dataTypes.length]; + for (int i = 0; i < dataTypes.length; i++) { + if (bitMaps[i] == null) { + bitMaps[i] = new BitMap(times.length); + } + bitMaps[i].mark(i % times.length); + } + MeasurementSchema[] schemas = new MeasurementSchema[6]; + for (int i = 0; i < 6; i++) { + schemas[i] = new MeasurementSchema(measurements[i], dataTypes[i], TSEncoding.PLAIN); + } + + return new InsertTabletNode( + new PlanNodeId(""), + new PartialPath(devicePath), + false, + measurements, + dataTypes, + schemas, + times, + bitMaps, + columns, + times.length); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManagerTest.java new file mode 100644 index 0000000000000..f3a1bf7e4116f --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/memory/LoadTsFileMemoryManagerTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.load.memory; + +import org.apache.iotdb.db.exception.load.LoadRuntimeOutOfMemoryException; + +import org.junit.Assert; +import org.junit.Test; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.util.concurrent.atomic.AtomicLong; + +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; + +public class LoadTsFileMemoryManagerTest { + + @Test + public void testAllocateDataCacheMemoryBlockDoesNotDoubleCountMemory() throws Exception { + final long allocatedMemoryInBytes = 2L * 1024 * 1024; + final LoadTsFileMemoryManager manager = spy(newMemoryManager()); + + doAnswer( + invocation -> { + setUsedMemorySize(manager, allocatedMemoryInBytes); + return allocatedMemoryInBytes; + }) + .when(manager) + .tryAllocateFromQuery(anyLong()); + + manager.allocateDataCacheMemoryBlock(); + + Assert.assertEquals(allocatedMemoryInBytes, manager.getUsedMemorySizeInBytes()); + Assert.assertNotNull(getDataCacheMemoryBlock(manager)); + } + + @Test + public void testAllocateDataCacheMemoryBlockRollsBackPartialAllocationOnFailure() + throws Exception { + final long allocatedMemoryInBytes = 512L; + final LoadTsFileMemoryManager manager = spy(newMemoryManager()); + + doAnswer( + invocation -> { + setUsedMemorySize(manager, allocatedMemoryInBytes); + return allocatedMemoryInBytes; + }) + .when(manager) + .tryAllocateFromQuery(anyLong()); + doAnswer( + invocation -> { + setUsedMemorySize(manager, 0L); + return null; + }) + .when(manager) + .releaseToQuery(anyLong()); + + try { + manager.allocateDataCacheMemoryBlock(); + Assert.fail("Expected LoadRuntimeOutOfMemoryException"); + } catch (LoadRuntimeOutOfMemoryException e) { + Assert.assertEquals(0L, manager.getUsedMemorySizeInBytes()); + Assert.assertNull(getDataCacheMemoryBlock(manager)); + } + } + + private static LoadTsFileMemoryManager newMemoryManager() throws Exception { + final Constructor constructor = + LoadTsFileMemoryManager.class.getDeclaredConstructor(); + constructor.setAccessible(true); + return constructor.newInstance(); + } + + private static void setUsedMemorySize( + final LoadTsFileMemoryManager manager, final long usedMemorySizeInBytes) throws Exception { + final Field field = LoadTsFileMemoryManager.class.getDeclaredField("usedMemorySizeInBytes"); + field.setAccessible(true); + ((AtomicLong) field.get(manager)).set(usedMemorySizeInBytes); + } + + private static LoadTsFileDataCacheMemoryBlock getDataCacheMemoryBlock( + final LoadTsFileMemoryManager manager) throws Exception { + final Field field = LoadTsFileMemoryManager.class.getDeclaredField("dataCacheMemoryBlock"); + field.setAccessible(true); + return (LoadTsFileDataCacheMemoryBlock) field.get(manager); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/ChunkDataDirectWriteTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/ChunkDataDirectWriteTest.java new file mode 100644 index 0000000000000..e98541a9125b3 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/load/splitter/ChunkDataDirectWriteTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.load.splitter; + +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.metadata.IChunkMetadata; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.junit.Test; +import org.mockito.Mockito; + +import java.nio.ByteBuffer; + +public class ChunkDataDirectWriteTest { + + @Test + public void testNonAlignedChunkDataCanWriteWithoutSerdeRoundTrip() throws Exception { + final NonAlignedChunkData chunkData = createNonAlignedChunkData(); + chunkData.setNotDecode(); + final IChunkMetadata chunkMetadata = Mockito.mock(IChunkMetadata.class); + Mockito.doReturn(createInt32Statistics()).when(chunkMetadata).getStatistics(); + chunkData.writeEntireChunk(ByteBuffer.allocate(0), chunkMetadata); + + final TsFileIOWriter writer = Mockito.mock(TsFileIOWriter.class); + chunkData.writeToFileWriter(writer); + + Mockito.verify(writer).writeChunk(Mockito.any(Chunk.class)); + } + + @Test + public void testAlignedChunkDataCanWriteWithoutSerdeRoundTrip() throws Exception { + final AlignedChunkData chunkData = createAlignedChunkData(); + chunkData.setNotDecode(); + final IChunkMetadata chunkMetadata = Mockito.mock(IChunkMetadata.class); + Mockito.doReturn(createInt32Statistics()).when(chunkMetadata).getStatistics(); + chunkData.writeEntireChunk(ByteBuffer.allocate(0), chunkMetadata); + + final TsFileIOWriter writer = Mockito.mock(TsFileIOWriter.class); + chunkData.writeToFileWriter(writer); + + Mockito.verify(writer).writeChunk(Mockito.any(Chunk.class)); + } + + private static Statistics createInt32Statistics() { + final Statistics statistics = Statistics.getStatsByType(TSDataType.INT32); + statistics.update(1L, 1); + return statistics; + } + + private static NonAlignedChunkData createNonAlignedChunkData() { + return (NonAlignedChunkData) + ChunkData.createChunkData( + false, "root.sg.d1", createChunkHeader(), new TTimePartitionSlot(0L)); + } + + private static AlignedChunkData createAlignedChunkData() { + return (AlignedChunkData) + ChunkData.createChunkData( + true, "root.sg.d1", createChunkHeader(), new TTimePartitionSlot(0L)); + } + + private static ChunkHeader createChunkHeader() { + return new ChunkHeader( + "temperature", 0, TSDataType.INT32, CompressionType.UNCOMPRESSED, TSEncoding.PLAIN, 0); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuotaTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuotaTest.java new file mode 100644 index 0000000000000..5f11039eeb312 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/quotas/DefaultOperationQuotaTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.rescon.quotas; + +import org.apache.iotdb.common.rpc.thrift.TTimedQuota; +import org.apache.iotdb.common.rpc.thrift.ThrottleType; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; + +import org.apache.tsfile.utils.BitMap; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumMap; +import java.util.Map; + +public class DefaultOperationQuotaTest { + + @Test + public void testCheckQuotaWithNullAndSparseBitMaps() throws Exception { + final DefaultOperationQuota quota = new DefaultOperationQuota(createQuotaLimiter()); + + final InsertTabletStatement tabletWithoutBitMaps = new InsertTabletStatement(); + tabletWithoutBitMaps.setBitMaps(null); + quota.checkQuota(1, 0, tabletWithoutBitMaps); + + final InsertTabletStatement tabletWithSparseBitMaps = new InsertTabletStatement(); + final BitMap bitMap = new BitMap(8); + bitMap.mark(0); + tabletWithSparseBitMaps.setBitMaps(new BitMap[] {null, bitMap}); + quota.checkQuota(1, 0, tabletWithSparseBitMaps); + + final InsertMultiTabletsStatement multiTabletsStatement = new InsertMultiTabletsStatement(); + multiTabletsStatement.setInsertTabletStatementList( + Arrays.asList(tabletWithoutBitMaps, tabletWithSparseBitMaps)); + quota.checkQuota(1, 0, multiTabletsStatement); + } + + private static QuotaLimiter createQuotaLimiter() { + final Map quotas = new EnumMap<>(ThrottleType.class); + for (final ThrottleType throttleType : ThrottleType.values()) { + quotas.put(throttleType, new TTimedQuota(60_000L, 1_000_000_000L)); + } + return QuotaLimiter.fromThrottle(Collections.unmodifiableMap(quotas)); + } +} diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 018551d889a57..741a4c846cdee 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -465,9 +465,10 @@ cn_rpc_max_concurrent_client_num=3000 cn_connection_timeout_ms=60000 # selector thread (TAsyncClientManager) nums for async thread in a clientManager +# When <= 0, use max(1, CPU core number / 4). # effectiveMode: restart # Datatype: int -cn_selector_thread_nums_of_client_manager=1 +cn_selector_thread_nums_of_client_manager=0 # The maximum number of clients that can be allocated for a node in a clientManager. # when the number of the client to a single node exceeds this number, the thread for applying for a client will be blocked @@ -476,6 +477,14 @@ cn_selector_thread_nums_of_client_manager=1 # Datatype: int cn_max_client_count_for_each_node_in_client_manager=1000 +# The maximum number of idle clients that can be retained for a node in a clientManager. +# When the number of idle clients to a single node exceeds this number, excess idle clients will be evicted. +# Idle clients are determined by a time threshold (default 1 minute of inactivity). +# 0 means no idle clients will be retained, connections are destroyed immediately upon return. +# effectiveMode: restart +# Datatype: int +# cn_max_idle_client_count_for_each_node_in_client_manager=1000 + # The maximum session idle time. unit: ms # Idle sessions are the ones that performs neither query or non-query operations for a period of time # Set to 0 to disable session timeout @@ -494,25 +503,15 @@ dn_rpc_thrift_compression_enable=false # this feature is under development, set this as false before it is done. dn_rpc_advanced_compression_enable=false -# the number of rpc selector -# effectiveMode: restart -# Datatype: int -dn_rpc_selector_thread_count=1 - -# The min number of concurrent clients that can be connected to the dataNode. -# effectiveMode: restart -# Datatype: int -dn_rpc_min_concurrent_client_num=1 - # The maximum number of concurrent clients that can be connected to the dataNode. # effectiveMode: restart # Datatype: int dn_rpc_max_concurrent_client_num=1000 -# thrift max frame size, 512MB by default +# thrift max frame size in bytes. When set to 0, use min(64MB, datanode heap memory / 64) # effectiveMode: restart # Datatype: int -dn_thrift_max_frame_size=536870912 +dn_thrift_max_frame_size=0 # thrift init buffer size # effectiveMode: restart @@ -525,9 +524,10 @@ dn_thrift_init_buffer_size=1024 dn_connection_timeout_ms=60000 # selector thread (TAsyncClientManager) nums for async thread in a clientManager +# When <= 0, use max(1, CPU core number / 4). # effectiveMode: restart # Datatype: int -dn_selector_thread_count_of_client_manager=1 +dn_selector_thread_nums_of_client_manager=0 # The maximum number of clients that can be allocated for a node in a clientManager. # When the number of the client to a single node exceeds this number, the thread for applying for a client will be blocked @@ -536,6 +536,14 @@ dn_selector_thread_count_of_client_manager=1 # Datatype: int dn_max_client_count_for_each_node_in_client_manager=1000 +# The maximum number of idle clients that can be retained for a node in a clientManager. +# When the number of idle clients to a single node exceeds this number, excess idle clients will be evicted. +# Idle clients are determined by a time threshold (default 1 minute of inactivity). +# 0 means no idle clients will be retained, connections are destroyed immediately upon return. +# effectiveMode: restart +# Datatype: int +# dn_max_idle_client_count_for_each_node_in_client_manager=1000 + #################### ### REST Service Configuration #################### @@ -2025,6 +2033,17 @@ load_clean_up_task_execution_delay_time_seconds=1800 # Datatype: int load_write_throughput_bytes_per_second=-1 +# Whether the load_tsfile supports path allowed dirs check. +# effectiveMode: hot_reload +# Datatype: String +load_tsfile_source_path_check_enable=false + +# Comma-separated list of directories from which user-issued LOAD TSFILE statements can read. +# If empty, IoTDB only permits LOAD sources under the internal load TsFile directories. +# effectiveMode: hot_reload +# Datatype: String +load_tsfile_allowed_dirs= + # Whether to enable the active listening mode for tsfile loading. # effectiveMode: hot_reload # Datatype: Boolean diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java index 3ff47a2c5e1d6..0a6b1da048ac3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java @@ -72,6 +72,12 @@ public GenericKeyedObjectPool createCli public static class AsyncConfigNodeInternalServiceClientPoolFactory implements IClientPoolFactory { + private final int selectorNumOfAsyncClientManager; + + public AsyncConfigNodeInternalServiceClientPoolFactory(int selectorNumOfAsyncClientManager) { + this.selectorNumOfAsyncClientManager = selectorNumOfAsyncClientManager; + } + @Override public GenericKeyedObjectPool createClientPool( ClientManager manager) { @@ -82,7 +88,7 @@ public GenericKeyedObjectPool c new ThriftClientProperty.Builder() .setConnectionTimeoutMs(conf.getCnConnectionTimeoutInMS()) .setRpcThriftCompressionEnabled(conf.isRpcThriftCompressionEnabled()) - .setSelectorNumOfAsyncClientManager(conf.getSelectorNumOfClientManager()) + .setSelectorNumOfAsyncClientManager(selectorNumOfAsyncClientManager) .build(), ThreadName.ASYNC_CONFIGNODE_CLIENT_POOL.getName()), new ClientPoolProperty.Builder() @@ -120,6 +126,12 @@ public GenericKeyedObjectPool crea public static class AsyncDataNodeInternalServiceClientPoolFactory implements IClientPoolFactory { + private final int selectorNumOfAsyncClientManager; + + public AsyncDataNodeInternalServiceClientPoolFactory(int selectorNumOfAsyncClientManager) { + this.selectorNumOfAsyncClientManager = selectorNumOfAsyncClientManager; + } + @Override public GenericKeyedObjectPool createClientPool( ClientManager manager) { @@ -130,7 +142,8 @@ public GenericKeyedObjectPool cre new ThriftClientProperty.Builder() .setConnectionTimeoutMs(conf.getDnConnectionTimeoutInMS()) .setRpcThriftCompressionEnabled(conf.isRpcThriftCompressionEnabled()) - .setSelectorNumOfAsyncClientManager(conf.getSelectorNumOfClientManager()) + .setSelectorNumOfAsyncClientManager(selectorNumOfAsyncClientManager) + .setPrintLogWhenEncounterException(false) .build(), ThreadName.ASYNC_DATANODE_CLIENT_POOL.getName()), new ClientPoolProperty.Builder() @@ -145,6 +158,12 @@ public GenericKeyedObjectPool cre public static class AsyncDataNodeExternalServiceClientPoolFactory implements IClientPoolFactory { + private final int selectorNumOfAsyncClientManager; + + public AsyncDataNodeExternalServiceClientPoolFactory(int selectorNumOfAsyncClientManager) { + this.selectorNumOfAsyncClientManager = selectorNumOfAsyncClientManager; + } + @Override public GenericKeyedObjectPool createClientPool( ClientManager manager) { @@ -155,7 +174,7 @@ public GenericKeyedObjectPool cre new ThriftClientProperty.Builder() .setConnectionTimeoutMs(conf.getDnConnectionTimeoutInMS()) .setRpcThriftCompressionEnabled(conf.isRpcThriftCompressionEnabled()) - .setSelectorNumOfAsyncClientManager(conf.getSelectorNumOfClientManager()) + .setSelectorNumOfAsyncClientManager(selectorNumOfAsyncClientManager) .build(), ThreadName.ASYNC_DATANODE_CLIENT_POOL.getName()), new ClientPoolProperty.Builder() @@ -170,6 +189,12 @@ public GenericKeyedObjectPool cre public static class AsyncConfigNodeHeartbeatServiceClientPoolFactory implements IClientPoolFactory { + private final int selectorNumOfAsyncClientManager; + + public AsyncConfigNodeHeartbeatServiceClientPoolFactory(int selectorNumOfAsyncClientManager) { + this.selectorNumOfAsyncClientManager = selectorNumOfAsyncClientManager; + } + @Override public GenericKeyedObjectPool createClientPool( ClientManager manager) { @@ -181,7 +206,7 @@ public GenericKeyedObjectPool c new ThriftClientProperty.Builder() .setConnectionTimeoutMs(conf.getCnConnectionTimeoutInMS()) .setRpcThriftCompressionEnabled(conf.isRpcThriftCompressionEnabled()) - .setSelectorNumOfAsyncClientManager(conf.getSelectorNumOfClientManager()) + .setSelectorNumOfAsyncClientManager(selectorNumOfAsyncClientManager) .setPrintLogWhenEncounterException(false) .build(), ThreadName.ASYNC_CONFIGNODE_HEARTBEAT_CLIENT_POOL.getName()), @@ -196,6 +221,13 @@ public GenericKeyedObjectPool c public static class AsyncDataNodeHeartbeatServiceClientPoolFactory implements IClientPoolFactory { + + private final int selectorNumOfAsyncClientManager; + + public AsyncDataNodeHeartbeatServiceClientPoolFactory(int selectorNumOfAsyncClientManager) { + this.selectorNumOfAsyncClientManager = selectorNumOfAsyncClientManager; + } + @Override public GenericKeyedObjectPool createClientPool( ClientManager manager) { @@ -206,7 +238,7 @@ public GenericKeyedObjectPool cre new ThriftClientProperty.Builder() .setConnectionTimeoutMs(conf.getCnConnectionTimeoutInMS()) .setRpcThriftCompressionEnabled(conf.isRpcThriftCompressionEnabled()) - .setSelectorNumOfAsyncClientManager(conf.getSelectorNumOfClientManager()) + .setSelectorNumOfAsyncClientManager(selectorNumOfAsyncClientManager) .setPrintLogWhenEncounterException(false) .build(), ThreadName.ASYNC_DATANODE_HEARTBEAT_CLIENT_POOL.getName()), @@ -246,6 +278,13 @@ public static class SyncDataNodeMPPDataExchangeServiceClientPoolFactory public static class AsyncDataNodeMPPDataExchangeServiceClientPoolFactory implements IClientPoolFactory { + private final int selectorNumOfAsyncClientManager; + + public AsyncDataNodeMPPDataExchangeServiceClientPoolFactory( + int selectorNumOfAsyncClientManager) { + this.selectorNumOfAsyncClientManager = selectorNumOfAsyncClientManager; + } + @Override public GenericKeyedObjectPool createClientPool( @@ -257,7 +296,7 @@ public static class AsyncDataNodeMPPDataExchangeServiceClientPoolFactory new ThriftClientProperty.Builder() .setConnectionTimeoutMs(conf.getDnConnectionTimeoutInMS()) .setRpcThriftCompressionEnabled(conf.isRpcThriftCompressionEnabled()) - .setSelectorNumOfAsyncClientManager(conf.getSelectorNumOfClientManager()) + .setSelectorNumOfAsyncClientManager(selectorNumOfAsyncClientManager) .build(), ThreadName.ASYNC_DATANODE_MPP_DATA_EXCHANGE_CLIENT_POOL.getName()), new ClientPoolProperty.Builder() @@ -284,7 +323,7 @@ public GenericKeyedObjectPool cre .setRpcThriftCompressionEnabled(conf.isPipeSinkRPCThriftCompressionEnabled()) .setSelectorNumOfAsyncClientManager(conf.getPipeAsyncSinkSelectorNumber()) .build(), - ThreadName.PIPE_ASYNC_CONNECTOR_CLIENT_POOL.getName()), + ThreadName.PIPE_ASYNC_SINK_CLIENT_POOL.getName()), new ClientPoolProperty.Builder() .setMaxClientNumForEachNode(conf.getPipeAsyncSinkMaxClientNumber()) .build() @@ -310,7 +349,7 @@ public GenericKeyedObjectPool cre .setSelectorNumOfAsyncClientManager(conf.getPipeAsyncSinkSelectorNumber()) .setPrintLogWhenEncounterException(conf.isPrintLogWhenEncounterException()) .build(), - ThreadName.PIPE_ASYNC_CONNECTOR_CLIENT_POOL.getName()), + ThreadName.PIPE_ASYNC_SINK_CLIENT_POOL.getName()), new ClientPoolProperty.Builder() .setMaxClientNumForEachNode(conf.getPipeAsyncSinkMaxTsFileClientNumber()) .build() @@ -323,6 +362,13 @@ public GenericKeyedObjectPool cre public static class AsyncAINodeHeartbeatServiceClientPoolFactory implements IClientPoolFactory { + + private final int selectorNumOfAsyncClientManager; + + public AsyncAINodeHeartbeatServiceClientPoolFactory(int selectorNumOfAsyncClientManager) { + this.selectorNumOfAsyncClientManager = selectorNumOfAsyncClientManager; + } + @Override public GenericKeyedObjectPool createClientPool( ClientManager manager) { @@ -333,7 +379,7 @@ public GenericKeyedObjectPool createClientP new ThriftClientProperty.Builder() .setConnectionTimeoutMs(conf.getCnConnectionTimeoutInMS()) .setRpcThriftCompressionEnabled(conf.isRpcThriftCompressionEnabled()) - .setSelectorNumOfAsyncClientManager(conf.getSelectorNumOfClientManager()) + .setSelectorNumOfAsyncClientManager(selectorNumOfAsyncClientManager) .setPrintLogWhenEncounterException(false) .build(), ThreadName.ASYNC_DATANODE_HEARTBEAT_CLIENT_POOL.getName()), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/async/AsyncPipeDataTransferServiceClient.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/async/AsyncPipeDataTransferServiceClient.java index 41e8c59511a6f..2cdfbd865c817 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/async/AsyncPipeDataTransferServiceClient.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/async/AsyncPipeDataTransferServiceClient.java @@ -135,6 +135,10 @@ public void setShouldReturnSelf(final boolean shouldReturnSelf) { this.shouldReturnSelf.set(shouldReturnSelf); } + public boolean shouldReturnSelf() { + return shouldReturnSelf.get(); + } + public void setTimeoutDynamically(final int timeout) { try { ((TNonblockingSocket) ___transport).setTimeout(timeout); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/property/ClientPoolProperty.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/property/ClientPoolProperty.java index 5bb7c22ee4994..1f818afe543da 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/property/ClientPoolProperty.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/property/ClientPoolProperty.java @@ -19,6 +19,8 @@ package org.apache.iotdb.commons.client.property; +import org.apache.iotdb.commons.conf.CommonDescriptor; + import org.apache.commons.pool2.impl.GenericKeyedObjectPoolConfig; import java.time.Duration; @@ -49,7 +51,11 @@ public static class Builder { * the maximum number of clients that can be allocated for a node. When some clients are idle * for more than {@code maxIdleTimeForClient}, they will be cleaned up. */ - private int maxClientNumForEachNode = DefaultProperty.MAX_CLIENT_NUM_FOR_EACH_NODE; + private int maxClientNumForEachNode = + CommonDescriptor.getInstance().getConfig().getMaxClientNumForEachNode(); + + private int maxIdleClientNumForEachNode = + CommonDescriptor.getInstance().getConfig().getMaxIdleClientNumForEachNode(); /** * the minimum amount of time a client may sit idle in the pool before it is eligible for @@ -74,6 +80,11 @@ public Builder setMaxClientNumForEachNode(int maxClientNumForEachNode) { return this; } + public Builder setMaxIdleClientNumForEachNode(int maxIdleClientNumForEachNode) { + this.maxIdleClientNumForEachNode = maxIdleClientNumForEachNode; + return this; + } + public Builder setMinIdleTimeForClient(long minIdleTimeForClient) { this.minIdleTimeForClient = minIdleTimeForClient; return this; @@ -87,7 +98,7 @@ public Builder setTimeBetweenEvictionRuns(long timeBetweenEvictionRuns) { public ClientPoolProperty build() { GenericKeyedObjectPoolConfig poolConfig = new GenericKeyedObjectPoolConfig<>(); poolConfig.setMaxTotalPerKey(maxClientNumForEachNode); - poolConfig.setMaxIdlePerKey(maxClientNumForEachNode); + poolConfig.setMaxIdlePerKey(maxIdleClientNumForEachNode); poolConfig.setTimeBetweenEvictionRuns(Duration.ofMillis(timeBetweenEvictionRuns)); poolConfig.setMinEvictableIdleTime(Duration.ofMillis(minIdleTimeForClient)); poolConfig.setMaxWait(Duration.ofMillis(waitClientTimeoutMs)); @@ -105,5 +116,6 @@ private DefaultProperty() {} public static final long MIN_IDLE_TIME_FOR_CLIENT_MS = TimeUnit.MINUTES.toMillis(1); public static final long TIME_BETWEEN_EVICTION_RUNS_MS = TimeUnit.MINUTES.toMillis(1); public static final int MAX_CLIENT_NUM_FOR_EACH_NODE = 1000; + public static final int MAX_IDLE_CLIENT_NUM_FOR_EACH_NODE = 1000; } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/property/ThriftClientProperty.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/property/ThriftClientProperty.java index f8fe16166a8c8..f157ee7df936f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/property/ThriftClientProperty.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/property/ThriftClientProperty.java @@ -117,7 +117,10 @@ private DefaultProperty() {} public static final boolean RPC_THRIFT_COMPRESSED_ENABLED = false; public static final int CONNECTION_TIMEOUT_MS = (int) TimeUnit.SECONDS.toMillis(20); public static final int CONNECTION_NEVER_TIMEOUT_MS = 0; - public static final int SELECTOR_NUM_OF_ASYNC_CLIENT_MANAGER = 1; + public static final int SELECTOR_NUM_OF_ASYNC_CLIENT_MANAGER = + Runtime.getRuntime().availableProcessors() / 4 > 0 + ? Runtime.getRuntime().availableProcessors() / 4 + : 1; public static final boolean PRINT_LOG_WHEN_ENCOUNTER_EXCEPTION = true; } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestManager.java index 3cca39bb635da..0290d33d3a46b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestManager.java @@ -53,15 +53,15 @@ public abstract class AsyncRequestManager { private static final int MAX_RETRY_NUM = 6; - protected AsyncRequestManager() { - initClientManager(); + protected AsyncRequestManager(int selectorNumOfAsyncClientManager) { + initClientManager(selectorNumOfAsyncClientManager); actionMapBuilder = ImmutableMap.builder(); initActionMapBuilder(); this.actionMap = this.actionMapBuilder.build(); checkActionMapCompleteness(); } - protected abstract void initClientManager(); + protected abstract void initClientManager(int selectorNumOfAsyncClientManager); protected abstract void initActionMapBuilder(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/ConfigNodeInternalServiceAsyncRequestManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/ConfigNodeInternalServiceAsyncRequestManager.java index 791a1e5df0e96..3b50c29fba6bf 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/ConfigNodeInternalServiceAsyncRequestManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/ConfigNodeInternalServiceAsyncRequestManager.java @@ -28,12 +28,18 @@ public abstract class ConfigNodeInternalServiceAsyncRequestManager extends AsyncRequestManager< RequestType, TConfigNodeLocation, AsyncConfigNodeInternalServiceClient> { + + protected ConfigNodeInternalServiceAsyncRequestManager(int selectorNumOfAsyncClientManager) { + super(selectorNumOfAsyncClientManager); + } + @Override - protected void initClientManager() { + protected void initClientManager(int selectorNumOfAsyncClientManager) { clientManager = new IClientManager.Factory() .createClientManager( - new ClientPoolFactory.AsyncConfigNodeInternalServiceClientPoolFactory()); + new ClientPoolFactory.AsyncConfigNodeInternalServiceClientPoolFactory( + selectorNumOfAsyncClientManager)); } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/DataNodeInternalServiceRequestManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/DataNodeInternalServiceRequestManager.java index fcb1b01857df8..722d4f241ebe1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/DataNodeInternalServiceRequestManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/DataNodeInternalServiceRequestManager.java @@ -28,12 +28,18 @@ public abstract class DataNodeInternalServiceRequestManager extends AsyncRequestManager< RequestType, TDataNodeLocation, AsyncDataNodeInternalServiceClient> { + + protected DataNodeInternalServiceRequestManager(int selectorNumOfAsyncClientManager) { + super(selectorNumOfAsyncClientManager); + } + @Override - protected void initClientManager() { + protected void initClientManager(int selectorNumOfAsyncClientManager) { clientManager = new IClientManager.Factory() .createClientManager( - new ClientPoolFactory.AsyncDataNodeInternalServiceClientPoolFactory()); + new ClientPoolFactory.AsyncDataNodeInternalServiceClientPoolFactory( + selectorNumOfAsyncClientManager)); } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java index 696b6b8ce07fb..a902893094056 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java @@ -44,7 +44,6 @@ public enum ThreadName { MPP_COORDINATOR_EXECUTOR_POOL("MPP-Coordinator-Executor"), DATANODE_INTERNAL_RPC_SERVICE("DataNodeInternalRPC-Service"), DATANODE_INTERNAL_RPC_PROCESSOR("DataNodeInternalRPC-Processor"), - MPP_COORDINATOR_WRITE_EXECUTOR("MPP-Coordinator-Write-Executor"), ASYNC_DATANODE_MPP_DATA_EXCHANGE_CLIENT_POOL("AsyncDataNodeMPPDataExchangeServiceClientPool"), // -------------------------- Compaction -------------------------- COMPACTION_WORKER("Compaction-Worker"), @@ -129,10 +128,10 @@ public enum ThreadName { GPRC_DEFAULT_WORKER_ELG("grpc-default-worker-ELG"), GROUP_MANAGEMENT("groupManagement"), // -------------------------- Compute -------------------------- - PIPE_EXTRACTOR_DISRUPTOR("Pipe-Extractor-Disruptor"), + PIPE_SOURCE_DISRUPTOR("Pipe-Source-Disruptor"), PIPE_PROCESSOR_EXECUTOR_POOL("Pipe-Processor-Executor-Pool"), - PIPE_CONNECTOR_EXECUTOR_POOL("Pipe-Connector-Executor-Pool"), PIPE_CONSENSUS_EXECUTOR_POOL("Pipe-Consensus-Executor-Pool"), + PIPE_SINK_EXECUTOR_POOL("Pipe-Sink-Executor-Pool"), PIPE_CONFIGNODE_EXECUTOR_POOL("Pipe-ConfigNode-Executor-Pool"), PIPE_SUBTASK_CALLBACK_EXECUTOR_POOL("Pipe-SubTask-Callback-Executor-Pool"), PIPE_TSFILE_ASYNC_SEND_POOL("Pipe-TsFile-Async-Send-Pool"), @@ -142,7 +141,7 @@ public enum ThreadName { PIPE_RUNTIME_PERIODICAL_JOB_EXECUTOR("Pipe-Runtime-Periodical-Job-Executor"), PIPE_RUNTIME_PERIODICAL_PHANTOM_REFERENCE_CLEANER( "Pipe-Runtime-Periodical-Phantom-Reference-Cleaner"), - PIPE_ASYNC_CONNECTOR_CLIENT_POOL("Pipe-Async-Connector-Client-Pool"), + PIPE_ASYNC_SINK_CLIENT_POOL("Pipe-Async-Sink-Client-Pool"), PIPE_RECEIVER_AIR_GAP_AGENT("Pipe-Receiver-Air-Gap-Agent"), PIPE_AIR_GAP_RECEIVER("Pipe-Air-Gap-Receiver"), PIPE_PARALLEL_EXECUTION_POOL("Pipe-Parallel-Execution-Pool"), @@ -221,7 +220,6 @@ public enum ThreadName { MPP_COORDINATOR_EXECUTOR_POOL, DATANODE_INTERNAL_RPC_SERVICE, DATANODE_INTERNAL_RPC_PROCESSOR, - MPP_COORDINATOR_WRITE_EXECUTOR, ASYNC_DATANODE_MPP_DATA_EXCHANGE_CLIENT_POOL)); private static final Set compactionThreadNames = new HashSet<>(Arrays.asList(COMPACTION_WORKER, COMPACTION_SUB_TASK, COMPACTION_SCHEDULE)); @@ -289,9 +287,9 @@ public enum ThreadName { private static final Set computeThreadNames = new HashSet<>( Arrays.asList( - PIPE_EXTRACTOR_DISRUPTOR, + PIPE_SOURCE_DISRUPTOR, PIPE_PROCESSOR_EXECUTOR_POOL, - PIPE_CONNECTOR_EXECUTOR_POOL, + PIPE_SINK_EXECUTOR_POOL, PIPE_CONSENSUS_EXECUTOR_POOL, PIPE_CONFIGNODE_EXECUTOR_POOL, PIPE_SUBTASK_CALLBACK_EXECUTOR_POOL, @@ -300,7 +298,7 @@ public enum ThreadName { PIPE_RUNTIME_PROCEDURE_SUBMITTER, PIPE_RUNTIME_PERIODICAL_JOB_EXECUTOR, PIPE_RUNTIME_PERIODICAL_PHANTOM_REFERENCE_CLEANER, - PIPE_ASYNC_CONNECTOR_CLIENT_POOL, + PIPE_ASYNC_SINK_CLIENT_POOL, PIPE_RECEIVER_AIR_GAP_AGENT, PIPE_AIR_GAP_RECEIVER, PIPE_PARALLEL_EXECUTION_POOL, diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index ec9ce06fd1d41..37929ed18603f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -146,13 +146,18 @@ public class CommonConfig { * ClientManager will have so many selector threads (TAsyncClientManager) to distribute to its * clients. */ - private int selectorNumOfClientManager = 1; + private int selectorNumOfClientManager = + Runtime.getRuntime().availableProcessors() / 4 > 0 + ? Runtime.getRuntime().availableProcessors() / 4 + : 1; /** Whether to use thrift compression. */ private boolean isRpcThriftCompressionEnabled = false; private int maxClientNumForEachNode = DefaultProperty.MAX_CLIENT_NUM_FOR_EACH_NODE; + private int maxIdleClientNumForEachNode = DefaultProperty.MAX_IDLE_CLIENT_NUM_FOR_EACH_NODE; + /** What will the system do when unrecoverable error occurs. */ private HandleSystemErrorStrategy handleSystemErrorStrategy = HandleSystemErrorStrategy.CHANGE_TO_READ_ONLY; @@ -215,8 +220,8 @@ public class CommonConfig { private int pipeDataStructureTabletRowSize = 2048; - // 128MB - private int pipeDataStructureTabletSizeInBytes = 60 * 1024 * 1024; + // 16MB + private int pipeDataStructureTabletSizeInBytes = 16 * 1024 * 1024; private double pipeDataStructureTabletMemoryBlockAllocationRejectThreshold = 0.3; private double pipeDataStructureTsFileMemoryBlockAllocationRejectThreshold = 0.3; private volatile double pipeTotalFloatingMemoryProportion = 0.5; @@ -277,11 +282,11 @@ public class CommonConfig { private int pipeAsyncSinkForcedRetryTabletEventQueueSize = 20; private int pipeAsyncSinkForcedRetryTotalEventQueueSize = 30; private long pipeAsyncSinkMaxRetryExecutionTimeMsPerCall = 500; - private int pipeAsyncConnectorSelectorNumber = + private int pipeAsyncSinkSelectorNumber = Math.max(4, Runtime.getRuntime().availableProcessors() / 2); - private int pipeAsyncConnectorMaxClientNumber = + private int pipeAsyncSinkMaxClientNumber = Math.max(32, Runtime.getRuntime().availableProcessors() * 2); - private int pipeAsyncConnectorMaxTsFileClientNumber = + private int pipeAsyncSinkMaxTsFileClientNumber = Math.max(16, Runtime.getRuntime().availableProcessors()); private boolean printLogWhenEncounterException = false; @@ -289,8 +294,7 @@ public class CommonConfig { private double pipeAllSinksRateLimitBytesPerSecond = -1; private int rateLimiterHotReloadCheckIntervalMs = 1000; - private int pipeConnectorRequestSliceThresholdBytes = - (int) (RpcUtils.THRIFT_FRAME_MAX_SIZE * 0.8); + private int pipeSinkRequestSliceThresholdBytes = (int) (RpcUtils.THRIFT_FRAME_MAX_SIZE * 0.8); private boolean isSeperatedPipeHeartbeatEnabled = true; private int pipeHeartbeatIntervalSecondsForCollectingPipeMeta = 3; @@ -630,6 +634,14 @@ public void setMaxClientNumForEachNode(int maxClientNumForEachNode) { this.maxClientNumForEachNode = maxClientNumForEachNode; } + public int getMaxIdleClientNumForEachNode() { + return maxIdleClientNumForEachNode; + } + + public void setMaxIdleClientNumForEachNode(int maxIdleClientNumForEachNode) { + this.maxIdleClientNumForEachNode = maxIdleClientNumForEachNode; + } + HandleSystemErrorStrategy getHandleSystemErrorStrategy() { return handleSystemErrorStrategy; } @@ -1002,7 +1014,7 @@ public int getPipeSinkHandshakeTimeoutMs() { } public void setPipeSinkHandshakeTimeoutMs(long pipeSinkHandshakeTimeoutMs) { - final int fPipeConnectorHandshakeTimeoutMs = this.pipeSinkHandshakeTimeoutMs; + final int fPipeSinkHandshakeTimeoutMs = this.pipeSinkHandshakeTimeoutMs; try { this.pipeSinkHandshakeTimeoutMs = Math.toIntExact(pipeSinkHandshakeTimeoutMs); } catch (ArithmeticException e) { @@ -1010,7 +1022,7 @@ public void setPipeSinkHandshakeTimeoutMs(long pipeSinkHandshakeTimeoutMs) { logger.warn( "Given pipe connector handshake timeout is too large, set to {} ms.", Integer.MAX_VALUE); } finally { - if (fPipeConnectorHandshakeTimeoutMs != this.pipeSinkHandshakeTimeoutMs) { + if (fPipeSinkHandshakeTimeoutMs != this.pipeSinkHandshakeTimeoutMs) { logger.info("pipeSinkHandshakeTimeoutMs is set to {}.", this.pipeSinkHandshakeTimeoutMs); } } @@ -1041,16 +1053,16 @@ public int getPipeSinkTransferTimeoutMs() { } public void setPipeSinkTransferTimeoutMs(long pipeSinkTransferTimeoutMs) { - final int fPipeConnectorTransferTimeoutMs = this.pipeSinkTransferTimeoutMs; + final int fPipeSinkTransferTimeoutMs = this.pipeSinkTransferTimeoutMs; try { this.pipeSinkTransferTimeoutMs = Math.toIntExact(pipeSinkTransferTimeoutMs); } catch (ArithmeticException e) { this.pipeSinkTransferTimeoutMs = Integer.MAX_VALUE; logger.warn( - "Given pipe connector transfer timeout is too large, set to {} ms.", Integer.MAX_VALUE); + "Given pipe sink transfer timeout is too large, set to {} ms.", Integer.MAX_VALUE); } finally { - if (fPipeConnectorTransferTimeoutMs != this.pipeSinkTransferTimeoutMs) { - logger.info("pipeConnectorTransferTimeoutMs is set to {}.", pipeSinkTransferTimeoutMs); + if (fPipeSinkTransferTimeoutMs != this.pipeSinkTransferTimeoutMs) { + logger.info("pipeSinkTransferTimeoutMs is set to {}.", pipeSinkTransferTimeoutMs); } } } @@ -1064,7 +1076,7 @@ public void setPipeSinkReadFileBufferSize(int pipeSinkReadFileBufferSize) { return; } this.pipeSinkReadFileBufferSize = pipeSinkReadFileBufferSize; - logger.info("pipeConnectorReadFileBufferSize is set to {}.", pipeSinkReadFileBufferSize); + logger.info("pipeSinkReadFileBufferSize is set to {}.", pipeSinkReadFileBufferSize); } public boolean isPipeSinkReadFileBufferMemoryControlEnabled() { @@ -1164,60 +1176,58 @@ public long getPipeAsyncSinkMaxRetryExecutionTimeMsPerCall() { } public int getPipeAsyncSinkSelectorNumber() { - return pipeAsyncConnectorSelectorNumber; + return pipeAsyncSinkSelectorNumber; } - public void setPipeAsyncConnectorSelectorNumber(int pipeAsyncConnectorSelectorNumber) { - if (pipeAsyncConnectorSelectorNumber <= 0) { + public void setPipeAsyncSinkSelectorNumber(int pipeAsyncSinkSelectorNumber) { + if (pipeAsyncSinkSelectorNumber <= 0) { logger.info( - "pipeAsyncConnectorSelectorNumber should be greater than 0, configuring it not to change."); + "pipeAsyncSinkSelectorNumber should be greater than 0, configuring it not to change."); return; } - pipeAsyncConnectorSelectorNumber = Math.max(4, pipeAsyncConnectorSelectorNumber); - if (this.pipeAsyncConnectorSelectorNumber == pipeAsyncConnectorSelectorNumber) { + pipeAsyncSinkSelectorNumber = Math.max(4, pipeAsyncSinkSelectorNumber); + if (this.pipeAsyncSinkSelectorNumber == pipeAsyncSinkSelectorNumber) { return; } - this.pipeAsyncConnectorSelectorNumber = pipeAsyncConnectorSelectorNumber; - logger.info("pipeAsyncConnectorSelectorNumber is set to {}.", pipeAsyncConnectorSelectorNumber); + this.pipeAsyncSinkSelectorNumber = pipeAsyncSinkSelectorNumber; + logger.info("pipeAsyncSinkSelectorNumber is set to {}.", pipeAsyncSinkSelectorNumber); } public int getPipeAsyncSinkMaxClientNumber() { - return pipeAsyncConnectorMaxClientNumber; + return pipeAsyncSinkMaxClientNumber; } - public void setPipeAsyncConnectorMaxClientNumber(int pipeAsyncConnectorMaxClientNumber) { - if (pipeAsyncConnectorMaxClientNumber <= 0) { + public void setPipeAsyncSinkMaxClientNumber(int pipeAsyncSinkMaxClientNumber) { + if (pipeAsyncSinkMaxClientNumber <= 0) { logger.info( - " pipeAsyncConnectorMaxClientNumber should be greater than 0, configuring it not to change."); + " pipeAsyncSinkMaxClientNumber should be greater than 0, configuring it not to change."); return; } - pipeAsyncConnectorMaxClientNumber = Math.max(32, pipeAsyncConnectorMaxClientNumber); - if (this.pipeAsyncConnectorMaxClientNumber == pipeAsyncConnectorMaxClientNumber) { + pipeAsyncSinkMaxClientNumber = Math.max(32, pipeAsyncSinkMaxClientNumber); + if (this.pipeAsyncSinkMaxClientNumber == pipeAsyncSinkMaxClientNumber) { return; } - this.pipeAsyncConnectorMaxClientNumber = pipeAsyncConnectorMaxClientNumber; - logger.info( - "pipeAsyncConnectorMaxClientNumber is set to {}.", pipeAsyncConnectorMaxClientNumber); + this.pipeAsyncSinkMaxClientNumber = pipeAsyncSinkMaxClientNumber; + logger.info("pipeAsyncSinkMaxClientNumber is set to {}.", pipeAsyncSinkMaxClientNumber); } public int getPipeAsyncSinkMaxTsFileClientNumber() { - return pipeAsyncConnectorMaxTsFileClientNumber; + return pipeAsyncSinkMaxTsFileClientNumber; } - public void setPipeAsyncConnectorMaxTsFileClientNumber( - int pipeAsyncConnectorMaxTsFileClientNumber) { - if (pipeAsyncConnectorMaxTsFileClientNumber <= 0) { + public void setPipeAsyncSinkMaxTsFileClientNumber(int pipeAsyncSinkMaxTsFileClientNumber) { + if (pipeAsyncSinkMaxTsFileClientNumber <= 0) { logger.info( - "pipeAsyncConnectorMaxTsFileClientNumber should be greater than 0, configuring it not to change."); + "pipeAsyncSinkMaxTsFileClientNumber should be greater than 0, configuring it not to change."); return; } - pipeAsyncConnectorMaxTsFileClientNumber = Math.max(16, pipeAsyncConnectorMaxTsFileClientNumber); - if (this.pipeAsyncConnectorMaxTsFileClientNumber == pipeAsyncConnectorMaxTsFileClientNumber) { + pipeAsyncSinkMaxTsFileClientNumber = Math.max(16, pipeAsyncSinkMaxTsFileClientNumber); + if (this.pipeAsyncSinkMaxTsFileClientNumber == pipeAsyncSinkMaxTsFileClientNumber) { return; } - this.pipeAsyncConnectorMaxTsFileClientNumber = pipeAsyncConnectorMaxTsFileClientNumber; + this.pipeAsyncSinkMaxTsFileClientNumber = pipeAsyncSinkMaxTsFileClientNumber; logger.info( - "pipeAsyncConnectorMaxClientNumber is set to {}.", pipeAsyncConnectorMaxTsFileClientNumber); + "pipeAsyncSinkMaxTsFileClientNumber is set to {}.", pipeAsyncSinkMaxTsFileClientNumber); } public boolean isPrintLogWhenEncounterException() { @@ -1321,12 +1331,12 @@ public long getPipeSinkRetryIntervalMs() { return pipeSinkRetryIntervalMs; } - public void setPipeSinkRetryIntervalMs(long pipeConnectorRetryIntervalMs) { - if (this.pipeSinkRetryIntervalMs == pipeConnectorRetryIntervalMs) { + public void setPipeSinkRetryIntervalMs(long pipeSinkRetryIntervalMs) { + if (this.pipeSinkRetryIntervalMs == pipeSinkRetryIntervalMs) { return; } - this.pipeSinkRetryIntervalMs = pipeConnectorRetryIntervalMs; - logger.info("pipeSinkRetryIntervalMs is set to {}", pipeConnectorRetryIntervalMs); + this.pipeSinkRetryIntervalMs = pipeSinkRetryIntervalMs; + logger.info("pipeSinkRetryIntervalMs is set to {}", pipeSinkRetryIntervalMs); } public boolean isPipeSinkRetryLocallyForConnectionError() { @@ -2127,18 +2137,16 @@ public void setRateLimiterHotReloadCheckIntervalMs(int rateLimiterHotReloadCheck } public int getPipeSinkRequestSliceThresholdBytes() { - return pipeConnectorRequestSliceThresholdBytes; + return pipeSinkRequestSliceThresholdBytes; } - public void setPipeConnectorRequestSliceThresholdBytes( - int pipeConnectorRequestSliceThresholdBytes) { - if (this.pipeConnectorRequestSliceThresholdBytes == pipeConnectorRequestSliceThresholdBytes) { + public void setPipeSinkRequestSliceThresholdBytes(int pipeSinkRequestSliceThresholdBytes) { + if (this.pipeSinkRequestSliceThresholdBytes == pipeSinkRequestSliceThresholdBytes) { return; } - this.pipeConnectorRequestSliceThresholdBytes = pipeConnectorRequestSliceThresholdBytes; + this.pipeSinkRequestSliceThresholdBytes = pipeSinkRequestSliceThresholdBytes; logger.info( - "pipeConnectorRequestSliceThresholdBytes is set to {}", - pipeConnectorRequestSliceThresholdBytes); + "pipeConnectorRequestSliceThresholdBytes is set to {}", pipeSinkRequestSliceThresholdBytes); } public long getTwoStageAggregateMaxCombinerLiveTimeInMs() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java index 961d4fbb8f248..ab7d949044d0b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java @@ -128,13 +128,16 @@ public void loadCommonProps(TrimProperties properties) throws IOException { "cn_connection_timeout_ms", String.valueOf(config.getCnConnectionTimeoutInMS())) .trim())); - config.setSelectorNumOfClientManager( + int cnSelectorNumOfClientManager = Integer.parseInt( properties .getProperty( "cn_selector_thread_nums_of_client_manager", String.valueOf(config.getSelectorNumOfClientManager())) - .trim())); + .trim()); + if (cnSelectorNumOfClientManager > 0) { + config.setSelectorNumOfClientManager(cnSelectorNumOfClientManager); + } config.setMaxClientNumForEachNode( Integer.parseInt( @@ -144,6 +147,17 @@ public void loadCommonProps(TrimProperties properties) throws IOException { String.valueOf(config.getMaxClientNumForEachNode())) .trim())); + int cnMaxIdleClientNumForEachNode = + Integer.parseInt( + properties + .getProperty( + "cn_max_idle_client_count_for_each_node_in_client_manager", + String.valueOf(config.getMaxIdleClientNumForEachNode())) + .trim()); + if (cnMaxIdleClientNumForEachNode >= 0) { + config.setMaxIdleClientNumForEachNode(cnMaxIdleClientNumForEachNode); + } + config.setDnConnectionTimeoutInMS( Integer.parseInt( properties @@ -159,13 +173,16 @@ public void loadCommonProps(TrimProperties properties) throws IOException { String.valueOf(config.isRpcThriftCompressionEnabled())) .trim())); - config.setSelectorNumOfClientManager( + int dnSelectorNumOfClientManager = Integer.parseInt( properties .getProperty( "dn_selector_thread_nums_of_client_manager", String.valueOf(config.getSelectorNumOfClientManager())) - .trim())); + .trim()); + if (dnSelectorNumOfClientManager > 0) { + config.setSelectorNumOfClientManager(dnSelectorNumOfClientManager); + } config.setMaxClientNumForEachNode( Integer.parseInt( @@ -175,6 +192,17 @@ public void loadCommonProps(TrimProperties properties) throws IOException { String.valueOf(config.getMaxClientNumForEachNode())) .trim())); + int dnMaxIdleClientNumForEachNode = + Integer.parseInt( + properties + .getProperty( + "dn_max_idle_client_count_for_each_node_in_client_manager", + String.valueOf(config.getMaxIdleClientNumForEachNode())) + .trim()); + if (dnMaxIdleClientNumForEachNode >= 0) { + config.setMaxIdleClientNumForEachNode(dnMaxIdleClientNumForEachNode); + } + config.setHandleSystemErrorStrategy( HandleSystemErrorStrategy.valueOf( properties diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java index 18ac41409ba9d..f9f69f486e6ce 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java @@ -268,7 +268,6 @@ private IoTDBConstant() {} public static final String MQTT_MAX_MESSAGE_SIZE = "mqtt_max_message_size"; // thrift - public static final int LEFT_SIZE_IN_REQUEST = 4 * 1024 * 1024; public static final int DEFAULT_FETCH_SIZE = 5000; public static final int DEFAULT_CONNECTION_TIMEOUT_MS = 0; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ReferenceCountedJarMetaKeeper.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ReferenceCountedJarMetaKeeper.java new file mode 100644 index 0000000000000..52102aa96f2b3 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/executable/ReferenceCountedJarMetaKeeper.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.executable; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public class ReferenceCountedJarMetaKeeper { + + private final Map jarNameToMd5Map = new HashMap<>(); + private final Map jarNameToReferenceCountMap = new HashMap<>(); + + public synchronized boolean containsJar(final String jarName) { + return jarNameToMd5Map.containsKey(jarName); + } + + public synchronized boolean needToSaveJar(final String jarName) { + return !containsJar(jarName); + } + + public synchronized boolean jarNameExistsAndMatchesMd5(final String jarName, final String md5) { + return containsJar(jarName) && Objects.equals(jarNameToMd5Map.get(jarName), md5); + } + + public synchronized void addReference(final String jarName, final String md5) { + if (jarNameToReferenceCountMap.containsKey(jarName)) { + jarNameToReferenceCountMap.put(jarName, jarNameToReferenceCountMap.get(jarName) + 1); + return; + } + + jarNameToReferenceCountMap.put(jarName, 1); + jarNameToMd5Map.put(jarName, md5); + } + + public synchronized void removeReference(final String jarName) { + final Integer referenceCount = jarNameToReferenceCountMap.get(jarName); + if (referenceCount == null || referenceCount <= 1) { + jarNameToReferenceCountMap.remove(jarName); + jarNameToMd5Map.remove(jarName); + return; + } + + jarNameToReferenceCountMap.put(jarName, referenceCount - 1); + } + + public synchronized void clear() { + jarNameToMd5Map.clear(); + jarNameToReferenceCountMap.clear(); + } + + public synchronized Map getJarNameToMd5Map() { + return new HashMap<>(jarNameToMd5Map); + } + + public synchronized void serializeJarNameToMd5(final OutputStream outputStream) + throws IOException { + ReadWriteIOUtils.write(jarNameToMd5Map.size(), outputStream); + for (final Map.Entry entry : jarNameToMd5Map.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + ReadWriteIOUtils.write(entry.getValue(), outputStream); + } + } + + public synchronized void deserializeJarNameToMd5(final InputStream inputStream) + throws IOException { + clear(); + + int size = ReadWriteIOUtils.readInt(inputStream); + while (size > 0) { + addReference( + ReadWriteIOUtils.readString(inputStream), ReadWriteIOUtils.readString(inputStream)); + size--; + } + } + + public synchronized void serializeJarNameToMd5AndReferenceCount(final OutputStream outputStream) + throws IOException { + int size = 0; + for (final Map.Entry entry : jarNameToReferenceCountMap.entrySet()) { + if (entry.getValue() > 0 && jarNameToMd5Map.containsKey(entry.getKey())) { + size++; + } + } + + ReadWriteIOUtils.write(size, outputStream); + for (final Map.Entry entry : jarNameToReferenceCountMap.entrySet()) { + final String jarName = entry.getKey(); + final int referenceCount = entry.getValue(); + if (referenceCount <= 0 || !jarNameToMd5Map.containsKey(jarName)) { + continue; + } + ReadWriteIOUtils.write(jarName, outputStream); + ReadWriteIOUtils.write(jarNameToMd5Map.get(jarName), outputStream); + ReadWriteIOUtils.write(referenceCount, outputStream); + } + } + + public synchronized void deserializeJarNameToMd5AndReferenceCount(final InputStream inputStream) + throws IOException { + clear(); + + final int jarSize = ReadWriteIOUtils.readInt(inputStream); + for (int i = 0; i < jarSize; i++) { + final String jarName = ReadWriteIOUtils.readString(inputStream); + final String md5 = ReadWriteIOUtils.readString(inputStream); + final int referenceCount = ReadWriteIOUtils.readInt(inputStream); + if (referenceCount > 0) { + jarNameToMd5Map.put(jarName, md5); + jarNameToReferenceCountMap.put(jarName, referenceCount); + } + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/meta/ConfigNodePipePluginMetaKeeper.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/meta/ConfigNodePipePluginMetaKeeper.java index da4eb9cf714da..2db7cf4613585 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/meta/ConfigNodePipePluginMetaKeeper.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/meta/ConfigNodePipePluginMetaKeeper.java @@ -19,80 +19,48 @@ package org.apache.iotdb.commons.pipe.agent.plugin.meta; -import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.apache.iotdb.commons.executable.ReferenceCountedJarMetaKeeper; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.HashMap; -import java.util.Map; public class ConfigNodePipePluginMetaKeeper extends PipePluginMetaKeeper { - protected final Map jarNameToMd5Map; - protected final Map jarNameToReferenceCountMap; + protected final ReferenceCountedJarMetaKeeper jarMetaKeeper; public ConfigNodePipePluginMetaKeeper() { super(); - jarNameToMd5Map = new HashMap<>(); - jarNameToReferenceCountMap = new HashMap<>(); + jarMetaKeeper = new ReferenceCountedJarMetaKeeper(); } public synchronized boolean containsJar(String jarName) { - return jarNameToMd5Map.containsKey(jarName); + return jarMetaKeeper.containsJar(jarName); } public synchronized boolean jarNameExistsAndMatchesMd5(String jarName, String md5) { - return jarNameToMd5Map.containsKey(jarName) && jarNameToMd5Map.get(jarName).equals(md5); + return jarMetaKeeper.jarNameExistsAndMatchesMd5(jarName, md5); } public synchronized void addJarNameAndMd5(String jarName, String md5) { - if (jarNameToReferenceCountMap.containsKey(jarName)) { - jarNameToReferenceCountMap.put(jarName, jarNameToReferenceCountMap.get(jarName) + 1); - } else { - jarNameToReferenceCountMap.put(jarName, 1); - jarNameToMd5Map.put(jarName, md5); - } + jarMetaKeeper.addReference(jarName, md5); } public synchronized void removeJarNameAndMd5IfPossible(String jarName) { - if (jarNameToReferenceCountMap.containsKey(jarName)) { - int count = jarNameToReferenceCountMap.get(jarName); - if (count == 1) { - jarNameToReferenceCountMap.remove(jarName); - jarNameToMd5Map.remove(jarName); - } else { - jarNameToReferenceCountMap.put(jarName, count - 1); - } - } + jarMetaKeeper.removeReference(jarName); } @Override public void processTakeSnapshot(OutputStream outputStream) throws IOException { - ReadWriteIOUtils.write(jarNameToMd5Map.size(), outputStream); - for (Map.Entry entry : jarNameToMd5Map.entrySet()) { - ReadWriteIOUtils.write(entry.getKey(), outputStream); - ReadWriteIOUtils.write(entry.getValue(), outputStream); - ReadWriteIOUtils.write(jarNameToReferenceCountMap.get(entry.getKey()), outputStream); - } + jarMetaKeeper.serializeJarNameToMd5AndReferenceCount(outputStream); super.processTakeSnapshot(outputStream); } @Override public void processLoadSnapshot(InputStream inputStream) throws IOException { - jarNameToMd5Map.clear(); - jarNameToReferenceCountMap.clear(); - - final int jarSize = ReadWriteIOUtils.readInt(inputStream); - for (int i = 0; i < jarSize; i++) { - final String jarName = ReadWriteIOUtils.readString(inputStream); - final String md5 = ReadWriteIOUtils.readString(inputStream); - final int count = ReadWriteIOUtils.readInt(inputStream); - jarNameToMd5Map.put(jarName, md5); - jarNameToReferenceCountMap.put(jarName, count); - } + jarMetaKeeper.deserializeJarNameToMd5AndReferenceCount(inputStream); super.processLoadSnapshot(inputStream); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/service/PipePluginClassLoader.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/service/PipePluginClassLoader.java index a2bc1df1a2797..745d93566fc40 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/service/PipePluginClassLoader.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/service/PipePluginClassLoader.java @@ -35,6 +35,10 @@ @ThreadSafe public class PipePluginClassLoader extends URLClassLoader { + private static final String[] PARENT_FIRST_CLASS_PREFIXES = { + "java.", "javax.", "jdk.", "sun.", "org.slf4j.", "org.apache.iotdb.pipe.api." + }; + private final String libRoot; /** @@ -50,7 +54,11 @@ public class PipePluginClassLoader extends URLClassLoader { private volatile boolean deprecated; public PipePluginClassLoader(String libRoot) throws IOException { - super(new URL[0]); + this(libRoot, ClassLoader.getSystemClassLoader()); + } + + PipePluginClassLoader(String libRoot, ClassLoader parent) throws IOException { + super(new URL[0], parent); this.libRoot = libRoot; activeInstanceCount = new AtomicLong(0); deprecated = false; @@ -82,6 +90,38 @@ public synchronized void markAsDeprecated() throws IOException { closeIfPossible(); } + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + synchronized (getClassLoadingLock(name)) { + Class loadedClass = findLoadedClass(name); + if (loadedClass == null) { + loadedClass = + shouldLoadFromParentFirst(name) ? super.loadClass(name, false) : loadClassLocally(name); + } + if (resolve) { + resolveClass(loadedClass); + } + return loadedClass; + } + } + + private Class loadClassLocally(String name) throws ClassNotFoundException { + try { + return findClass(name); + } catch (ClassNotFoundException e) { + return super.loadClass(name, false); + } + } + + private boolean shouldLoadFromParentFirst(String name) { + for (String prefix : PARENT_FIRST_CLASS_PREFIXES) { + if (name.startsWith(prefix)) { + return true; + } + } + return false; + } + private void closeIfPossible() throws IOException { if (deprecated && activeInstanceCount.get() == 0) { close(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java index 726bc1b6a1f93..fcff317b3f337 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/PipeTaskAgent.java @@ -125,6 +125,10 @@ protected boolean tryReadLockWithTimeOut(final long timeOutInSeconds) { } } + protected boolean tryReadLockWithTimeOutInMs(final long timeOutInMs) { + return tryReadLockWithTimeOut(convertMsToCeilSeconds(timeOutInMs)); + } + protected void releaseReadLock() { pipeMetaKeeper.releaseReadLock(); } @@ -143,10 +147,18 @@ protected boolean tryWriteLockWithTimeOut(final long timeOutInSeconds) { } } + protected boolean tryWriteLockWithTimeOutInMs(final long timeOutInMs) { + return tryWriteLockWithTimeOut(convertMsToCeilSeconds(timeOutInMs)); + } + protected void releaseWriteLock() { pipeMetaKeeper.releaseWriteLock(); } + private long convertMsToCeilSeconds(final long timeOutInMs) { + return Math.max(1L, (Math.max(0L, timeOutInMs) + 999L) / 1000L); + } + ////////////////////////// Pipe Task Management Entry ////////////////////////// public TPushPipeMetaRespExceptionMessage handleSinglePipeMetaChanges( @@ -363,7 +375,7 @@ protected TPushPipeMetaRespExceptionMessage handleDropPipeInternal(final String public List handlePipeMetaChanges( final List pipeMetaListFromCoordinator) { - if (!tryWriteLockWithTimeOut( + if (!tryWriteLockWithTimeOutInMs( CommonDescriptor.getInstance().getConfig().getDnConnectionTimeoutInMS() * 2L / 3)) { return null; } @@ -1091,7 +1103,7 @@ private void stopAllPipesWithCriticalExceptionInternal(final int currentNodeId) public void collectPipeMetaList(final TPipeHeartbeatReq req, final TPipeHeartbeatResp resp) throws TException { - if (!tryReadLockWithTimeOut( + if (!tryReadLockWithTimeOutInMs( CommonDescriptor.getInstance().getConfig().getDnConnectionTimeoutInMS() * 2L / 3)) { return; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java index 8773b03f9f308..c7b91f36d222b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java @@ -19,6 +19,7 @@ package org.apache.iotdb.commons.pipe.agent.task.connection; +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; import org.apache.iotdb.commons.pipe.config.PipeConfig; import org.apache.iotdb.commons.pipe.event.EnrichedEvent; import org.apache.iotdb.commons.pipe.metric.PipeEventCounter; @@ -27,7 +28,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Set; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; @@ -44,6 +47,8 @@ public abstract class BlockingPendingQueue { protected final AtomicBoolean isClosed = new AtomicBoolean(false); + protected final Set droppedPipeTaskKeys = ConcurrentHashMap.newKeySet(); + protected BlockingPendingQueue( final BlockingQueue pendingQueue, final PipeEventCounter eventCounter) { this.pendingQueue = pendingQueue; @@ -51,7 +56,10 @@ protected BlockingPendingQueue( } public boolean offer(final E event) { - checkBeforeOffer(event); + if (!checkBeforeOffer(event)) { + return false; + } + final boolean offered = pendingQueue.offer(event); if (offered) { eventCounter.increaseEventCount(event); @@ -60,7 +68,9 @@ public boolean offer(final E event) { } public boolean put(final E event) { - checkBeforeOffer(event); + if (!checkBeforeOffer(event)) { + return false; + } try { pendingQueue.put(event); eventCounter.increaseEventCount(event); @@ -101,6 +111,7 @@ public void clear() { isClosed.set(true); pendingQueue.clear(); eventCounter.reset(); + droppedPipeTaskKeys.clear(); } /** DO NOT FORGET to set eventCounter to new value after invoking this method. */ @@ -120,14 +131,20 @@ public void discardAllEvents() { return true; }); eventCounter.reset(); + droppedPipeTaskKeys.clear(); + } + + public void discardEventsOfPipe( + final String pipeNameToDrop, final long creationTimeToDrop, final int regionId) { + discardEventsOfPipe(new CommitterKey(pipeNameToDrop, creationTimeToDrop, regionId, -1)); } - public void discardEventsOfPipe(final String pipeNameToDrop, final int regionId) { + public void discardEventsOfPipe(final CommitterKey committerKey) { + droppedPipeTaskKeys.add(committerKey); pendingQueue.removeIf( event -> { if (event instanceof EnrichedEvent - && pipeNameToDrop.equals(((EnrichedEvent) event).getPipeName()) - && regionId == ((EnrichedEvent) event).getRegionId()) { + && isEventFromPipe((EnrichedEvent) event, committerKey)) { if (((EnrichedEvent) event).clearReferenceCount(BlockingPendingQueue.class.getName())) { eventCounter.decreaseEventCount(event); } @@ -157,9 +174,48 @@ public int getPipeHeartbeatEventCount() { return eventCounter.getPipeHeartbeatEventCount(); } - protected void checkBeforeOffer(final E event) { - if (isClosed.get() && event instanceof EnrichedEvent) { + protected boolean checkBeforeOffer(final E event) { + final boolean shouldReject = isClosed.get() || isEventFromDroppedPipe(event); + if (shouldReject && event instanceof EnrichedEvent) { ((EnrichedEvent) event).clearReferenceCount(BlockingPendingQueue.class.getName()); } + return !shouldReject; + } + + protected static boolean isEventFromPipe( + final EnrichedEvent event, + final String pipeNameToDrop, + final long creationTimeToDrop, + final int regionId) { + return pipeNameToDrop.equals(event.getPipeName()) + && creationTimeToDrop == event.getCreationTime() + && regionId == event.getRegionId(); + } + + protected static boolean isEventFromPipe( + final EnrichedEvent event, final CommitterKey committerKey) { + return committerKey.getPipeName().equals(event.getPipeName()) + && committerKey.getCreationTime() == event.getCreationTime() + && committerKey.getRegionId() == event.getRegionId() + && (committerKey.getRestartTimes() < 0 || committerKey.equals(event.getCommitterKey())); + } + + protected boolean isEventFromDroppedPipe(final E event) { + return event instanceof EnrichedEvent + && ((EnrichedEvent) event).getPipeName() != null + && isEventFromDroppedPipe((EnrichedEvent) event); + } + + public boolean isEventFromDroppedPipe(final EnrichedEvent event) { + return droppedPipeTaskKeys.stream().anyMatch(key -> isEventFromPipe(event, key)); + } + + public boolean isPipeDropped(final String pipeName, final long creationTime, final int regionId) { + return droppedPipeTaskKeys.stream() + .anyMatch( + key -> + key.getPipeName().equals(pipeName) + && key.getCreationTime() == creationTime + && key.getRegionId() == regionId); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java index 7056b052a3ee1..9e1653a25167d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/progress/PipeEventCommitManager.java @@ -169,6 +169,11 @@ private boolean commitSingleId( return true; } + public CommitterKey getCommitterKey( + final String pipeName, final long creationTime, final int regionId) { + return generateCommitterKey(pipeName, creationTime, regionId); + } + private CommitterKey generateCommitterKey( final String pipeName, final long creationTime, final int regionId) { return taskAgent.getCommitterKey( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java index 02284671803d5..ee035ba3eb1f6 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/PipeDescriptor.java @@ -443,11 +443,13 @@ public static void loadPipeInternalConfig(CommonConfig config, TrimProperties pr "rate_limiter_hot_reload_check_interval_ms", String.valueOf(config.getRateLimiterHotReloadCheckIntervalMs())))); - config.setPipeConnectorRequestSliceThresholdBytes( + config.setPipeSinkRequestSliceThresholdBytes( Integer.parseInt( - properties.getProperty( - "pipe_connector_request_slice_threshold_bytes", - String.valueOf(config.getPipeSinkRequestSliceThresholdBytes())))); + Optional.ofNullable(properties.getProperty("pipe_sink_request_slice_threshold_bytes")) + .orElse( + properties.getProperty( + "pipe_connector_request_slice_threshold_bytes", + String.valueOf(config.getPipeSinkRequestSliceThresholdBytes()))))); config.setPipeReceiverLoginPeriodicVerificationIntervalMs( Long.parseLong( @@ -613,7 +615,7 @@ public static void loadPipeExternalConfig( "pipe_async_connector_selector_number", isHotModify); if (value != null) { - config.setPipeAsyncConnectorSelectorNumber(Integer.parseInt(value)); + config.setPipeAsyncSinkSelectorNumber(Integer.parseInt(value)); } value = @@ -623,7 +625,7 @@ public static void loadPipeExternalConfig( "pipe_async_connector_max_client_number", isHotModify); if (value != null) { - config.setPipeAsyncConnectorMaxClientNumber(Integer.parseInt(value)); + config.setPipeAsyncSinkMaxClientNumber(Integer.parseInt(value)); } value = @@ -633,7 +635,7 @@ public static void loadPipeExternalConfig( "pipe_async_connector_max_tsfile_client_number", isHotModify); if (value != null) { - config.setPipeAsyncConnectorMaxTsFileClientNumber(Integer.parseInt(value)); + config.setPipeAsyncSinkMaxTsFileClientNumber(Integer.parseInt(value)); } value = diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java index 2eaf6f903de48..e04f2578daac9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSinkConstant.java @@ -231,6 +231,11 @@ public class PipeSinkConstant { "connector.opcua.timeout-seconds"; public static final long CONNECTOR_OPC_UA_TIMEOUT_SECONDS_DEFAULT_VALUE = 10L; + public static final String CONNECTOR_OPC_UA_DEBOUNCE_TIME_MS_KEY = + "connector.opcua.debounce-time-ms"; + public static final String SINK_OPC_UA_DEBOUNCE_TIME_MS_KEY = "sink.opcua.debounce-time-ms"; + public static final long CONNECTOR_OPC_UA_DEBOUNCE_TIME_MS_DEFAULT_VALUE = 50L; + public static final String CONNECTOR_LEADER_CACHE_ENABLE_KEY = "connector.leader-cache.enable"; public static final String SINK_LEADER_CACHE_ENABLE_KEY = "sink.leader-cache.enable"; public static final boolean CONNECTOR_LEADER_CACHE_ENABLE_DEFAULT_VALUE = true; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSourceConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSourceConstant.java index e9adf64bc03d1..aba780fb0d1af 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSourceConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeSourceConstant.java @@ -24,6 +24,13 @@ public class PipeSourceConstant { public static final String EXTRACTOR_KEY = "extractor"; public static final String SOURCE_KEY = "source"; + public static final String EXTRACTOR_IOTDB_USER_KEY = "extractor.user"; + public static final String SOURCE_IOTDB_USER_KEY = "source.user"; + public static final String EXTRACTOR_IOTDB_USERNAME_KEY = "extractor.username"; + public static final String SOURCE_IOTDB_USERNAME_KEY = "source.username"; + public static final String EXTRACTOR_IOTDB_PASSWORD_KEY = "extractor.password"; + public static final String SOURCE_IOTDB_PASSWORD_KEY = "source.password"; + public static final String EXTRACTOR_INCLUSION_KEY = "extractor.inclusion"; public static final String SOURCE_INCLUSION_KEY = "source.inclusion"; public static final String EXTRACTOR_INCLUSION_DEFAULT_VALUE = "data.insert"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/Triple.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/Triple.java new file mode 100644 index 0000000000000..275ccb20ea3ac --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/Triple.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.datastructure; + +import java.util.Objects; + +public class Triple { + public final L first; + public final M second; + public final R third; + + public Triple(final L first, final M second, final R third) { + this.first = first; + this.second = second; + this.third = third; + } + + public L getFirst() { + return first; + } + + public M getSecond() { + return second; + } + + public R getThird() { + return third; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final Triple triple = (Triple) o; + return first.equals(triple.first) && second.equals(triple.second) && third.equals(triple.third); + } + + @Override + public int hashCode() { + return Objects.hash(first, second, third); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java index 9153acf9b2e64..7346ae2bccd48 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java @@ -126,6 +126,10 @@ protected TPipeTransferResp handleTransferHandshakeV1(final PipeTransferHandshak String.format( "Pipe-Receiver-%s-%s:%s", receiverId.get(), getSenderHost(), getSenderPort())); + // Handshake restarts the transfer session. Reset the current writing state before recycling the + // old receiver dir, otherwise the old file handle can survive across handshakes. + resetCurrentWritingFileState(); + // Clear the original receiver file dir if exists if (receiverFileDirWithIdSuffix.get() != null) { if (receiverFileDirWithIdSuffix.get().exists()) { @@ -461,16 +465,7 @@ protected final void updateWritingFileIfNeeded(final String fileName, final bool receiverFileDirWithIdSuffix.get().getPath()); } } - Path baseDir = receiverFileDirWithIdSuffix.get().toPath().toAbsolutePath().normalize(); - Path targetPath = baseDir.resolve(fileName).toAbsolutePath().normalize(); - - if (!targetPath.startsWith(baseDir)) { - LOGGER.error( - "Receiver id = {}: Path traversal attempt detected! Filename: {}", - receiverId.get(), - fileName); - throw new IOException("Illegal fileName: " + fileName + " (Path traversal detected)"); - } + final Path targetPath = resolveReceiverFilePath(fileName); writingFile = targetPath.toFile(); writingFileWriter = new RandomAccessFile(writingFile, "rw"); @@ -481,7 +476,37 @@ protected final void updateWritingFileIfNeeded(final String fileName, final bool } private boolean isFileExistedAndNameCorrect(final String fileName) { - return writingFile != null && writingFile.exists() && writingFile.getName().equals(fileName); + try { + return writingFile != null + && writingFile.exists() + && receiverFileDirWithIdSuffix.get() != null + && writingFile + .toPath() + .toAbsolutePath() + .normalize() + .equals(resolveReceiverFilePath(fileName)); + } catch (final IOException e) { + PipeLogger.log( + LOGGER::warn, + e, + "Receiver id = %s: Illegal file name %s when checking writing file.", + receiverId.get(), + fileName); + return false; + } + } + + private Path resolveReceiverFilePath(final String fileName) throws IOException { + try { + return PipeReceiverFilePathUtils.resolveFilePath( + receiverFileDirWithIdSuffix.get().toPath(), fileName); + } catch (final IOException e) { + LOGGER.error( + "Receiver id = {}: Path traversal attempt detected! Filename: {}", + receiverId.get(), + fileName); + throw e; + } } private void closeCurrentWritingFileWriter(final boolean fsyncAfterClose) { @@ -526,6 +551,11 @@ private void deleteCurrentWritingFile() { } } + private void resetCurrentWritingFileState() { + closeCurrentWritingFileWriter(false); + writingFile = null; + } + private void deleteFile(final File file) { if (file.exists()) { try { @@ -568,6 +598,8 @@ private boolean isWritingFileOffsetCorrect(final long offset) throws IOException } protected final TPipeTransferResp handleTransferFileSealV1(final PipeTransferFileSealReqV1 req) { + File sealedWritingFile = null; + boolean shouldDeleteSealedFile = true; try { if (!isWritingFileAvailable()) { final TSStatus status = @@ -584,7 +616,8 @@ protected final TPipeTransferResp handleTransferFileSealV1(final PipeTransferFil return resp; } - final String fileAbsolutePath = writingFile.getAbsolutePath(); + sealedWritingFile = writingFile; + final String fileAbsolutePath = sealedWritingFile.getAbsolutePath(); // Sync here is necessary to ensure that the data is written to the disk. Or data region may // load the file before the data is written to the disk and cause unexpected behavior after @@ -603,11 +636,13 @@ protected final TPipeTransferResp handleTransferFileSealV1(final PipeTransferFil writingFileWriter.close(); writingFileWriter = null; - // writingFile will be deleted after load if no exception occurs + // Clear the reference before loading so the next file transfer can not reuse the same path. + // The loader owns cleanup after a successful load. writingFile = null; final TSStatus status = loadFileV1(req, fileAbsolutePath); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + shouldDeleteSealedFile = false; LOGGER.info( "Receiver id = {}: Seal file {} successfully.", receiverId.get(), fileAbsolutePath); } else { @@ -636,16 +671,34 @@ protected final TPipeTransferResp handleTransferFileSealV1(final PipeTransferFil // All pieces of the writing file and its mod (if exists) should be retransmitted by the // sender. closeCurrentWritingFileWriter(false); - deleteCurrentWritingFile(); + if (shouldDeleteSealedFile) { + if (writingFile != null) { + deleteCurrentWritingFile(); + } else if (sealedWritingFile != null) { + deleteFile(sealedWritingFile); + } + } } } protected final TPipeTransferResp handleTransferFileSealV2(final PipeTransferFileSealReqV2 req) { - final List files = - req.getFileNames().stream() - .map(fileName -> new File(receiverFileDirWithIdSuffix.get(), fileName)) - .collect(Collectors.toList()); + final List fileNames = req.getFileNames(); try { + final List files = + fileNames.stream() + .map( + fileName -> { + if (Objects.isNull(fileName)) { + return null; + } + try { + return resolveReceiverFilePath(fileName).toFile(); + } catch (final IOException e) { + throw new IllegalArgumentException(e); + } + }) + .collect(Collectors.toList()); + if (!isWritingFileAvailable()) { final TSStatus status = RpcUtils.getStatus( @@ -707,17 +760,20 @@ protected final TPipeTransferResp handleTransferFileSealV2(final PipeTransferFil } return new TPipeTransferResp(status); } catch (final Exception e) { + final Throwable rootCause = e instanceof IllegalArgumentException ? e.getCause() : e; PipeLogger.log( LOGGER::warn, - e, + rootCause, "Receiver id = %s: Failed to seal file %s from req %s.", receiverId.get(), - files, + fileNames, req); return new TPipeTransferResp( RpcUtils.getStatus( TSStatusCode.PIPE_TRANSFER_FILE_ERROR, - String.format("Failed to seal file %s because %s", writingFile, e.getMessage()))); + String.format( + "Failed to seal file %s because %s", + fileNames, rootCause == null ? e.getMessage() : rootCause.getMessage()))); } finally { // If the writing file is not sealed successfully, the writing file will be deleted. // All pieces of the writing file and its mod(if exists) should be retransmitted by the diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/PipeReceiverFilePathUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/PipeReceiverFilePathUtils.java new file mode 100644 index 0000000000000..bc7275d4ebe16 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/PipeReceiverFilePathUtils.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.receiver; + +import java.io.IOException; +import java.nio.file.Path; + +public final class PipeReceiverFilePathUtils { + + private PipeReceiverFilePathUtils() { + // Utility class + } + + public static Path resolveFilePath(final Path baseDir, final String fileName) throws IOException { + final Path normalizedBaseDir = baseDir.toAbsolutePath().normalize(); + final Path normalizedTargetPath = + normalizedBaseDir.resolve(fileName).toAbsolutePath().normalize(); + + if (!normalizedTargetPath.startsWith(normalizedBaseDir)) { + throw new IOException("Illegal fileName: " + fileName + " (Path traversal detected)"); + } + + return normalizedTargetPath; + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/PipeReceiverStatusHandler.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/PipeReceiverStatusHandler.java index d48a9fdbcb2d5..604ad044dd6af 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/PipeReceiverStatusHandler.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/PipeReceiverStatusHandler.java @@ -46,6 +46,7 @@ public class PipeReceiverStatusHandler { private static final String NO_PERMISSION = "No permission"; private static final String UNCLASSIFIED_EXCEPTION = "Unclassified exception"; private static final String NO_PERMISSION_STR = "No permissions for this operation"; + private static final int MAX_RECORD_MESSAGE_LENGTH_IN_LOG = 2048; private final boolean isRetryAllowedWhenConflictOccurs; private final long retryMaxMillisWhenConflictOccurs; @@ -134,6 +135,7 @@ public void handle( "User conflict exception: will be ignored because retry is not allowed. event: {}. status: {}", shouldRecordIgnoredDataWhenConflictOccurs ? recordMessage : "not recorded", status); + logDiscardedUserConflictData("retry is not allowed", recordMessage, status); return; } @@ -147,6 +149,7 @@ public void handle( "User conflict exception: retry timeout. will be ignored. event: {}. status: {}", shouldRecordIgnoredDataWhenConflictOccurs ? recordMessage : "not recorded", status); + logDiscardedUserConflictData("retry timeout", recordMessage, status); resetExceptionStatus(); return; } @@ -252,6 +255,32 @@ private static String getNoPermission(final boolean noPermission) { return noPermission ? NO_PERMISSION : UNCLASSIFIED_EXCEPTION; } + private void logDiscardedUserConflictData( + final String reason, final String recordMessage, final TSStatus status) { + if (!LOGGER.isWarnEnabled()) { + return; + } + + LOGGER.warn( + "User conflict exception: discarded data info because {}. data: {}. receiver message: {}. status: {}", + reason, + summarizeRecordMessage(recordMessage), + status.getMessage(), + status); + } + + private String summarizeRecordMessage(final String recordMessage) { + if (Objects.isNull(recordMessage) || recordMessage.isEmpty()) { + return ""; + } + + final String normalizedRecordMessage = + recordMessage.replace('\r', ' ').replace('\n', ' ').trim(); + return normalizedRecordMessage.length() <= MAX_RECORD_MESSAGE_LENGTH_IN_LOG + ? normalizedRecordMessage + : normalizedRecordMessage.substring(0, MAX_RECORD_MESSAGE_LENGTH_IN_LOG) + "...(truncated)"; + } + private void recordExceptionStatusIfNecessary(final String message) { if (!Objects.equals(exceptionRecordedMessage.get(), message)) { exceptionFirstEncounteredTime.set(System.currentTimeMillis()); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBSyncClient.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBSyncClient.java index b7f42295e6cc3..1ad5d0a855f23 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBSyncClient.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBSyncClient.java @@ -22,9 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.client.ThriftClient; import org.apache.iotdb.commons.client.property.ThriftClientProperty; -import org.apache.iotdb.commons.pipe.config.PipeConfig; -import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; -import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferSliceReq; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.common.PipeTransferSliceReqBuilder; import org.apache.iotdb.pipe.api.exception.PipeConnectionException; import org.apache.iotdb.rpc.DeepCopyRpcTransportFactory; import org.apache.iotdb.rpc.TSStatusCode; @@ -39,15 +37,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.atomic.AtomicInteger; - public class IoTDBSyncClient extends IClientRPCService.Client implements ThriftClient, AutoCloseable { private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBSyncClient.class); - private static final AtomicInteger SLICE_ORDER_ID_GENERATOR = new AtomicInteger(0); - private final String ipAddress; private final int port; private final TEndPoint endPoint; @@ -100,9 +94,8 @@ public void setTimeout(int timeout) { @Override public TPipeTransferResp pipeTransfer(final TPipeTransferReq req) throws TException { - final int bodySizeLimit = PipeConfig.getInstance().getPipeSinkRequestSliceThresholdBytes(); - if (req.getVersion() != IoTDBSinkRequestVersion.VERSION_1.getVersion() - || req.body.limit() < bodySizeLimit) { + final int bodySizeLimit = PipeTransferSliceReqBuilder.getBodySizeLimit(); + if (!PipeTransferSliceReqBuilder.shouldSlice(req, bodySizeLimit)) { return super.pipeTransfer(req); } @@ -115,23 +108,13 @@ public TPipeTransferResp pipeTransfer(final TPipeTransferReq req) throws TExcept bodySizeLimit); try { - final int sliceOrderId = SLICE_ORDER_ID_GENERATOR.getAndIncrement(); - // Slice the buffer to avoid the buffer being too large - final int sliceCount = - req.body.limit() / bodySizeLimit + (req.body.limit() % bodySizeLimit == 0 ? 0 : 1); + final int sliceOrderId = PipeTransferSliceReqBuilder.nextSliceOrderId(); + final int sliceCount = PipeTransferSliceReqBuilder.getSliceCount(req, bodySizeLimit); for (int i = 0; i < sliceCount; ++i) { - final int startIndexInBody = i * bodySizeLimit; - final int endIndexInBody = Math.min((i + 1) * bodySizeLimit, req.body.limit()); final TPipeTransferResp sliceResp = super.pipeTransfer( - PipeTransferSliceReq.toTPipeTransferReq( - sliceOrderId, - req.getType(), - i, - sliceCount, - req.body.duplicate(), - startIndexInBody, - endIndexInBody)); + PipeTransferSliceReqBuilder.buildSliceReq( + req, sliceOrderId, i, sliceCount, bodySizeLimit)); if (i == sliceCount - 1) { return sliceResp; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilder.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilder.java new file mode 100644 index 0000000000000..b108d6f1d3ab6 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilder.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.payload.thrift.common; + +import org.apache.iotdb.commons.pipe.config.PipeConfig; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferSliceReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; + +public final class PipeTransferSliceReqBuilder { + + private static final AtomicInteger SLICE_ORDER_ID_GENERATOR = new AtomicInteger(0); + + private PipeTransferSliceReqBuilder() { + // Utility class + } + + public static int getBodySizeLimit() { + return PipeConfig.getInstance().getPipeSinkRequestSliceThresholdBytes(); + } + + public static boolean shouldSlice(final TPipeTransferReq req, final int bodySizeLimit) { + return req.getVersion() == IoTDBSinkRequestVersion.VERSION_1.getVersion() + && req.body.limit() >= bodySizeLimit; + } + + public static int nextSliceOrderId() { + return SLICE_ORDER_ID_GENERATOR.getAndIncrement(); + } + + public static int getSliceCount(final TPipeTransferReq req, final int bodySizeLimit) { + return req.body.limit() / bodySizeLimit + (req.body.limit() % bodySizeLimit == 0 ? 0 : 1); + } + + public static PipeTransferSliceReq buildSliceReq( + final TPipeTransferReq originalReq, + final int sliceOrderId, + final int sliceIndex, + final int sliceCount, + final int bodySizeLimit) + throws IOException { + final int startIndexInBody = sliceIndex * bodySizeLimit; + final int endIndexInBody = Math.min((sliceIndex + 1) * bodySizeLimit, originalReq.body.limit()); + return PipeTransferSliceReq.toTPipeTransferReq( + sliceOrderId, + originalReq.getType(), + sliceIndex, + sliceCount, + originalReq.body.duplicate(), + startIndexInBody, + endIndexInBody); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBAirGapSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBAirGapSink.java index 763bfa644ead7..498baeda5822f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBAirGapSink.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBAirGapSink.java @@ -104,14 +104,6 @@ public void customize( final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration) throws Exception { super.customize(parameters, configuration); - - if (isTabletBatchModeEnabled) { - LOGGER.warn( - "Batch mode is enabled by the given parameters. " - + "IoTDBAirGapConnector does not support batch mode. " - + "Disable batch mode."); - } - for (int i = 0; i < nodeUrls.size(); i++) { isSocketAlive.add(false); sockets.add(null); @@ -323,15 +315,17 @@ protected int nextSocketIndex() { protected boolean send( final String pipeName, final long creationTime, final AirGapSocket socket, byte[] bytes) throws IOException { + bytes = compressIfNeeded(bytes); + rateLimitIfNeeded(pipeName, creationTime, socket.getEndPoint(), bytes.length); + return sendBytes(socket, bytes); + } + + protected boolean sendBytes(final AirGapSocket socket, byte[] bytes) throws IOException { if (!socket.isConnected()) { throw new SocketException( String.format("Socket %s is closed, will try to handshake", socket)); } - bytes = compressIfNeeded(bytes); - - rateLimitIfNeeded(pipeName, creationTime, socket.getEndPoint(), bytes.length); - final BufferedOutputStream outputStream = new BufferedOutputStream(socket.getOutputStream()); bytes = enrichWithLengthAndChecksum(bytes); outputStream.write(eLanguageEnable ? enrichWithELanguage(bytes) : bytes); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java index 66f3eda6a1a48..2f2d54e7b4fe0 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java @@ -139,7 +139,7 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.SINK_RATE_LIMIT_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSinkConstant.SINK_SKIP_IF_KEY; -public abstract class IoTDBSink implements PipeConnector { +public abstract class IoTDBSink implements PipeConnector, PipeConnectorWithEventDiscard { private static final String PARSE_URL_ERROR_FORMATTER = "Exception occurred while parsing node urls from target servers: {}"; @@ -621,7 +621,8 @@ public void rateLimitIfNeeded( * When a pipe is dropped, the connector maybe reused and will not be closed. We need to discard * its batched or queued events in the output pipe connector. */ - public synchronized void discardEventsOfPipe(final String pipeName, final int regionId) { + public synchronized void discardEventsOfPipe( + final String pipeName, final long creationTime, final int regionId) { // Do nothing by default } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/PipeConnectorWithEventDiscard.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/PipeConnectorWithEventDiscard.java new file mode 100644 index 0000000000000..4ffc0c25ed244 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/PipeConnectorWithEventDiscard.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.protocol; + +import org.apache.iotdb.commons.pipe.agent.task.progress.CommitterKey; + +public interface PipeConnectorWithEventDiscard { + + void discardEventsOfPipe(String pipeName, long creationTime, int regionId); + + default void discardEventsOfPipe(final CommitterKey committerKey) { + discardEventsOfPipe( + committerKey.getPipeName(), committerKey.getCreationTime(), committerKey.getRegionId()); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilter.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilter.java index a9e5fea0b73ca..d59e59ba2bc88 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilter.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/SchemaFilter.java @@ -91,7 +91,12 @@ public static List extract(SchemaFilter schemaFilter, SchemaFilter } private static void internalExtract( - List result, SchemaFilter schemaFilter, SchemaFilterType filterType) { + final List result, + final SchemaFilter schemaFilter, + final SchemaFilterType filterType) { + if (schemaFilter == null) { + return; + } if (schemaFilter.getSchemaFilterType().equals(filterType)) { result.add(schemaFilter); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/common/AbstractAboveDatabaseMNode.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/common/AbstractAboveDatabaseMNode.java index ab0592480e5e2..fc379e34e5b02 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/common/AbstractAboveDatabaseMNode.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/common/AbstractAboveDatabaseMNode.java @@ -37,6 +37,10 @@ public AbstractAboveDatabaseMNode(BasicNode basicMNode) { this.basicMNode = basicMNode; } + public BasicNode getBasicMNode() { + return basicMNode; + } + @Override public String getName() { return basicMNode.getName(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/utils/IMNodeIterator.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/utils/IMNodeIterator.java index 75fd9d40c67dc..d1f410c546015 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/utils/IMNodeIterator.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/utils/IMNodeIterator.java @@ -22,7 +22,7 @@ import java.util.Iterator; -public interface IMNodeIterator> extends Iterator { +public interface IMNodeIterator> extends Iterator, AutoCloseable { void skipTemplateChildren(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java index caa8019ef8717..28b192e99cae1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java @@ -413,6 +413,10 @@ protected Iterator getCurrentChildrenIterator() { } } + protected final IStateMatchInfo getCurrentStateMatchInfo() { + return currentStateMatchInfo; + } + // Release a child node. protected void releaseNode(N node) {} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java index b3a3e3a891cd7..59f4653f0ae1f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java @@ -335,6 +335,9 @@ public void deserialize(InputStream bufferedInputStream) public void clear() { ttlCacheTree.removeAllChildren(); ttlCacheTree.addChild(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD, Long.MAX_VALUE); + + // root.** + ttlCount = 1; } static class CacheNode { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java index b316c5f155de6..640d154937b4b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerGroupMeta.java @@ -69,8 +69,11 @@ public ConsumerGroupMeta deepCopy() { final ConsumerGroupMeta copied = new ConsumerGroupMeta(); copied.consumerGroupId = consumerGroupId; copied.creationTime = creationTime; - copied.topicNameToSubscribedConsumerIdSet = - new ConcurrentHashMap<>(topicNameToSubscribedConsumerIdSet); + copied.topicNameToSubscribedConsumerIdSet = new ConcurrentHashMap<>(); + topicNameToSubscribedConsumerIdSet.forEach( + (topicName, subscribedConsumerIds) -> + copied.topicNameToSubscribedConsumerIdSet.put( + topicName, new HashSet<>(subscribedConsumerIds))); copied.consumerIdToConsumerMeta = new ConcurrentHashMap<>(consumerIdToConsumerMeta); copied.topicNameToSubscriptionCreationTime = new ConcurrentHashMap<>(topicNameToSubscriptionCreationTime); @@ -136,6 +139,10 @@ public boolean containsConsumer(final String consumerId) { return consumerIdToConsumerMeta.containsKey(consumerId); } + public ConsumerMeta getConsumerMeta(final String consumerId) { + return consumerIdToConsumerMeta.get(consumerId); + } + public boolean isEmpty() { // When there are no consumers in a consumer group, it means that the ConsumerGroupMeta is // empty, and at this time, the topicNameToSubscribedConsumerIdSet is also empty. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerMeta.java index f1bb9b4608531..75cca9ccbcdd9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/consumer/ConsumerMeta.java @@ -54,6 +54,26 @@ public String getConsumerId() { return consumerId; } + public String getConsumerGroupId() { + return config.getConsumerGroupId(); + } + + public String getUsername() { + return config.getUsername(); + } + + public String getPassword() { + return config.getPassword(); + } + + public String getEncryptedPassword() { + return config.getEncryptedPassword(); + } + + public String getSubscriptionAuthPassword() { + return Objects.nonNull(getEncryptedPassword()) ? getEncryptedPassword() : getPassword(); + } + public ByteBuffer serialize() throws IOException { PublicBAOS byteArrayOutputStream = new PublicBAOS(); DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java index 93cd87e4ae386..ee9177112e710 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/subscription/meta/topic/TopicMeta.java @@ -181,11 +181,22 @@ public static TopicMeta deserialize(final ByteBuffer byteBuffer) { /////////////////////////////// utilities /////////////////////////////// public Map generateExtractorAttributes() { + return generateExtractorAttributes(null, null); + } + + public Map generateExtractorAttributes( + final String username, final String password) { final Map extractorAttributes = new HashMap<>(); // disable meta sync extractorAttributes.put("source", "iotdb-source"); extractorAttributes.put("inclusion", "data.insert"); extractorAttributes.put("inclusion.exclusion", "data.delete"); + if (Objects.nonNull(username)) { + extractorAttributes.put("username", username); + } + if (Objects.nonNull(password)) { + extractorAttributes.put("password", password); + } // path extractorAttributes.putAll(config.getAttributesWithPathOrPattern()); // time diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java index fe413bf85ef86..557adf43f6274 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/FileUtils.java @@ -54,6 +54,7 @@ public class FileUtils { "Renamed file {} to {} because it already exists in the target directory: {}"; private static final String COPY_FILE_MESSAGE = "Copy file {} to {} because it already exists in the target directory: {}"; + private static final String ILLEGAL_EMPTY_PATH_MESSAGE = "The path cannot be empty. "; private static final String ILLEGAL_PATH_MESSAGE = "The path cannot be '.', '..', './' or '.\\'. "; @@ -556,6 +557,9 @@ private static void copyFileRename(final File sourceFile, final File targetFile) } public static String getIllegalError4Directory(final String path) { + if (path == null || path.isEmpty()) { + return ILLEGAL_EMPTY_PATH_MESSAGE; + } if (path.equals(".") || path.equals("..") || path.contains("/") || path.contains("\\")) { return ILLEGAL_PATH_MESSAGE; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/WindowsOSUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/WindowsOSUtils.java index 3019c47cf0e1c..0be89f92747a9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/WindowsOSUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/WindowsOSUtils.java @@ -23,15 +23,27 @@ import java.util.Arrays; import java.util.HashSet; +import java.util.Locale; import java.util.Set; public class WindowsOSUtils { private static final String ILLEGAL_WINDOWS_CHARS = "\\/:*?\"<>|"; private static final Set ILLEGAL_WINDOWS_NAMES = - new HashSet<>(Arrays.asList("CON", "PRN", "AUX", "NUL", "COM1-COM9, LPT1-LPT9")); + new HashSet<>( + Arrays.asList( + "CON", + "PRN", + "AUX", + "NUL", + "COM\u00B9", + "COM\u00B2", + "COM\u00B3", + "LPT\u00B9", + "LPT\u00B2", + "LPT\u00B3")); static { - for (int i = 0; i < 10; ++i) { + for (int i = 1; i < 10; ++i) { ILLEGAL_WINDOWS_NAMES.add("COM" + i); ILLEGAL_WINDOWS_NAMES.add("LPT" + i); } @@ -39,26 +51,39 @@ public class WindowsOSUtils { public static final String OS_SEGMENT_ERROR = String.format( - "In Windows System, the path shall not contains %s, equals one of %s, or ends with '.' or ' '.", + "In Windows System, the path shall not contain %s or ASCII control characters, equals one of %s with or without an extension, or ends with '.' or ' '.", ILLEGAL_WINDOWS_CHARS, ILLEGAL_WINDOWS_NAMES); public static boolean isLegalPathSegment4Windows(final String pathSegment) { if (!SystemUtils.IS_OS_WINDOWS) { return true; } - for (final char illegalChar : ILLEGAL_WINDOWS_CHARS.toCharArray()) { - if (pathSegment.indexOf(illegalChar) != -1) { - return false; - } + if (containsIllegalWindowsChar(pathSegment)) { + return false; } if (pathSegment.endsWith(".") || pathSegment.endsWith(" ")) { return false; } - for (final String illegalName : ILLEGAL_WINDOWS_NAMES) { - if (pathSegment.equalsIgnoreCase(illegalName)) { - return false; - } + if (isIllegalWindowsName(pathSegment)) { + return false; } return true; } + + private static boolean containsIllegalWindowsChar(final String pathSegment) { + for (int i = 0; i < pathSegment.length(); ++i) { + final char ch = pathSegment.charAt(i); + if (ch < ' ' || ILLEGAL_WINDOWS_CHARS.indexOf(ch) != -1) { + return true; + } + } + return false; + } + + private static boolean isIllegalWindowsName(final String pathSegment) { + final int extensionStartIndex = pathSegment.indexOf('.'); + final String nameWithoutExtension = + extensionStartIndex < 0 ? pathSegment : pathSegment.substring(0, extensionStartIndex); + return ILLEGAL_WINDOWS_NAMES.contains(nameWithoutExtension.toUpperCase(Locale.ENGLISH)); + } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java index 70c32ca4801d8..7296a0b3f78ca 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/ClientManagerTest.java @@ -214,6 +214,7 @@ public void evictionTest() throws Exception { manager, new ThriftClientProperty.Builder().build()), new ClientPoolProperty.Builder() .setMaxClientNumForEachNode(maxClientForEachNode) + .setMaxIdleClientNumForEachNode(maxClientForEachNode) .setMinIdleTimeForClient(minIdleDuration) .setTimeBetweenEvictionRuns(evictionRunsDuration) .build() @@ -294,6 +295,7 @@ public void maxTotalTest() throws Exception { manager, new ThriftClientProperty.Builder().build()), new ClientPoolProperty.Builder() .setMaxClientNumForEachNode(maxTotalClientForEachNode) + .setMaxIdleClientNumForEachNode(maxTotalClientForEachNode) .setWaitClientTimeoutMs(waitClientTimeoutMs) .build() .getConfig()); @@ -369,6 +371,7 @@ public void maxWaitClientTimeoutTest() throws Exception { new ClientPoolProperty.Builder() .setWaitClientTimeoutMs(waitClientTimeoutMS) .setMaxClientNumForEachNode(maxTotalClientForEachNode) + .setMaxIdleClientNumForEachNode(maxTotalClientForEachNode) .build() .getConfig()); } @@ -617,7 +620,13 @@ public GenericKeyedObjectPool crea new ThriftClientProperty.Builder() .setConnectionTimeoutMs(CONNECTION_TIMEOUT) .build()), - new ClientPoolProperty.Builder().build().getConfig()); + new ClientPoolProperty.Builder() + .setMaxClientNumForEachNode( + ClientPoolProperty.DefaultProperty.MAX_CLIENT_NUM_FOR_EACH_NODE) + .setMaxIdleClientNumForEachNode( + ClientPoolProperty.DefaultProperty.MAX_IDLE_CLIENT_NUM_FOR_EACH_NODE) + .build() + .getConfig()); } } @@ -632,7 +641,13 @@ public GenericKeyedObjectPool cre manager, new ThriftClientProperty.Builder().setConnectionTimeoutMs(CONNECTION_TIMEOUT).build(), ThreadName.ASYNC_DATANODE_CLIENT_POOL.getName()), - new ClientPoolProperty.Builder().build().getConfig()); + new ClientPoolProperty.Builder() + .setMaxClientNumForEachNode( + ClientPoolProperty.DefaultProperty.MAX_CLIENT_NUM_FOR_EACH_NODE) + .setMaxIdleClientNumForEachNode( + ClientPoolProperty.DefaultProperty.MAX_IDLE_CLIENT_NUM_FOR_EACH_NODE) + .build() + .getConfig()); } } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/executable/ReferenceCountedJarMetaKeeperTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/executable/ReferenceCountedJarMetaKeeperTest.java new file mode 100644 index 0000000000000..6a48701f5747a --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/executable/ReferenceCountedJarMetaKeeperTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.executable; + +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +public class ReferenceCountedJarMetaKeeperTest { + + private static final String JAR_NAME = "shared.jar"; + private static final String MD5 = "12345"; + + @Test + public void testReferenceCounting() { + final ReferenceCountedJarMetaKeeper keeper = new ReferenceCountedJarMetaKeeper(); + + keeper.addReference(JAR_NAME, MD5); + keeper.addReference(JAR_NAME, MD5); + + Assert.assertTrue(keeper.containsJar(JAR_NAME)); + Assert.assertFalse(keeper.needToSaveJar(JAR_NAME)); + Assert.assertTrue(keeper.jarNameExistsAndMatchesMd5(JAR_NAME, MD5)); + Assert.assertFalse(keeper.jarNameExistsAndMatchesMd5(JAR_NAME, "54321")); + + keeper.removeReference(JAR_NAME); + Assert.assertTrue(keeper.containsJar(JAR_NAME)); + + keeper.removeReference(JAR_NAME); + Assert.assertFalse(keeper.containsJar(JAR_NAME)); + } + + @Test + public void testJarNameToMd5Snapshot() throws IOException { + final ReferenceCountedJarMetaKeeper keeper = new ReferenceCountedJarMetaKeeper(); + keeper.addReference(JAR_NAME, MD5); + keeper.addReference(JAR_NAME, MD5); + + final ReferenceCountedJarMetaKeeper loaded = new ReferenceCountedJarMetaKeeper(); + loaded.deserializeJarNameToMd5( + new ByteArrayInputStream(serializeJarNameToMd5(keeper).toByteArray())); + + Assert.assertTrue(loaded.jarNameExistsAndMatchesMd5(JAR_NAME, MD5)); + loaded.removeReference(JAR_NAME); + Assert.assertFalse(loaded.containsJar(JAR_NAME)); + } + + @Test + public void testJarNameToMd5AndReferenceCountSnapshot() throws IOException { + final ReferenceCountedJarMetaKeeper keeper = new ReferenceCountedJarMetaKeeper(); + keeper.addReference(JAR_NAME, MD5); + keeper.addReference(JAR_NAME, MD5); + + final ReferenceCountedJarMetaKeeper loaded = new ReferenceCountedJarMetaKeeper(); + loaded.deserializeJarNameToMd5AndReferenceCount( + new ByteArrayInputStream(serializeJarNameToMd5AndReferenceCount(keeper).toByteArray())); + + Assert.assertTrue(loaded.jarNameExistsAndMatchesMd5(JAR_NAME, MD5)); + loaded.removeReference(JAR_NAME); + Assert.assertTrue(loaded.containsJar(JAR_NAME)); + + loaded.removeReference(JAR_NAME); + Assert.assertFalse(loaded.containsJar(JAR_NAME)); + } + + @Test + public void testDeserializeJarNameToMd5AndReferenceCountSkipsZeroReferenceCount() + throws IOException { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ReadWriteIOUtils.write(1, outputStream); + ReadWriteIOUtils.write(JAR_NAME, outputStream); + ReadWriteIOUtils.write(MD5, outputStream); + ReadWriteIOUtils.write(0, outputStream); + + final ReferenceCountedJarMetaKeeper loaded = new ReferenceCountedJarMetaKeeper(); + loaded.deserializeJarNameToMd5AndReferenceCount( + new ByteArrayInputStream(outputStream.toByteArray())); + + Assert.assertFalse(loaded.containsJar(JAR_NAME)); + } + + private ByteArrayOutputStream serializeJarNameToMd5(final ReferenceCountedJarMetaKeeper keeper) + throws IOException { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + keeper.serializeJarNameToMd5(outputStream); + return outputStream; + } + + private ByteArrayOutputStream serializeJarNameToMd5AndReferenceCount( + final ReferenceCountedJarMetaKeeper keeper) throws IOException { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + keeper.serializeJarNameToMd5AndReferenceCount(outputStream); + return outputStream; + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/plugin/service/PipePluginClassLoaderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/plugin/service/PipePluginClassLoaderTest.java new file mode 100644 index 0000000000000..39126656fc1ab --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/agent/plugin/service/PipePluginClassLoaderTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.agent.plugin.service; + +import org.junit.Assert; +import org.junit.Test; + +import javax.tools.JavaCompiler; +import javax.tools.StandardJavaFileManager; +import javax.tools.ToolProvider; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.stream.Stream; + +public class PipePluginClassLoaderTest { + + @Test + public void testPluginClassesShouldOverrideParentClasses() throws Exception { + final Path tempDir = Files.createTempDirectory("pipe-plugin-classloader-test"); + try { + final Path parentSources = Files.createDirectory(tempDir.resolve("parent-sources")); + final Path parentClasses = Files.createDirectory(tempDir.resolve("parent-classes")); + final Path childSources = Files.createDirectory(tempDir.resolve("child-sources")); + final Path childClasses = Files.createDirectory(tempDir.resolve("child-classes")); + + final String sampleSource = + "package test.plugin;" + + "public class Sample {" + + " public String ping() {" + + " return test.dep.Helper.value();" + + " }" + + "}"; + final String parentHelperSource = + "package test.dep;" + + "public class Helper {" + + " public static String value() {" + + " return \"parent\";" + + " }" + + "}"; + final String childHelperSource = + "package test.dep;" + + "public class Helper {" + + " public static String value() {" + + " return \"child\";" + + " }" + + "}"; + + compile( + parentSources, + parentClasses, + createSources(sampleSource, false), + createSources(parentHelperSource, true)); + compile( + childSources, + childClasses, + createSources(sampleSource, false), + createSources(childHelperSource, true)); + + final Path parentJar = tempDir.resolve("parent.jar"); + final Path childJar = tempDir.resolve("child.jar"); + createJar(parentJar, parentClasses, Arrays.asList("test/plugin/Sample.class")); + createJar( + childJar, + childClasses, + Arrays.asList("test/plugin/Sample.class", "test/dep/Helper.class")); + + try (final URLClassLoader parentClassLoader = + new URLClassLoader(new URL[] {parentJar.toUri().toURL()}, null); + final PipePluginClassLoader pluginClassLoader = + new PipePluginClassLoader(childJar.toString(), parentClassLoader)) { + final Class sampleClass = Class.forName("test.plugin.Sample", true, pluginClassLoader); + Assert.assertSame(pluginClassLoader, sampleClass.getClassLoader()); + final Object sample = sampleClass.getDeclaredConstructor().newInstance(); + Assert.assertEquals("child", sampleClass.getMethod("ping").invoke(sample)); + } + } finally { + deleteRecursively(tempDir); + } + } + + private static Map createSources( + final String source, final boolean helperSource) { + final Map sources = new LinkedHashMap<>(); + sources.put(helperSource ? "test.dep.Helper" : "test.plugin.Sample", source); + return sources; + } + + private static void compile( + final Path sourcesDir, final Path classesDir, final Map... sourceGroups) + throws IOException { + final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + Assert.assertNotNull("A JDK is required to run this test.", compiler); + + final List sourceFiles = new ArrayList<>(); + for (final Map sourceGroup : sourceGroups) { + for (final Map.Entry entry : sourceGroup.entrySet()) { + final Path sourceFile = sourcesDir.resolve(entry.getKey().replace('.', '/') + ".java"); + Files.createDirectories(sourceFile.getParent()); + Files.write(sourceFile, entry.getValue().getBytes(StandardCharsets.UTF_8)); + sourceFiles.add(sourceFile.toFile()); + } + } + + try (final StandardJavaFileManager fileManager = + compiler.getStandardFileManager(null, null, StandardCharsets.UTF_8)) { + final boolean success = + compiler + .getTask( + null, + fileManager, + null, + Arrays.asList("-d", classesDir.toString()), + null, + fileManager.getJavaFileObjectsFromFiles(sourceFiles)) + .call(); + Assert.assertTrue(success); + } + } + + private static void createJar( + final Path jarPath, final Path classesDir, final List classEntries) + throws IOException { + try (final JarOutputStream jarOutputStream = + new JarOutputStream(Files.newOutputStream(jarPath))) { + for (final String classEntry : classEntries) { + jarOutputStream.putNextEntry(new JarEntry(classEntry)); + jarOutputStream.write(Files.readAllBytes(classesDir.resolve(classEntry))); + jarOutputStream.closeEntry(); + } + } + } + + private static void deleteRecursively(final Path path) throws IOException { + if (path == null || !Files.exists(path)) { + return; + } + try (final Stream stream = Files.walk(path)) { + for (final Path subPath : + (Iterable) stream.sorted(Comparator.reverseOrder())::iterator) { + Files.deleteIfExists(subPath); + } + } + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/config/PipeDescriptorTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/config/PipeDescriptorTest.java new file mode 100644 index 0000000000000..00d982122719b --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/config/PipeDescriptorTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.config; + +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.TrimProperties; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class PipeDescriptorTest { + + private final CommonConfig config = CommonDescriptor.getInstance().getConfig(); + + private int originalRequestSliceThresholdBytes; + + @Before + public void setUp() { + originalRequestSliceThresholdBytes = config.getPipeSinkRequestSliceThresholdBytes(); + } + + @After + public void tearDown() { + config.setPipeSinkRequestSliceThresholdBytes(originalRequestSliceThresholdBytes); + } + + @Test + public void testPipeRequestSliceThresholdSupportsSinkAndConnectorKeys() { + final TrimProperties connectorProperties = new TrimProperties(); + connectorProperties.setProperty("pipe_connector_request_slice_threshold_bytes", "123"); + PipeDescriptor.loadPipeInternalConfig(config, connectorProperties); + Assert.assertEquals(123, config.getPipeSinkRequestSliceThresholdBytes()); + + final TrimProperties sinkProperties = new TrimProperties(); + sinkProperties.setProperty("pipe_sink_request_slice_threshold_bytes", "456"); + PipeDescriptor.loadPipeInternalConfig(config, sinkProperties); + Assert.assertEquals(456, config.getPipeSinkRequestSliceThresholdBytes()); + + final TrimProperties bothProperties = new TrimProperties(); + bothProperties.setProperty("pipe_connector_request_slice_threshold_bytes", "123"); + bothProperties.setProperty("pipe_sink_request_slice_threshold_bytes", "456"); + PipeDescriptor.loadPipeInternalConfig(config, bothProperties); + Assert.assertEquals(456, config.getPipeSinkRequestSliceThresholdBytes()); + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiverTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiverTest.java index 6658927b9bb4e..c692cb1513a4f 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiverTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiverTest.java @@ -20,9 +20,13 @@ package org.apache.iotdb.commons.pipe.receiver; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeRequestType; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferFileSealReqV1; import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferFileSealReqV2; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferHandshakeV1Req; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; import org.apache.iotdb.service.rpc.thrift.TPipeTransferResp; @@ -31,8 +35,12 @@ import java.io.File; import java.io.IOException; +import java.io.RandomAccessFile; +import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; import java.util.List; public class IoTDBFileReceiverTest { @@ -63,9 +71,80 @@ public void testAllowNormalFileName() throws Exception { } } + @Test + public void testRejectPathTraversalFileNameInSealRequest() throws Exception { + final Path baseDir = Files.createTempDirectory("iotdb-file-receiver-test"); + final DummyFileReceiver receiver = new DummyFileReceiver(baseDir.toFile()); + try { + receiver.createWritingFile("normal.tsfile", false); + + final TPipeTransferResp response = + receiver.sealFiles( + Arrays.asList("../outside.mod", "normal.tsfile"), Arrays.asList(0L, 0L)); + + Assert.assertEquals( + TSStatusCode.PIPE_TRANSFER_FILE_ERROR.getStatusCode(), response.getStatus().getCode()); + Assert.assertTrue(response.getStatus().getMessage().contains("Illegal fileName")); + } finally { + receiver.handleExit(); + } + } + + @Test + public void testHandshakeResetsWritingFileState() throws Exception { + final Path baseDir = Files.createTempDirectory("iotdb-file-receiver-test"); + final DummyFileReceiver receiver = new DummyFileReceiver(baseDir.toFile()); + try { + receiver.handshake(); + receiver.createWritingFile("normal.tsfile", true); + receiver.writeToCurrentWritingFile(new byte[] {1, 2, 3}); + + final File oldReceiverDir = receiver.getCurrentReceiverDir(); + Assert.assertNotNull(receiver.getCurrentWritingFile()); + Assert.assertNotNull(receiver.getCurrentWritingFileWriter()); + + receiver.handshake(); + + Assert.assertFalse(oldReceiverDir.exists()); + Assert.assertNull(receiver.getCurrentWritingFile()); + Assert.assertNull(receiver.getCurrentWritingFileWriter()); + Assert.assertNotEquals( + oldReceiverDir.getAbsolutePath(), receiver.getCurrentReceiverDir().getAbsolutePath()); + } finally { + receiver.handleExit(); + } + } + + @Test + public void testSealFileV1FailureDeletesTransferredFile() throws Exception { + final Path baseDir = Files.createTempDirectory("iotdb-file-receiver-test"); + final DummyFileReceiver receiver = new DummyFileReceiver(baseDir.toFile()); + try { + receiver.createWritingFile("normal.tsfile", true); + receiver.writeToCurrentWritingFile(new byte[] {1, 2, 3}); + receiver.setLoadFileV1Status( + new TSStatus(TSStatusCode.PIPE_TRANSFER_FILE_ERROR.getStatusCode())); + + final File transferredFile = receiver.getWritingFileInBaseDir("normal.tsfile"); + final TPipeTransferResp response = receiver.sealFileV1("normal.tsfile", 3L); + + Assert.assertEquals( + TSStatusCode.PIPE_TRANSFER_FILE_ERROR.getStatusCode(), response.getStatus().getCode()); + Assert.assertFalse(transferredFile.exists()); + Assert.assertNull(receiver.getCurrentWritingFile()); + Assert.assertNull(receiver.getCurrentWritingFileWriter()); + } finally { + receiver.handleExit(); + } + } + private static class DummyFileReceiver extends IoTDBFileReceiver { + private final File receiverFileBaseDir; + private TSStatus loadFileV1Status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + DummyFileReceiver(final File baseDir) { + receiverFileBaseDir = baseDir; receiverFileDirWithIdSuffix.set(baseDir); } @@ -73,13 +152,55 @@ void createWritingFile(final String fileName, final boolean isSingleFile) throws updateWritingFileIfNeeded(fileName, isSingleFile); } + void handshake() throws IOException { + handleTransferHandshakeV1( + DummyHandshakeReq.toTPipeTransferReq( + CommonDescriptor.getInstance().getConfig().getTimestampPrecision())); + } + + void writeToCurrentWritingFile(final byte[] bytes) throws Exception { + getCurrentWritingFileWriter().write(bytes); + } + + void setLoadFileV1Status(final TSStatus status) { + loadFileV1Status = status; + } + + TPipeTransferResp sealFileV1(final String fileName, final long fileLength) throws IOException { + return handleTransferFileSealV1(DummyFileSealReqV1.toTPipeTransferReq(fileName, fileLength)); + } + + TPipeTransferResp sealFiles(final List fileNames, final List fileLengths) + throws IOException { + return handleTransferFileSealV2( + DummyFileSealReqV2.toTPipeTransferReq(fileNames, fileLengths, Collections.emptyMap())); + } + File getWritingFileInBaseDir(final String fileName) { return receiverFileDirWithIdSuffix.get().toPath().resolve(fileName).toFile(); } + File getCurrentReceiverDir() { + return receiverFileDirWithIdSuffix.get(); + } + + File getCurrentWritingFile() throws Exception { + return (File) getField("writingFile").get(this); + } + + RandomAccessFile getCurrentWritingFileWriter() throws Exception { + return (RandomAccessFile) getField("writingFileWriter").get(this); + } + + private Field getField(final String fieldName) throws NoSuchFieldException { + final Field field = IoTDBFileReceiver.class.getDeclaredField(fieldName); + field.setAccessible(true); + return field; + } + @Override protected String getReceiverFileBaseDir() { - return receiverFileDirWithIdSuffix.get().getAbsolutePath(); + return receiverFileBaseDir.getAbsolutePath(); } @Override @@ -105,7 +226,7 @@ protected TSStatus login() { @Override protected TSStatus loadFileV1( final PipeTransferFileSealReqV1 req, final String fileAbsolutePath) { - return new TSStatus(200); + return loadFileV1Status; } @Override @@ -125,4 +246,49 @@ public TPipeTransferResp receive(TPipeTransferReq req) { return null; } } + + private static class DummyHandshakeReq extends PipeTransferHandshakeV1Req { + + static DummyHandshakeReq toTPipeTransferReq(final String timestampPrecision) + throws IOException { + return (DummyHandshakeReq) + new DummyHandshakeReq().convertToTPipeTransferReq(timestampPrecision); + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.HANDSHAKE_DATANODE_V1; + } + } + + private static class DummyFileSealReqV1 extends PipeTransferFileSealReqV1 { + + static DummyFileSealReqV1 toTPipeTransferReq(final String fileName, final long fileLength) + throws IOException { + return (DummyFileSealReqV1) + new DummyFileSealReqV1().convertToTPipeTransferReq(fileName, fileLength); + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.TRANSFER_TS_FILE_SEAL; + } + } + + private static class DummyFileSealReqV2 extends PipeTransferFileSealReqV2 { + + static DummyFileSealReqV2 toTPipeTransferReq( + final List fileNames, + final List fileLengths, + final java.util.Map parameters) + throws IOException { + return (DummyFileSealReqV2) + new DummyFileSealReqV2().convertToTPipeTransferReq(fileNames, fileLengths, parameters); + } + + @Override + protected PipeRequestType getPlanType() { + return PipeRequestType.TRANSFER_SCHEMA_SNAPSHOT_SEAL; + } + } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java new file mode 100644 index 0000000000000..290ce3979807d --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferSliceReqBuilderTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.pipe.sink.payload.thrift.common; + +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.IoTDBSinkRequestVersion; +import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferSliceReq; +import org.apache.iotdb.service.rpc.thrift.TPipeTransferReq; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.nio.ByteBuffer; + +public class PipeTransferSliceReqBuilderTest { + + private final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); + + private int originalRequestSliceThresholdBytes; + + @Before + public void setUp() { + originalRequestSliceThresholdBytes = commonConfig.getPipeSinkRequestSliceThresholdBytes(); + commonConfig.setPipeSinkRequestSliceThresholdBytes(4); + } + + @After + public void tearDown() { + commonConfig.setPipeSinkRequestSliceThresholdBytes(originalRequestSliceThresholdBytes); + } + + @Test + public void testBuildSliceReq() throws Exception { + final TPipeTransferReq req = createReq(IoTDBSinkRequestVersion.VERSION_1.getVersion(), 10); + final int bodySizeLimit = PipeTransferSliceReqBuilder.getBodySizeLimit(); + + Assert.assertTrue(PipeTransferSliceReqBuilder.shouldSlice(req, bodySizeLimit)); + Assert.assertEquals(3, PipeTransferSliceReqBuilder.getSliceCount(req, bodySizeLimit)); + + final PipeTransferSliceReq firstSlice = + PipeTransferSliceReqBuilder.buildSliceReq(req, 123, 0, 3, bodySizeLimit); + final PipeTransferSliceReq secondSlice = + PipeTransferSliceReqBuilder.buildSliceReq(req, 123, 1, 3, bodySizeLimit); + final PipeTransferSliceReq thirdSlice = + PipeTransferSliceReqBuilder.buildSliceReq(req, 123, 2, 3, bodySizeLimit); + + Assert.assertArrayEquals(new byte[] {0, 1, 2, 3}, firstSlice.getSliceBody()); + Assert.assertArrayEquals(new byte[] {4, 5, 6, 7}, secondSlice.getSliceBody()); + Assert.assertArrayEquals(new byte[] {8, 9}, thirdSlice.getSliceBody()); + Assert.assertEquals(0, firstSlice.getSliceIndex()); + Assert.assertEquals(1, secondSlice.getSliceIndex()); + Assert.assertEquals(2, thirdSlice.getSliceIndex()); + Assert.assertEquals(3, firstSlice.getSliceCount()); + Assert.assertEquals(req.getType(), firstSlice.getOriginReqType()); + Assert.assertEquals(10, firstSlice.getOriginBodySize()); + } + + @Test + public void testShouldSliceOnlyForVersion1RequestsAboveThreshold() { + final int bodySizeLimit = PipeTransferSliceReqBuilder.getBodySizeLimit(); + + Assert.assertFalse( + PipeTransferSliceReqBuilder.shouldSlice( + createReq(IoTDBSinkRequestVersion.VERSION_1.getVersion(), 3), bodySizeLimit)); + Assert.assertFalse( + PipeTransferSliceReqBuilder.shouldSlice( + createReq((byte) (IoTDBSinkRequestVersion.VERSION_1.getVersion() + 1), 10), + bodySizeLimit)); + Assert.assertTrue( + PipeTransferSliceReqBuilder.shouldSlice( + createReq(IoTDBSinkRequestVersion.VERSION_1.getVersion(), 4), bodySizeLimit)); + } + + private static TPipeTransferReq createReq(final byte version, final int bodySize) { + final byte[] body = new byte[bodySize]; + for (int i = 0; i < body.length; ++i) { + body[i] = (byte) i; + } + + final TPipeTransferReq req = new TPipeTransferReq(); + req.version = version; + req.type = (short) 123; + req.body = ByteBuffer.wrap(body); + return req; + } +} diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/consumer/ConsumerGroupDeSerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/consumer/ConsumerGroupDeSerTest.java index d41b3706cd1f2..9ef7191ba6683 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/consumer/ConsumerGroupDeSerTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/consumer/ConsumerGroupDeSerTest.java @@ -69,4 +69,24 @@ public void test() throws IOException { consumerGroupMeta.getConsumerGroupId(), consumerGroupMeta2.getConsumerGroupId()); Assert.assertEquals(consumerGroupMeta.getCreationTime(), consumerGroupMeta2.getCreationTime()); } + + @Test + public void testDeepCopyShouldNotShareSubscribedConsumerSets() { + Map consumerAttributes = new HashMap<>(); + consumerAttributes.put("username", "user"); + consumerAttributes.put("password", "password"); + + ConsumerGroupMeta consumerGroupMeta = + new ConsumerGroupMeta( + "test_consumer_group", 1, new ConsumerMeta("test_consumer1", 1, consumerAttributes)); + consumerGroupMeta.addSubscription("test_consumer1", Collections.singleton("test_topic")); + + ConsumerGroupMeta copiedConsumerGroupMeta = consumerGroupMeta.deepCopy(); + copiedConsumerGroupMeta.removeSubscription( + "test_consumer1", Collections.singleton("test_topic")); + + Assert.assertTrue( + consumerGroupMeta.getConsumersSubscribingTopic("test_topic").contains("test_consumer1")); + Assert.assertTrue(copiedConsumerGroupMeta.getConsumersSubscribingTopic("test_topic").isEmpty()); + } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/topic/TopicDeSerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/topic/TopicDeSerTest.java index 4973f06d27cf2..d9c280e14938c 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/topic/TopicDeSerTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/subscription/topic/TopicDeSerTest.java @@ -54,4 +54,15 @@ public void test() throws IOException { Assert.assertEquals( topicMeta.getSubscribedConsumerGroupIds(), topicMeta2.getSubscribedConsumerGroupIds()); } + + @Test + public void testGenerateExtractorAttributesWithEncryptedPassword() { + final TopicMeta topicMeta = new TopicMeta("test_topic", 1, new HashMap<>()); + + final Map extractorAttributes = + topicMeta.generateExtractorAttributes("test_user", "encrypted-password"); + + Assert.assertEquals("test_user", extractorAttributes.get("username")); + Assert.assertEquals("encrypted-password", extractorAttributes.get("password")); + } } diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/FileUtilsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/FileUtilsTest.java index c5384e07fadae..f4b1ed962bc8e 100644 --- a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/FileUtilsTest.java +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/FileUtilsTest.java @@ -26,6 +26,7 @@ import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -61,6 +62,13 @@ public void testFileUtils() throws WriteProcessException, IOException { FileUtils.moveFileWithMD5Check(tstFile, targetDir); } + @Test + public void testGetIllegalError4DirectoryRejectsEmptyPath() { + Assert.assertNotNull(FileUtils.getIllegalError4Directory(null)); + Assert.assertNotNull(FileUtils.getIllegalError4Directory("")); + Assert.assertNull(FileUtils.getIllegalError4Directory("valid_dir")); + } + private void generateFile(File tsfile) throws WriteProcessException, IOException { try (TsFileWriter writer = new TsFileWriter(tsfile)) { writer.registerAlignedTimeseries( diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/WindowsOSUtilsTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/WindowsOSUtilsTest.java new file mode 100644 index 0000000000000..f088ba80730f2 --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/utils/WindowsOSUtilsTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.utils; + +import org.apache.commons.lang3.SystemUtils; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.iotdb.commons.utils.WindowsOSUtils.isLegalPathSegment4Windows; + +public class WindowsOSUtilsTest { + @Test + public void testIllegalDetection() { + if (!SystemUtils.IS_OS_WINDOWS) { + return; + } + Assert.assertTrue(isLegalPathSegment4Windows("abc")); + Assert.assertTrue(isLegalPathSegment4Windows(".A!")); + + Assert.assertFalse(isLegalPathSegment4Windows("C.")); + Assert.assertFalse(isLegalPathSegment4Windows("a:b<|")); + Assert.assertFalse(isLegalPathSegment4Windows("COM1")); + Assert.assertFalse(isLegalPathSegment4Windows("com1")); + Assert.assertFalse(isLegalPathSegment4Windows("COM1.txt")); + Assert.assertFalse(isLegalPathSegment4Windows("NUL.log")); + Assert.assertFalse(isLegalPathSegment4Windows("LPT9.tmp")); + Assert.assertFalse(isLegalPathSegment4Windows("COM\u00B9")); + Assert.assertFalse(isLegalPathSegment4Windows("LPT\u00B2.log")); + Assert.assertFalse(isLegalPathSegment4Windows("name\tpart")); + Assert.assertFalse(isLegalPathSegment4Windows("name" + Character.toString((char) 0) + "part")); + + Assert.assertTrue(isLegalPathSegment4Windows("COM0")); + Assert.assertTrue(isLegalPathSegment4Windows("LPT0")); + Assert.assertTrue(isLegalPathSegment4Windows("COM\u00B4")); + } +} diff --git a/library-udf/src/assembly/tools/register-UDF.bat b/library-udf/src/assembly/tools/register-UDF.bat index c6683bc96ac5e..f30db5d2a3a7d 100644 --- a/library-udf/src/assembly/tools/register-UDF.bat +++ b/library-udf/src/assembly/tools/register-UDF.bat @@ -102,6 +102,7 @@ call ../sbin/start-cli.bat -h %host% -p %rpcPort% -u %user% -pw %pass% -e "creat @REM Machine Learning call ../sbin/start-cli.bat -h %host% -p %rpcPort% -u %user% -pw %pass% -e "create function ar as 'org.apache.iotdb.library.dlearn.UDTFAR'" +call ../sbin/start-cli.bat -h %host% -p %rpcPort% -u %user% -pw %pass% -e "create function cluster as 'org.apache.iotdb.library.dlearn.UDTFCluster'" @REM Match call ../sbin/start-cli.bat -h %host% -p %rpcPort% -u %user% -pw %pass% -e "create function pattern_match as 'org.apache.iotdb.library.match.UDAFPatternMatch'" diff --git a/library-udf/src/assembly/tools/register-UDF.sh b/library-udf/src/assembly/tools/register-UDF.sh index 16ab59f143baa..faaa4df68b5d1 100755 --- a/library-udf/src/assembly/tools/register-UDF.sh +++ b/library-udf/src/assembly/tools/register-UDF.sh @@ -102,6 +102,7 @@ pass=root # Machine Learning ../sbin/start-cli.sh -h $host -p $rpcPort -u $user -pw $pass -e "create function ar as 'org.apache.iotdb.library.dlearn.UDTFAR'" +../sbin/start-cli.sh -h $host -p $rpcPort -u $user -pw $pass -e "create function cluster as 'org.apache.iotdb.library.dlearn.UDTFCluster'" # Match ../sbin/start-cli.sh -h $host -p $rpcPort -u $user -pw $pass -e "create function pattern_match as 'org.apache.iotdb.library.match.UDAFPatternMatch'" diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFCluster.java b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFCluster.java new file mode 100644 index 0000000000000..f64f9747992bc --- /dev/null +++ b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFCluster.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.library.dlearn; + +import org.apache.iotdb.library.dlearn.util.cluster.KMeans; +import org.apache.iotdb.library.dlearn.util.cluster.KShape; +import org.apache.iotdb.library.dlearn.util.cluster.MedoidShape; +import org.apache.iotdb.library.util.Util; +import org.apache.iotdb.udf.api.UDTF; +import org.apache.iotdb.udf.api.access.Row; +import org.apache.iotdb.udf.api.collector.PointCollector; +import org.apache.iotdb.udf.api.customizer.config.UDTFConfigurations; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameterValidator; +import org.apache.iotdb.udf.api.customizer.parameter.UDFParameters; +import org.apache.iotdb.udf.api.customizer.strategy.RowByRowAccessStrategy; +import org.apache.iotdb.udf.api.exception.UDFException; +import org.apache.iotdb.udf.api.type.Type; + +import java.util.ArrayList; +import java.util.List; + +/** + * Clusters a time series by partitioning it into non-overlapping subsequences of length l. + * Parameters: l, k, method (default kmeans), norm, maxiter, output; medoidshape also uses + * sample_rate (greedy sampling ratio; use 1 when the window count is small). Requires at least k + * windows. + */ +public class UDTFCluster implements UDTF { + + private static final String METHOD_KMEANS = "kmeans"; + private static final String METHOD_KSHAPE = "kshape"; + private static final String METHOD_MEDOIDSHAPE = "medoidshape"; + + private static final String OUTPUT_LABEL = "label"; + private static final String OUTPUT_CENTROID = "centroid"; + + private static final int DEFAULT_MAX_ITER = 200; + private static final double DEFAULT_SAMPLE_RATE = 0.3; + private static final String DEFAULT_METHOD = METHOD_KMEANS; + + private int l; + private int k; + private String method; + private boolean norm; + private int maxIter; + private String output; + private double sampleRate; + + private final List timestamps = new ArrayList<>(); + private final List values = new ArrayList<>(); + + @Override + public void validate(UDFParameterValidator validator) throws Exception { + validator + .validateInputSeriesNumber(1) + .validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE) + .validate( + x -> (int) x > 0, + "Parameter l must be a positive integer.", + validator.getParameters().getInt("l")) + .validate( + x -> (int) x >= 2, + "Parameter k must be at least 2.", + validator.getParameters().getInt("k")) + .validate( + x -> { + String m = ((String) x).toLowerCase(); + return METHOD_KMEANS.equals(m) + || METHOD_KSHAPE.equals(m) + || METHOD_MEDOIDSHAPE.equals(m); + }, + "Parameter method must be one of: kmeans, kshape, medoidshape.", + validator.getParameters().getStringOrDefault("method", DEFAULT_METHOD)) + .validate( + x -> (int) x >= 1, + "Parameter maxiter must be a positive integer.", + validator.getParameters().getIntOrDefault("maxiter", DEFAULT_MAX_ITER)) + .validate( + x -> { + String o = ((String) x).toLowerCase(); + return OUTPUT_LABEL.equals(o) || OUTPUT_CENTROID.equals(o); + }, + "Parameter output must be label or centroid.", + validator.getParameters().getStringOrDefault("output", OUTPUT_LABEL)) + .validate( + x -> { + double d = ((Number) x).doubleValue(); + return d > 0 && d <= 1.0; + }, + "Parameter sample_rate must be in (0, 1].", + validator.getParameters().getDoubleOrDefault("sample_rate", DEFAULT_SAMPLE_RATE)); + } + + @Override + public void beforeStart(UDFParameters parameters, UDTFConfigurations configurations) + throws Exception { + this.output = parameters.getStringOrDefault("output", OUTPUT_LABEL).toLowerCase(); + if (OUTPUT_CENTROID.equals(output)) { + configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.DOUBLE); + } else { + configurations.setAccessStrategy(new RowByRowAccessStrategy()).setOutputDataType(Type.INT32); + } + this.l = parameters.getInt("l"); + this.k = parameters.getInt("k"); + this.method = parameters.getStringOrDefault("method", DEFAULT_METHOD).toLowerCase(); + this.norm = parameters.getBooleanOrDefault("norm", true); + this.maxIter = parameters.getIntOrDefault("maxiter", DEFAULT_MAX_ITER); + this.sampleRate = parameters.getDoubleOrDefault("sample_rate", DEFAULT_SAMPLE_RATE); + timestamps.clear(); + values.clear(); + } + + @Override + public void transform(Row row, PointCollector collector) throws Exception { + if (!row.isNull(0)) { + timestamps.add(row.getTime()); + values.add(Util.getValueAsDouble(row)); + } + } + + @Override + public void terminate(PointCollector collector) throws Exception { + int n = values.size(); + if (n < l) { + throw new UDFException( + "Time series length must be at least l; got " + n + " points, l=" + l + "."); + } + int numWindows = n / l; + if (numWindows < k) { + throw new UDFException( + "Not enough non-overlapping windows: got " + + numWindows + + " windows, need at least k=" + + k + + "."); + } + + double[][] windows = new double[numWindows][l]; + long[] windowStartTime = new long[numWindows]; + for (int w = 0; w < numWindows; w++) { + windowStartTime[w] = timestamps.get(w * l); + for (int j = 0; j < l; j++) { + windows[w][j] = values.get(w * l + j); + } + } + + if (OUTPUT_LABEL.equals(output)) { + int[] labels; + if (METHOD_KMEANS.equals(method)) { + KMeans km = new KMeans(); + km.fit(windows, k, norm, maxIter); + labels = km.getLabels(); + } else if (METHOD_KSHAPE.equals(method)) { + KShape ks = new KShape(); + ks.fit(windows, k, norm, maxIter); + labels = ks.getLabels(); + } else if (METHOD_MEDOIDSHAPE.equals(method)) { + MedoidShape ms = new MedoidShape(); + ms.setSampleRate(sampleRate); + ms.fit(windows, k, norm, maxIter); + labels = ms.getLabels(); + } else { + throw new UDFException("Unsupported method: " + method); + } + for (int w = 0; w < numWindows; w++) { + collector.putInt(windowStartTime[w], labels[w]); + } + } else { + double[][] centroids; + if (METHOD_KMEANS.equals(method)) { + KMeans km = new KMeans(); + km.fit(windows, k, norm, maxIter); + centroids = km.getCentroids(); + } else if (METHOD_KSHAPE.equals(method)) { + KShape ks = new KShape(); + ks.fit(windows, k, norm, maxIter); + centroids = ks.getCentroids(); + } else if (METHOD_MEDOIDSHAPE.equals(method)) { + MedoidShape ms = new MedoidShape(); + ms.setSampleRate(sampleRate); + ms.fit(windows, k, norm, maxIter); + centroids = ms.getCentroids(); + } else { + throw new UDFException("Unsupported method: " + method); + } + emitConcatenatedCentroids(collector, centroids); + } + } + + private static void emitConcatenatedCentroids(PointCollector collector, double[][] centroids) + throws Exception { + long t = 0L; + for (double[] row : centroids) { + for (double v : row) { + collector.putDouble(t++, v); + } + } + } +} diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/util/cluster/ClusterUtils.java b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/util/cluster/ClusterUtils.java new file mode 100644 index 0000000000000..a646adb2bed30 --- /dev/null +++ b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/util/cluster/ClusterUtils.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.library.dlearn.util.cluster; + +import org.apache.commons.math3.complex.Complex; +import org.apache.commons.math3.transform.DftNormalization; +import org.apache.commons.math3.transform.FastFourierTransformer; +import org.apache.commons.math3.transform.TransformType; + +import java.util.Arrays; + +/** + * Subsequence z-normalize, Euclidean distance, and FFT-based NCC / SBD (shared by KShape and + * MedoidShape). + */ +public final class ClusterUtils { + + public static final double EPS = 1e-9; + + private static final FastFourierTransformer FFT = + new FastFourierTransformer(DftNormalization.STANDARD); + + private ClusterUtils() {} + + public static double[] maybeZNormalize(double[] a, boolean normalize) { + if (normalize) { + return zNormalize(a); + } + return Arrays.copyOf(a, a.length); + } + + public static double[] zNormalize(double[] a) { + int n = a.length; + double sum = 0.0; + for (double v : a) { + sum += v; + } + double mean = sum / n; + double var = 0.0; + for (double v : a) { + double d = v - mean; + var += d * d; + } + var /= n; + double std = Math.sqrt(Math.max(var, 0.0)); + double[] z = new double[n]; + if (std < EPS) { + return z; + } + for (int i = 0; i < n; i++) { + z[i] = (a[i] - mean) / std; + } + return z; + } + + public static double squaredEuclidean(double[] a, double[] b) { + double s = 0.0; + for (int i = 0; i < a.length; i++) { + double d = a[i] - b[i]; + s += d * d; + } + return s; + } + + public static int findLargestCluster(int[] counts) { + int best = 0; + for (int i = 1; i < counts.length; i++) { + if (counts[i] > counts[best]) { + best = i; + } + } + return best; + } + + /** + * Maximum over the normalized cross-correlation sequence (FFT); used for SBD and MedoidShape + * objective. + */ + public static double maxNcc(double[] x, double[] y) { + double[] cc = nccFft(x, y); + double max = Double.NEGATIVE_INFINITY; + for (double v : cc) { + if (v > max) { + max = v; + } + } + return max; + } + + /** SBD: 1 − max NCC (consistent with the NCC-based definition in k-Shape / FastKShape). */ + public static double shapeDistance(double[] x, double[] y) { + return 1.0 - maxNcc(x, y); + } + + public static double symmetricSbd(double[] a, double[] b) { + return 0.5 * (shapeDistance(a, b) + shapeDistance(b, a)); + } + + private static double[] nccFft(double[] x, double[] y) { + int xLen = x.length; + double den = l2Norm(x) * l2Norm(y); + if (den < 1e-9) { + den = Double.POSITIVE_INFINITY; + } + int fftSize = 1 << (32 - Integer.numberOfLeadingZeros(2 * xLen - 1)); + + Complex[] cx = new Complex[fftSize]; + Complex[] cy = new Complex[fftSize]; + for (int i = 0; i < fftSize; i++) { + cx[i] = new Complex(i < xLen ? x[i] : 0.0, 0.0); + cy[i] = new Complex(i < xLen ? y[i] : 0.0, 0.0); + } + Complex[] fx = FFT.transform(cx, TransformType.FORWARD); + Complex[] fy = FFT.transform(cy, TransformType.FORWARD); + Complex[] prod = new Complex[fftSize]; + for (int i = 0; i < fftSize; i++) { + prod[i] = fx[i].multiply(fy[i].conjugate()); + } + Complex[] ccFull = FFT.transform(prod, TransformType.INVERSE); + + double[] ccPacked = new double[2 * xLen - 1]; + int p = 0; + for (int i = fftSize - (xLen - 1); i < fftSize; i++) { + ccPacked[p++] = ccFull[i].getReal() / den; + } + for (int i = 0; i < xLen; i++) { + ccPacked[p++] = ccFull[i].getReal() / den; + } + return ccPacked; + } + + private static double l2Norm(double[] v) { + double s = 0.0; + for (double x : v) { + s += x * x; + } + return Math.sqrt(s); + } +} diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/util/cluster/KMeans.java b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/util/cluster/KMeans.java new file mode 100644 index 0000000000000..37c44eade96bf --- /dev/null +++ b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/util/cluster/KMeans.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.library.dlearn.util.cluster; + +import java.util.Arrays; + +/** + * Univariate subsequence k-means (Lloyd); optionally z-normalize, then cluster in Euclidean space. + */ +public class KMeans { + + private double[][] centroids; + private int[] labels; + + public void fit(double[][] samples, int k, boolean normalize, int maxIterations) { + validate(samples, k, maxIterations); + int n = samples.length; + int dim = samples[0].length; + + double[][] z = new double[n][dim]; + for (int i = 0; i < n; i++) { + z[i] = ClusterUtils.maybeZNormalize(samples[i], normalize); + } + + centroids = new double[k][dim]; + for (int c = 0; c < k; c++) { + System.arraycopy(z[c], 0, centroids[c], 0, dim); + } + + labels = new int[n]; + Arrays.fill(labels, -1); + + for (int iter = 0; iter < maxIterations; iter++) { + double[][] prevCentroids = new double[k][dim]; + for (int c = 0; c < k; c++) { + System.arraycopy(centroids[c], 0, prevCentroids[c], 0, dim); + } + + boolean changed = false; + for (int i = 0; i < n; i++) { + int best = 0; + double bestDist = Double.POSITIVE_INFINITY; + for (int c = 0; c < k; c++) { + double d = ClusterUtils.squaredEuclidean(z[i], centroids[c]); + if (d < bestDist) { + bestDist = d; + best = c; + } + } + if (labels[i] != best) { + labels[i] = best; + changed = true; + } + } + + double[][] newCentroids = new double[k][dim]; + int[] counts = new int[k]; + for (int i = 0; i < n; i++) { + int c = labels[i]; + counts[c]++; + for (int d = 0; d < dim; d++) { + newCentroids[c][d] += z[i][d]; + } + } + for (int c = 0; c < k; c++) { + if (counts[c] == 0) { + int donor = ClusterUtils.findLargestCluster(counts); + System.arraycopy(prevCentroids[donor], 0, centroids[c], 0, dim); + for (int d = 0; d < dim; d++) { + centroids[c][d] += (d == 0 ? 1e-4 : -1e-4); + } + } else { + for (int d = 0; d < dim; d++) { + centroids[c][d] = newCentroids[c][d] / counts[c]; + } + } + } + + if (!changed) { + break; + } + } + } + + public double[][] getCentroids() { + return centroids; + } + + public int[] getLabels() { + return labels; + } + + private static void validate(double[][] samples, int k, int maxIterations) { + if (samples == null || samples.length == 0) { + throw new IllegalArgumentException("samples must be non-empty."); + } + if (k < 2 || k > samples.length) { + throw new IllegalArgumentException("k must satisfy 2 <= k <= samples.length."); + } + if (maxIterations < 1) { + throw new IllegalArgumentException("maxIterations must be at least 1."); + } + int dim = samples[0].length; + if (dim == 0) { + throw new IllegalArgumentException("sample dimension must be positive."); + } + for (double[] row : samples) { + if (row == null || row.length != dim) { + throw new IllegalArgumentException("All samples must have the same length."); + } + } + } +} diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/util/cluster/KShape.java b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/util/cluster/KShape.java new file mode 100644 index 0000000000000..315e1c51d2f06 --- /dev/null +++ b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/util/cluster/KShape.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.library.dlearn.util.cluster; + +import org.apache.commons.math3.linear.MatrixUtils; +import org.apache.commons.math3.linear.RealMatrix; +import org.apache.commons.math3.linear.RealVector; +import org.apache.commons.math3.linear.SingularValueDecomposition; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * k-Shape: assignment uses {@link ClusterUtils#shapeDistance} (SBD = 1 − max NCC); centroids are + * the first right singular vector of the cluster matrix from SVD, sign correction, then z-normalize + * or L2 normalization. + */ +public class KShape { + + private double[][] centroids; + private int[] labels; + + public void fit(double[][] samples, int k, boolean normalize, int maxIterations) { + validate(samples, k, maxIterations); + int n = samples.length; + int dim = samples[0].length; + + double[][] z = new double[n][dim]; + for (int i = 0; i < n; i++) { + z[i] = ClusterUtils.maybeZNormalize(samples[i], normalize); + } + + centroids = new double[k][dim]; + for (int c = 0; c < k; c++) { + System.arraycopy(z[c], 0, centroids[c], 0, dim); + } + + labels = new int[n]; + Arrays.fill(labels, -1); + + for (int iter = 0; iter < maxIterations; iter++) { + double[][] prevCentroids = new double[k][dim]; + for (int c = 0; c < k; c++) { + System.arraycopy(centroids[c], 0, prevCentroids[c], 0, dim); + } + + boolean changed = false; + for (int i = 0; i < n; i++) { + int best = 0; + double bestDist = Double.POSITIVE_INFINITY; + for (int c = 0; c < k; c++) { + double d = ClusterUtils.shapeDistance(z[i], centroids[c]); + if (d < bestDist) { + bestDist = d; + best = c; + } + } + if (labels[i] != best) { + labels[i] = best; + changed = true; + } + } + + int[] counts = new int[k]; + @SuppressWarnings("unchecked") + List[] byCluster = new List[k]; + for (int c = 0; c < k; c++) { + byCluster[c] = new ArrayList<>(); + } + for (int i = 0; i < n; i++) { + int c = labels[i]; + counts[c]++; + byCluster[c].add(z[i]); + } + + for (int c = 0; c < k; c++) { + if (counts[c] == 0) { + int donor = ClusterUtils.findLargestCluster(counts); + System.arraycopy(prevCentroids[donor], 0, centroids[c], 0, dim); + } else { + List members = byCluster[c]; + double[][] mat = new double[members.size()][dim]; + for (int i = 0; i < members.size(); i++) { + mat[i] = members.get(i); + } + centroids[c] = centroidFromSvd(mat, normalize); + } + } + + if (!changed) { + break; + } + } + } + + public double[][] getCentroids() { + return centroids; + } + + public int[] getLabels() { + return labels; + } + + private static double[] centroidFromSvd(double[][] members, boolean zNormalizeCentroid) { + int m = members.length; + int dim = members[0].length; + if (m == 1) { + double[] u = Arrays.copyOf(members[0], dim); + return zNormalizeCentroid ? ClusterUtils.zNormalize(u) : l2Unit(u); + } + RealMatrix y = MatrixUtils.createRealMatrix(members); + SingularValueDecomposition svd = new SingularValueDecomposition(y); + RealMatrix v = svd.getV(); + RealVector col0 = v.getColumnVector(0); + double[] r = col0.toArray(); + double sumDot = 0.0; + for (double[] row : members) { + sumDot += dot(row, r); + } + if (sumDot < 0) { + for (int i = 0; i < r.length; i++) { + r[i] = -r[i]; + } + } + return zNormalizeCentroid ? ClusterUtils.zNormalize(r) : l2Unit(r); + } + + private static double dot(double[] a, double[] b) { + double s = 0.0; + for (int i = 0; i < a.length; i++) { + s += a[i] * b[i]; + } + return s; + } + + private static double[] l2Unit(double[] v) { + double s = 0.0; + for (double x : v) { + s += x * x; + } + s = Math.sqrt(s); + if (s < ClusterUtils.EPS) { + return new double[v.length]; + } + double[] o = new double[v.length]; + for (int i = 0; i < v.length; i++) { + o[i] = v[i] / s; + } + return o; + } + + private static void validate(double[][] samples, int k, int maxIterations) { + if (samples == null || samples.length == 0) { + throw new IllegalArgumentException("samples must be non-empty."); + } + if (k < 2 || k > samples.length) { + throw new IllegalArgumentException("k must satisfy 2 <= k <= samples.length."); + } + if (maxIterations < 1) { + throw new IllegalArgumentException("maxIterations must be at least 1."); + } + int dim = samples[0].length; + if (dim == 0) { + throw new IllegalArgumentException("sample dimension must be positive."); + } + for (double[] row : samples) { + if (row == null || row.length != dim) { + throw new IllegalArgumentException("All samples must have the same length."); + } + } + } +} diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dlearn/util/cluster/MedoidShape.java b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/util/cluster/MedoidShape.java new file mode 100644 index 0000000000000..31f9d8c8df1a0 --- /dev/null +++ b/library-udf/src/main/java/org/apache/iotdb/library/dlearn/util/cluster/MedoidShape.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.library.dlearn.util.cluster; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Random; +import java.util.Set; + +/** + * Coarse clustering: {@link KMeans} uses {@code min(2k, n)} clusters (n = number of windows); + * greedy fastKShape picks k representatives; both labels and the objective use {@link + * ClusterUtils#maxNcc}. + */ +public class MedoidShape { + + private double sampleRate = 0.3; + private Random random = new Random(); + + /** Overrides the RNG used for greedy sampling (default is {@link Random#Random()}). */ + public void setRandom(Random random) { + this.random = Objects.requireNonNull(random); + } + + private double[][] centroids; + private int[] labels; + + public void setSampleRate(double sampleRate) { + if (sampleRate <= 0 || sampleRate > 1.0) { + throw new IllegalArgumentException("sampleRate must be in (0, 1]."); + } + this.sampleRate = sampleRate; + } + + public double getSampleRate() { + return sampleRate; + } + + public void fit(double[][] samples, int k, boolean normalize, int maxIterations) { + validate(samples, k, maxIterations); + int n = samples.length; + int dim = samples[0].length; + + int coarseK = Math.min(2 * k, n); + + double[][] x = new double[n][dim]; + for (int i = 0; i < n; i++) { + x[i] = ClusterUtils.maybeZNormalize(samples[i], normalize); + } + + KMeans coarse = new KMeans(); + coarse.fit(x, coarseK, false, maxIterations); + double[][] euclideanCentroids = coarse.getCentroids(); + int[] kmLabels = coarse.getLabels(); + long[] clusterSize = new long[coarseK]; + for (int lb : kmLabels) { + clusterSize[lb]++; + } + + centroids = fastKShape(x, k, sampleRate, dim, euclideanCentroids, clusterSize, random); + + labels = new int[n]; + for (int i = 0; i < n; i++) { + double maxNcc = Double.NEGATIVE_INFINITY; + int label = -1; + for (int j = 0; j < k; j++) { + double cur = ClusterUtils.maxNcc(x[i], centroids[j]); + if (cur > maxNcc) { + maxNcc = cur; + label = j; + } + } + labels[i] = label; + } + } + + public double[][] getCentroids() { + return centroids; + } + + public int[] getLabels() { + return labels; + } + + private static double[][] fastKShape( + double[][] x, + int k, + double r, + int dim, + double[][] euclideanCentroids, + long[] clusterSize, + Random rnd) { + int n = x.length; + if (n <= k) { + double[][] out = new double[k][dim]; + for (int i = 0; i < n; i++) { + out[i] = Arrays.copyOf(x[i], dim); + } + for (int i = n; i < k; i++) { + out[i] = Arrays.copyOf(x[n - 1], dim); + } + return out; + } + + List picked = new ArrayList<>(); + Set coresetIdx = new HashSet<>(); + + for (int round = 0; round < k; round++) { + List pool = new ArrayList<>(); + for (int i = 0; i < n; i++) { + if (!coresetIdx.contains(i)) { + pool.add(i); + } + } + if (pool.isEmpty()) { + throw new IllegalStateException("fastKShape: empty candidate pool."); + } + int sampleCount = Math.max(1, (int) (r * n)); + sampleCount = Math.min(sampleCount, pool.size()); + Collections.shuffle(pool, rnd); + List sampleIdx = pool.subList(0, sampleCount); + + double maxDelta = Double.NEGATIVE_INFINITY; + double[] bestSeg = null; + int bestIdx = -1; + + for (int idx : sampleIdx) { + double[] seq = x[idx]; + picked.add(seq); + double delta = evaluateAim(picked, euclideanCentroids, clusterSize); + picked.remove(picked.size() - 1); + if (delta > maxDelta) { + maxDelta = delta; + bestSeg = Arrays.copyOf(seq, dim); + bestIdx = idx; + } + } + + if (bestSeg == null) { + throw new IllegalStateException("fastKShape: no candidate selected."); + } + picked.add(bestSeg); + coresetIdx.add(bestIdx); + } + + double[][] out = new double[k][dim]; + for (int i = 0; i < k; i++) { + out[i] = picked.get(i); + } + return out; + } + + private static double evaluateAim( + List curCentroids, double[][] euclideanCentroids, long[] clusterSize) { + double res = 0.0; + for (int i = 0; i < euclideanCentroids.length; i++) { + double maxNcc = Double.NEGATIVE_INFINITY; + for (double[] cur : curCentroids) { + double n = ClusterUtils.maxNcc(cur, euclideanCentroids[i]); + if (n > maxNcc) { + maxNcc = n; + } + } + res += maxNcc * clusterSize[i]; + } + return res; + } + + private static void validate(double[][] samples, int k, int maxIterations) { + if (samples == null || samples.length == 0) { + throw new IllegalArgumentException("samples must be non-empty."); + } + if (k < 2) { + throw new IllegalArgumentException("k must be at least 2."); + } + if (k > samples.length) { + throw new IllegalArgumentException("k must not exceed the number of samples."); + } + if (maxIterations < 1) { + throw new IllegalArgumentException("maxIterations must be at least 1."); + } + int dim = samples[0].length; + if (dim == 0) { + throw new IllegalArgumentException("sample dimension must be positive."); + } + for (double[] row : samples) { + if (row == null || row.length != dim) { + throw new IllegalArgumentException("All samples must have the same length."); + } + } + } +} diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java index 047beafe1fc6a..0518e7a3d768a 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/UDAFQuantile.java @@ -65,59 +65,65 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati @Override public void transform(Row row, PointCollector collector) throws Exception { - double res = Util.getValueAsDouble(row); - sketch.update(dataToLong(res)); + final long encoded; + switch (dataType) { + case INT32: + encoded = row.getInt(0); + break; + case INT64: + encoded = row.getLong(0); + break; + default: + encoded = dataToLong(Util.getValueAsDouble(row)); + break; + } + sketch.update(encoded); } @Override public void terminate(PointCollector collector) throws Exception { - long result = sketch.findMinValueWithRank((long) (rank * sketch.getN())); - double res = longToResult(result); + long n = sketch.getN(); + // Nearest-rank: k-th smallest uses getApproxRank (strictly-less-than count) in [0, n-1]; + // rank=1 must map to k=n-1, not k=n which is unreachable and can overshoot the max sample. + long k = 0; + if (n > 0) { + k = (long) Math.ceil(rank * n) - 1; + if (k < 0) { + k = 0; + } else if (k >= n) { + k = n - 1; + } + } + long result = sketch.findMinValueWithRank(k); switch (dataType) { case INT32: - collector.putInt(0, (int) res); + collector.putInt(0, (int) result); break; case INT64: - collector.putLong(0, (long) res); + collector.putLong(0, result); break; case FLOAT: - collector.putFloat(0, (float) res); + collector.putFloat(0, (float) longToResult(result)); break; case DOUBLE: - collector.putDouble(0, res); + collector.putDouble(0, longToResult(result)); break; - case TIMESTAMP: - case DATE: - case TEXT: - case STRING: - case BLOB: - case BOOLEAN: default: break; } } - private long dataToLong(Object data) { - long result; + private long dataToLong(double res) { switch (dataType) { - case INT32: - return (int) data; case FLOAT: - result = Float.floatToIntBits((float) data); - return (float) data >= 0f ? result : result ^ Long.MAX_VALUE; - case INT64: - return (long) data; + float f = (float) res; + long flBits = Float.floatToIntBits(f); + return f >= 0f ? flBits : flBits ^ Long.MAX_VALUE; case DOUBLE: - result = Double.doubleToLongBits((double) data); - return (double) data >= 0d ? result : result ^ Long.MAX_VALUE; - case BLOB: - case BOOLEAN: - case STRING: - case TEXT: - case DATE: - case TIMESTAMP: + long d = Double.doubleToLongBits(res); + return res >= 0d ? d : d ^ Long.MAX_VALUE; default: - return (long) data; + return (long) res; } } @@ -129,16 +135,8 @@ private double longToResult(long result) { case DOUBLE: result = (result >>> 63) == 0 ? result : result ^ Long.MAX_VALUE; return Double.longBitsToDouble(result); - case INT64: - case INT32: - case DATE: - case TEXT: - case STRING: - case BOOLEAN: - case BLOB: - case TIMESTAMP: default: - return (result); + return (double) result; } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java index e1f0baa7c0602..47ca5e2b12fd8 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/ExactOrderStatistics.java @@ -31,7 +31,14 @@ import java.io.IOException; import java.util.NoSuchElementException; -/** Util for computing median, MAD, percentile. */ +/** + * Util for computing median, MAD, percentile. + * + *

      Percentile / quantile ({@link #getPercentile}) uses discrete nearest-rank: for sorted + * size {@code n} and {@code phi} in (0, 1], take 1-based rank {@code k = ceil(n * phi)} and 0-based + * index {@code k - 1}, clamped to {@code [0, n - 1]}. No interpolation; {@code phi = 0.5} is not + * required to match {@link #getMedian}. + */ public class ExactOrderStatistics { private final Type dataType; @@ -55,12 +62,6 @@ public ExactOrderStatistics(Type type) throws UDFInputSeriesDataTypeNotValidExce case DOUBLE: doubleArrayList = new DoubleArrayList(); break; - case STRING: - case TEXT: - case BOOLEAN: - case BLOB: - case DATE: - case TIMESTAMP: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( @@ -88,12 +89,6 @@ public void insert(Row row) throws UDFInputSeriesDataTypeNotValidException, IOEx doubleArrayList.add(vd); } break; - case DATE: - case TIMESTAMP: - case BLOB: - case BOOLEAN: - case TEXT: - case STRING: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( @@ -111,12 +106,6 @@ public double getMedian() throws UDFInputSeriesDataTypeNotValidException { return getMedian(floatArrayList); case DOUBLE: return getMedian(doubleArrayList); - case TEXT: - case STRING: - case BOOLEAN: - case BLOB: - case TIMESTAMP: - case DATE: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( @@ -199,12 +188,6 @@ public double getMad() throws UDFInputSeriesDataTypeNotValidException { return getMad(floatArrayList); case DOUBLE: return getMad(doubleArrayList); - case TIMESTAMP: - case DATE: - case BLOB: - case BOOLEAN: - case STRING: - case TEXT: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( @@ -251,12 +234,18 @@ public static double getMad(LongArrayList nums) { } } + /** Discrete nearest-rank index into sorted data of length {@code n}; see class Javadoc. */ + private static int discreteNearestRankIndex(int n, double phi) { + int idx = (int) Math.ceil(n * phi) - 1; + return Math.max(0, Math.min(n - 1, idx)); + } + public static float getPercentile(FloatArrayList nums, double phi) { if (nums.isEmpty()) { throw new NoSuchElementException(); } else { nums.sortThis(); - return nums.get((int) Math.ceil(nums.size() * phi)); + return nums.get(discreteNearestRankIndex(nums.size(), phi)); } } @@ -265,7 +254,7 @@ public static double getPercentile(DoubleArrayList nums, double phi) { throw new NoSuchElementException(); } else { nums.sortThis(); - return nums.get((int) Math.ceil(nums.size() * phi)); + return nums.get(discreteNearestRankIndex(nums.size(), phi)); } } @@ -279,12 +268,6 @@ public String getPercentile(double phi) throws UDFInputSeriesDataTypeNotValidExc return Float.toString(getPercentile(floatArrayList, phi)); case DOUBLE: return Double.toString(getPercentile(doubleArrayList, phi)); - case STRING: - case TEXT: - case BOOLEAN: - case BLOB: - case DATE: - case TIMESTAMP: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( @@ -297,7 +280,7 @@ public static int getPercentile(IntArrayList nums, double phi) { throw new NoSuchElementException(); } else { nums.sortThis(); - return nums.get((int) Math.ceil(nums.size() * phi)); + return nums.get(discreteNearestRankIndex(nums.size(), phi)); } } @@ -306,7 +289,7 @@ public static long getPercentile(LongArrayList nums, double phi) { throw new NoSuchElementException(); } else { nums.sortThis(); - return nums.get((int) Math.ceil(nums.size() * phi)); + return nums.get(discreteNearestRankIndex(nums.size(), phi)); } } } diff --git a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/GKArray.java b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/GKArray.java index 1870bdfb7a4c2..7dbcc934e7860 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/GKArray.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/dprofile/util/GKArray.java @@ -124,6 +124,19 @@ private void compress(List additionalEntries) { i++; + } else if (i >= additionalEntries.size()) { + // Only sketch entries left (must check before comparing additionalEntries.get(i)). + if (j + 1 < entries.size() + && entries.get(j).g + entries.get(j + 1).g + entries.get(j + 1).delta + <= removalThreshold) { + // Removable from sketch. + entries.get(j + 1).g += entries.get(j).g; + } else { + mergedEntries.add(entries.get(j)); + } + + j++; + } else if (additionalEntries.get(i).v < entries.get(j).v) { if (additionalEntries.get(i).g + entries.get(j).g + entries.get(j).delta <= removalThreshold) { @@ -136,7 +149,7 @@ private void compress(List additionalEntries) { i++; - } else { // the same as i == additionalEntries.size() + } else { if (j + 1 < entries.size() && entries.get(j).g + entries.get(j + 1).g + entries.get(j + 1).delta <= removalThreshold) { diff --git a/library-udf/src/main/java/org/apache/iotdb/library/match/PatternExecutor.java b/library-udf/src/main/java/org/apache/iotdb/library/match/PatternExecutor.java index 01d1c533493b6..4e5e73206709e 100644 --- a/library-udf/src/main/java/org/apache/iotdb/library/match/PatternExecutor.java +++ b/library-udf/src/main/java/org/apache/iotdb/library/match/PatternExecutor.java @@ -154,7 +154,7 @@ public List executeQuery(PatternContext queryCtx) { /** Execute the query in a particular smooth iteration */ private void executeQueryInSI(PatternContext queryCtx) { - int dsi = 0; + int dsi = 0; // dsi: data section index if (queryCtx.getDatasetSize() == null) { queryCtx.setDatasetSize( queryCtx.getDataPoints().get(queryCtx.getDataPoints().size() - 1).getX() @@ -237,11 +237,13 @@ private double calcHeight(List points) { */ private List

      findCurveSections( List tangents, List points, double minHeightPerc) { + List
      sections = new ArrayList<>(); Double lastTg = null; Point lastPt = null; double totalHeight = calcHeight(points); double lastSectHeight = 0; + for (int i = 0; i < tangents.size(); i++) { Double tangent = tangents.get(i); Point pt = points.get(i); @@ -257,6 +259,7 @@ private List
      findCurveSections( && (!(minHeightPerc > 0) || lastSectHeight / totalHeight > minHeightPerc)) { Section newSection = new Section(sign); sections.add(newSection); + newSection.getPoints().add(lastPt); newSection.getTangents().add(lastTg); } @@ -340,7 +343,6 @@ private boolean matchIn( || currSect.getNext().get(0).getSize() == currSect.getNext().get(0).getTimes())) { matchValue = this.calculateMatch(dataSectsForQ, newQSections, queryCtx, false); if (matchValue != null) { - // Keep only one (best) match if the same area is selected in different smooth iterations int duplicateMatchIdx = PatternMatchConfig.REMOVE_EQUAL_MATCHES @@ -488,6 +490,7 @@ private PatternCalculationResult calculatePointsMatch( querySect.setPoints(querySections.get(si).getPoints()); querySect.setWidth(calcWidth(querySect.getPoints())); querySect.setHeight(calcHeight(querySect.getPoints())); + if (querySect.getHeight() == 0) { continue; } @@ -507,6 +510,7 @@ private PatternCalculationResult calculatePointsMatch( } dataSect.setWidth(calcWidth(dataSect.getPoints())); dataSect.setHeight(calcHeight(dataSect.getPoints())); + if (dataSect.getHeight() == 0) { continue; } @@ -540,6 +544,7 @@ private PatternCalculationResult calculatePointsMatch( * scaleFactorY); } querySect.setCentroidY(querySect.getCentroidY() / querySect.getPoints().size()); + centroidsDifference = querySect.getPoints().get(0).getY() * (PatternMatchConfig.RESCALING_Y diff --git a/pom.xml b/pom.xml index 35c7eeb481e15..11ffc5335b293 100644 --- a/pom.xml +++ b/pom.xml @@ -60,7 +60,7 @@ 4.2.0 - 1.81 + 1.84 2.9.3 3.3.0 @@ -75,7 +75,6 @@ 2.11.1 4.4 false - 3.4.4 1.21.1 4.2.19 11.1.0 @@ -127,7 +126,7 @@ 0.18.0 - 4.1.126.Final + 4.1.134.Final 6.6.0 7.0.0 @@ -158,6 +157,7 @@ false 2.43.0 + 17.0.6 1.6.14 chmod @@ -454,11 +454,6 @@ h2-mvstore ${h2.version} - - com.lmax - disruptor - ${disruptor.version} - org.apache.httpcomponents httpclient @@ -869,9 +864,9 @@ 1.7.1 - com.googlecode.cmake-maven-project + io.github.cmake-maven-plugin cmake-maven-plugin - 3.29.3-b2 + 4.2.3-b1 org.apache.maven.plugins