From 88c91b7f701b81e7e06c40530494ef849b6a68b8 Mon Sep 17 00:00:00 2001 From: starrysky Date: Thu, 18 Jun 2026 14:42:42 +0800 Subject: [PATCH 1/5] fix: harden resp-compat client setup for Eloq e2e --- tests/docker_ha/Dockerfile.resp_compat | 2 + tests/e2e/cmd_stress_test.sh | 41 +++++++++++-------- .../resp_compat/resp_compatibility_eloq.py | 34 +++++++++++++++ 3 files changed, 60 insertions(+), 17 deletions(-) create mode 100644 tests/e2e/resp_compat/resp_compatibility_eloq.py diff --git a/tests/docker_ha/Dockerfile.resp_compat b/tests/docker_ha/Dockerfile.resp_compat index 5048f5ee..6a1f8997 100644 --- a/tests/docker_ha/Dockerfile.resp_compat +++ b/tests/docker_ha/Dockerfile.resp_compat @@ -12,4 +12,6 @@ WORKDIR /opt/resp-compatibility RUN pip install --no-cache-dir -r requirements.txt +COPY tests/e2e/resp_compat/resp_compatibility_eloq.py /opt/resp-compatibility/resp_compatibility_eloq.py + CMD ["sleep", "infinity"] diff --git a/tests/e2e/cmd_stress_test.sh b/tests/e2e/cmd_stress_test.sh index bb6bde09..02a6bdff 100755 --- a/tests/e2e/cmd_stress_test.sh +++ b/tests/e2e/cmd_stress_test.sh @@ -211,29 +211,34 @@ step() { # ── Master discovery (works independently of launch) ── discover_master() { echo " discovering cluster topology ..." - local nodes_info - nodes_info=$(docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T stress-python python3 -c " + local slots_info + slots_info=$(docker compose -f "${DOCKER_E2E_DIR}/docker-compose.yaml" exec -T stress-python python3 -c " import ssl TLS={'ssl':True,'ssl_cert_reqs':ssl.CERT_NONE,'ssl_check_hostname':False} from redis import Redis -r=Redis(host='${N1}',port=6379,password='${PASSWD}',socket_timeout=5,**TLS) -print(r.execute_command('CLUSTER','NODES').decode()) +r=Redis(host='${N1}',port=6379,password='${PASSWD}',socket_timeout=5,decode_responses=True,**TLS) +slots=r.execute_command('CLUSTER','SLOTS') +master=slots[0][2] +replica=slots[0][3] if len(slots[0]) > 3 else None +print(f\"MASTER={master[0]}:{master[1]}\") +print(f\"REPLICA={replica[0]}:{replica[1]}\" if replica else \"REPLICA=\") r.close() " 2>/dev/null) || { echo "FAIL: cannot connect to cluster for discovery"; return 1; } MASTER="" + REPLICA="" while IFS= read -r line; do - local addr role - addr=$(echo "$line" | awk '{print $2}' | cut -d@ -f1) - role=$(echo "$line" | awk '{print $3}') - if echo "$role" | grep -q 'master'; then - MASTER="${addr}" - elif echo "$role" | grep -q 'slave'; then - REPLICA="${addr}" - fi - done <<< "$nodes_info" + case "$line" in + MASTER=*) + MASTER="${line#MASTER=}" + ;; + REPLICA=*) + REPLICA="${line#REPLICA=}" + ;; + esac + done <<< "$slots_info" if [ -z "$MASTER" ]; then - echo "FAIL: could not discover master from CLUSTER NODES" + echo "FAIL: could not discover master from CLUSTER SLOTS" return 1 fi echo " master=${MASTER} replica=${REPLICA}" @@ -499,7 +504,9 @@ do_resp_compat() { local cluster_log="${SCRIPT_DIR}/resp-compat-cluster.log" local summary_log="${SCRIPT_DIR}/resp-compat-summary.log" local cts="/tmp/cts_filtered.json" - local script="/opt/resp-compatibility/resp_compatibility.py" + local script="/opt/resp-compatibility/resp_compatibility_eloq.py" + local ssl_flag="" + [ "${TLS_ENABLED}" = "1" ] && ssl_flag="--ssl" # Generate summary report header { @@ -524,7 +531,7 @@ with open('${cts}','w') as f: echo "--- standalone mode ---" docker compose -f "${compose_file}" exec -T resp-compat bash -c \ - "python3 -u ${script} --host ${master_host} --port ${master_port} --password ${PASSWD} --testfile ${cts} --specific-version ${RESP_COMPAT_VERSION} --show-failed >/tmp/standalone.log 2>&1" + "python3 -u ${script} --host ${master_host} --port ${master_port} --password ${PASSWD} ${ssl_flag} --testfile ${cts} --specific-version ${RESP_COMPAT_VERSION} --show-failed >/tmp/standalone.log 2>&1" docker compose -f "${compose_file}" cp resp-compat:/tmp/standalone.log "${standalone_log}" cat "${standalone_log}" local standalone_status=${PIPESTATUS[0]} @@ -578,7 +585,7 @@ r.close() echo "--- cluster mode ---" docker compose -f "${compose_file}" exec -T resp-compat bash -c \ - "python3 -u ${script} --host ${master_host} --port ${master_port} --password ${PASSWD} --testfile ${cts} --specific-version ${RESP_COMPAT_VERSION} --show-failed --cluster >/tmp/cluster.log 2>&1" + "python3 -u ${script} --host ${master_host} --port ${master_port} --password ${PASSWD} ${ssl_flag} --testfile ${cts} --specific-version ${RESP_COMPAT_VERSION} --show-failed --cluster >/tmp/cluster.log 2>&1" docker compose -f "${compose_file}" cp resp-compat:/tmp/cluster.log "${cluster_log}" cat "${cluster_log}" local cluster_status=${PIPESTATUS[0]} diff --git a/tests/e2e/resp_compat/resp_compatibility_eloq.py b/tests/e2e/resp_compat/resp_compatibility_eloq.py new file mode 100644 index 00000000..33b5d4c5 --- /dev/null +++ b/tests/e2e/resp_compat/resp_compatibility_eloq.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +import runpy +import ssl + +import redis + + +_ORIGINAL_REDIS = redis.Redis +_ORIGINAL_REDIS_CLUSTER = redis.RedisCluster + + +class EloqRedis(_ORIGINAL_REDIS): + def __init__(self, *args, **kwargs): + if kwargs.get("ssl"): + kwargs.setdefault("ssl_cert_reqs", ssl.CERT_NONE) + kwargs.setdefault("ssl_check_hostname", False) + kwargs.setdefault("protocol", 2) + super().__init__(*args, **kwargs) + + +class EloqRedisCluster(_ORIGINAL_REDIS_CLUSTER): + def __init__(self, *args, **kwargs): + if kwargs.get("ssl"): + kwargs.setdefault("ssl_cert_reqs", ssl.CERT_NONE) + kwargs.setdefault("ssl_check_hostname", False) + kwargs.setdefault("protocol", 2) + super().__init__(*args, **kwargs) + + +redis.Redis = EloqRedis +redis.RedisCluster = EloqRedisCluster + +runpy.run_path("/opt/resp-compatibility/resp_compatibility.py", run_name="__main__") From 87bb5ea29b6f1f829996c1a38ee5cbd8e10a3eea Mon Sep 17 00:00:00 2001 From: starrysky Date: Thu, 18 Jun 2026 15:01:04 +0800 Subject: [PATCH 2/5] ci: free runner disk before test job --- .github/workflows/test.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5afe763e..195bab33 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,25 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Free disk space + run: | + echo "::group::Disk usage before cleanup" + df -h + echo "::endgroup::" + + # Remove large preinstalled toolchains not used by this repository. + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + + sudo apt-get clean + docker system prune -af --volumes 2>/dev/null || true + + echo "::group::Disk usage after cleanup" + df -h + echo "::endgroup::" + - name: Install build dependencies uses: awalsh128/cache-apt-pkgs-action@latest with: From ce6d42b9ac5cb66ae98072ed2ca2ba1cad1cffb2 Mon Sep 17 00:00:00 2001 From: starrysky Date: Thu, 18 Jun 2026 17:07:20 +0800 Subject: [PATCH 3/5] fix: dump e2e backtraces on cluster change failure --- tests/e2e/cmd_stress_test.sh | 94 ++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 3 deletions(-) diff --git a/tests/e2e/cmd_stress_test.sh b/tests/e2e/cmd_stress_test.sh index 02a6bdff..7dd413c2 100755 --- a/tests/e2e/cmd_stress_test.sh +++ b/tests/e2e/cmd_stress_test.sh @@ -181,6 +181,93 @@ assert_export_contains() { || { echo "FAIL: export does not contain expected text: ${expected}"; return 1; } } +ssh_port_for_node() { + local host="${1%:*}" + case "${host}" in + 172.28.10.11) + echo 2221 + ;; + 172.28.10.12) + echo 2222 + ;; + *) + return 1 + ;; + esac +} + +dump_eloqkv_backtrace_for_node() { + local role="$1" + local node="$2" + local host="${node%:*}" + local ssh_port + if ! ssh_port="$(ssh_port_for_node "${node}")"; then + echo "---- ${role} (${node}) ----" + echo " skip: unknown SSH port for node" + return 0 + fi + + echo "---- ${role} (${node}) ----" + ssh_cmd "${ssh_port}" bash -lc "$(cat <<'EOF' +set +e +echo " hostname=$(hostname)" +echo " user=$(id -un)" +if ! command -v gdb >/dev/null 2>&1; then + echo " skip: gdb not installed" + exit 0 +fi +pid_line=$(ps -u eloq -o pid=,args= | grep '[e]loqkv' | head -n1) +if [ -z "${pid_line}" ]; then + echo " skip: eloqkv process not found" + ps -u eloq -o pid=,args= || true + exit 0 +fi +pid=$(printf '%s\n' "${pid_line}" | awk '{print $1}') +echo " pid=${pid}" +echo " cmd=${pid_line}" +sudo -n timeout 30s gdb -batch -nx \ + -ex 'set pagination off' \ + -ex 'thread apply all bt full' \ + -p "${pid}" 2>&1 || true +EOF + )" || true +} + +dump_cluster_change_backtraces() { + local master="${MASTER:-}" + local replica="${REPLICA:-}" + if discover_master >/dev/null 2>&1; then + master="${MASTER}" + replica="${REPLICA}" + fi + if [ -z "${master}" ]; then + master="${N1}:6379" + fi + if [ -z "${replica}" ]; then + replica="${N2}:6379" + fi + + echo "---- cluster node backtraces ----" + echo " master=${master}" + echo " standby=${replica}" + dump_eloqkv_backtrace_for_node "master" "${master}" + if [ "${replica}" != "${master}" ]; then + dump_eloqkv_backtrace_for_node "standby" "${replica}" + fi +} + +run_control_eloqctl_with_cluster_backtraces() { + local timeout_seconds="$1" + local log_file="$2" + shift 2 + local subcommand="$1" + local control_log_file="${CONTROL_ELOQCTL_HOME}/logs/last-${subcommand}.log" + local observer_timeout=$((timeout_seconds + 60)) + run_with_progress "${observer_timeout}" "${log_file}" --eloq-log "${control_log_file}" \ + bash -lc "$(control_ssh_exec_string_with_timeout "${timeout_seconds}" "$@")" \ + || { dump_failure_diagnostics "${log_file}"; dump_cluster_change_backtraces; return 1; } +} + cleanup() { if [ "${KEEP_LOGS:-0}" != "1" ]; then rm -f "${SCRIPT_DIR}/cmd-stress-"*.log "${SCRIPT_DIR}/launch-cmd-stress.log" "${TOPO}" @@ -309,7 +396,8 @@ do_monitor_update() { do_cluster_update() { echo "=== EloqKV rolling update check (${ELOQKV_VERSION} -> ${ELOQKV_UPDATE_VERSION}) ===" assert_cluster_registered || return 1 - run_control_eloqctl_with_progress 900 "${SCRIPT_DIR}/cmd-stress-cluster-update.log" \ + discover_master || return 1 + run_control_eloqctl_with_cluster_backtraces 900 "${SCRIPT_DIR}/cmd-stress-cluster-update.log" \ update "${CLUSTER}" "${ELOQKV_UPDATE_VERSION}" --password "${PASSWD}" \ || return 1 wait_cluster_ready || return 1 @@ -350,7 +438,7 @@ do_eloqctl_mutate() { echo "=== eloqctl mutation check ===" echo " failover ${original_master} -> ${original_replica}" - run_control_eloqctl_with_progress 240 "${SCRIPT_DIR}/cmd-stress-failover-1.log" \ + run_control_eloqctl_with_cluster_backtraces 240 "${SCRIPT_DIR}/cmd-stress-failover-1.log" \ failover "${CLUSTER}" \ --old-leader-host "${original_master_host}" --old-leader-port "${original_master_port}" \ --new-leader-host "${original_replica_host}" --new-leader-port "${original_replica_port}" \ @@ -366,7 +454,7 @@ do_eloqctl_mutate() { current_master_host="${MASTER%:*}" current_master_port="${MASTER##*:}" echo " failover ${MASTER} -> ${original_master}" - run_control_eloqctl_with_progress 240 "${SCRIPT_DIR}/cmd-stress-failover-2.log" \ + run_control_eloqctl_with_cluster_backtraces 240 "${SCRIPT_DIR}/cmd-stress-failover-2.log" \ failover "${CLUSTER}" \ --old-leader-host "${current_master_host}" --old-leader-port "${current_master_port}" \ --new-leader-host "${original_master_host}" --new-leader-port "${original_master_port}" \ From 47627b05a21dc591d6b378f768e4ede743b0f500 Mon Sep 17 00:00:00 2001 From: starrysky Date: Thu, 18 Jun 2026 18:01:22 +0800 Subject: [PATCH 4/5] fix: stream resp compat progress in e2e --- .github/workflows/test.yml | 2 +- tests/e2e/cmd_stress_test.sh | 30 ++++++++++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 195bab33..6abe5d0f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,7 +48,7 @@ jobs: test: name: Test runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 60 steps: - uses: actions/checkout@v4 diff --git a/tests/e2e/cmd_stress_test.sh b/tests/e2e/cmd_stress_test.sh index 7dd413c2..d4a74dd2 100755 --- a/tests/e2e/cmd_stress_test.sh +++ b/tests/e2e/cmd_stress_test.sh @@ -50,6 +50,7 @@ GRAFANA_HTTP_URL="${GRAFANA_HTTP_URL:-http://172.28.10.14:3301}" ALERTMANAGER_HTTP_URL="${ALERTMANAGER_HTTP_URL:-http://172.28.10.14:9093}" ALERTMANAGER_WEBHOOK_ADAPTER_HTTP_URL="${ALERTMANAGER_WEBHOOK_ADAPTER_HTTP_URL:-http://172.28.10.14:8080}" RESP_COMPAT_VERSION="${RESP_COMPAT_VERSION:-7.0.0}" +RESP_COMPAT_TIMEOUT_SECONDS="${RESP_COMPAT_TIMEOUT_SECONDS:-3600}" PASSWD="testpass" N1="172.28.10.11" @@ -595,6 +596,7 @@ do_resp_compat() { local script="/opt/resp-compatibility/resp_compatibility_eloq.py" local ssl_flag="" [ "${TLS_ENABLED}" = "1" ] && ssl_flag="--ssl" + local observer_timeout=$((RESP_COMPAT_TIMEOUT_SECONDS + 60)) # Generate summary report header { @@ -618,11 +620,15 @@ with open('${cts}','w') as f: " echo "--- standalone mode ---" - docker compose -f "${compose_file}" exec -T resp-compat bash -c \ - "python3 -u ${script} --host ${master_host} --port ${master_port} --password ${PASSWD} ${ssl_flag} --testfile ${cts} --specific-version ${RESP_COMPAT_VERSION} --show-failed >/tmp/standalone.log 2>&1" - docker compose -f "${compose_file}" cp resp-compat:/tmp/standalone.log "${standalone_log}" - cat "${standalone_log}" - local standalone_status=${PIPESTATUS[0]} + local standalone_inner_cmd + standalone_inner_cmd=$(cat </tmp/cluster.log 2>&1" - docker compose -f "${compose_file}" cp resp-compat:/tmp/cluster.log "${cluster_log}" - cat "${cluster_log}" - local cluster_status=${PIPESTATUS[0]} + local cluster_inner_cmd + cluster_inner_cmd=$(cat < Date: Thu, 18 Jun 2026 23:03:50 +0800 Subject: [PATCH 5/5] ci: skip resp compat on pull requests --- .github/workflows/test.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6abe5d0f..0fe68ac9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,7 +48,7 @@ jobs: test: name: Test runs-on: ubuntu-24.04 - timeout-minutes: 60 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 @@ -131,12 +131,15 @@ jobs: export REPEAT="${repeat}" export RESP_COMPAT_VERSION="${resp_compat_version}" export SKIP_DEPS=1 - export STEPS="launch,cluster-update,monitor-update,eloqctl-mutate,py-stress,go-stress,ts-stress,resp-compat,remove" + export STEPS="launch,cluster-update,monitor-update,eloqctl-mutate,py-stress,go-stress,ts-stress,remove" + if [ "${{ github.ref }}" = "refs/heads/main" ] && [ "${{ github.event_name }}" != "pull_request" ]; then + export STEPS="launch,cluster-update,monitor-update,eloqctl-mutate,py-stress,go-stress,ts-stress,resp-compat,remove" + fi echo "Running E2E with ELOQKV_VERSION=${ELOQKV_VERSION} ELOQKV_UPDATE_VERSION=${ELOQKV_UPDATE_VERSION} WORKERS=${WORKERS} INFLIGHT=${INFLIGHT} DURATION_SECONDS=${DURATION_SECONDS} REPEAT=${REPEAT} RESP_COMPAT_VERSION=${RESP_COMPAT_VERSION}" tests/e2e/devctl.sh stress "${STEPS}" - name: RESP Compatibility Report - if: always() + if: always() && github.ref == 'refs/heads/main' && github.event_name != 'pull_request' run: | echo "::group::Compatibility Summary" if [ -f tests/e2e/resp-compat-summary.log ]; then @@ -147,7 +150,7 @@ jobs: echo "::endgroup::" - name: Upload resp-compat logs - if: always() + if: always() && github.ref == 'refs/heads/main' && github.event_name != 'pull_request' uses: actions/upload-artifact@v4 with: name: resp-compat-logs