From 07efb865419dc40b9eb0c0a86347feb81605bc62 Mon Sep 17 00:00:00 2001 From: Moshe Vayner Date: Mon, 3 Aug 2026 13:34:42 -0400 Subject: [PATCH] [test] implement proper cleanup of reserved IPs in e2e tests Signed-off-by: Moshe Vayner --- .github/workflows/ci.yml | 1 + Makefile | 13 ++-- .../chainsaw-test.yaml | 17 +++-- .../chainsaw-test.yaml | 17 +++-- .../chainsaw-test.yaml | 48 ++++-------- .../chainsaw-test.yaml | 7 +- .../chainsaw-test.yaml | 47 ++++-------- .../chainsaw-test.yaml | 18 +++-- .../chainsaw-test.yaml | 49 +++++------- .../chainsaw-test.yaml | 28 +++++-- .../chainsaw-test.yaml | 17 +++-- e2e/test/scripts/cleanup-reserved-ips.sh | 75 +++++++++++++++++++ 12 files changed, 192 insertions(+), 145 deletions(-) create mode 100755 e2e/test/scripts/cleanup-reserved-ips.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8da8e3ad..afa91a9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,6 +103,7 @@ jobs: LINODE_CONTROL_PLANE_MACHINE_TYPE: g6-standard-2 LINODE_MACHINE_TYPE: g6-standard-2 WORKER_NODES: '2' + E2E_RESERVED_IP_TAG: ccm-e2e-${{ github.run_id }}-${{ github.run_attempt }} steps: - name: Harden Runner uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0 diff --git a/Makefile b/Makefile index 85725980..c94ef946 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,7 @@ LINODE_FIREWALL_ENABLED ?= true LINODE_REGION ?= us-lax LINODE_OS ?= linode/ubuntu22.04 LINODE_URL ?= https://api.linode.com +E2E_RESERVED_IP_TAG ?= ccm-e2e-$(shell git rev-parse --short HEAD) KUBECONFIG_PATH ?= $(CURDIR)/test-cluster-kubeconfig.yaml SUBNET_KUBECONFIG_PATH ?= $(CURDIR)/subnet-testing-kubeconfig.yaml MGMT_KUBECONFIG_PATH ?= $(CURDIR)/mgmt-cluster-kubeconfig.yaml @@ -202,11 +203,12 @@ mgmt-cluster: .PHONY: cleanup-cluster cleanup-cluster: - KUBECONFIG=$(KUBECONFIG_PATH) kubectl delete svc -A --field-selector spec.type=LoadBalancer - kubectl delete cluster -A --all --timeout=180s - kubectl delete linodefirewalls -A --all --timeout=180s - kubectl delete lvpc -A --all --timeout=180s - kind delete cluster -n caplccm + -KUBECONFIG=$(KUBECONFIG_PATH) kubectl delete svc -A --field-selector spec.type=LoadBalancer + -kubectl delete cluster -A --all --timeout=180s + -kubectl delete linodefirewalls -A --all --timeout=180s + -kubectl delete lvpc -A --all --timeout=180s + -kind delete cluster -n caplccm + E2E_RESERVED_IP_TAG=$(E2E_RESERVED_IP_TAG) LINODE_TOKEN=$(LINODE_TOKEN) LINODE_URL=$(LINODE_URL) ./e2e/test/scripts/cleanup-reserved-ips.sh sweep .PHONY: e2e-test e2e-test: @@ -218,6 +220,7 @@ e2e-test: REGION=$(LINODE_REGION) \ LINODE_TOKEN=$(LINODE_TOKEN) \ LINODE_URL=$(LINODE_URL) \ + E2E_RESERVED_IP_TAG=$(E2E_RESERVED_IP_TAG) \ chainsaw test e2e/test --parallel 2 --selector all $(E2E_FLAGS) .PHONY: e2e-test-ipv6-backends diff --git a/e2e/test/lb-created-with-reserved-ip-and-nb-id-annotations/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-and-nb-id-annotations/chainsaw-test.yaml index f4ff9961..a917e8f5 100644 --- a/e2e/test/lb-created-with-reserved-ip-and-nb-id-annotations/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-and-nb-id-annotations/chainsaw-test.yaml @@ -52,9 +52,13 @@ spec: -H "Content-Type: application/json" --fail-early --retry 3 \ -H "accept: application/json" \ "${LINODE_URL}/v4beta/networking/reserved/ips" \ - --data "{\"region\": \"$LINODE_REGION\"}" | jq .address) + --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address) reserved_ip=$(echo "$reserved_ip" | tr -d '"') + create_cm=$(kubectl -n $NAMESPACE create configmap nb-config --from-literal=Reserved_IP=$reserved_ip -o yaml --dry-run=client | kubectl apply -f -) + if [[ "$create_cm" != "configmap/nb-config created" ]]; then + echo "Unable to create configmap. Error: $create_cm" + fi LABEL="ccm-$(head /dev/urandom | tr -dc 'a-z0-9' | head -c 5)" data=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN LABEL=$LABEL REGION=$LINODE_REGION ../scripts/get-nb-create-data.sh svc-dummy) @@ -76,13 +80,13 @@ spec: echo "{\"reserved_ip\": \"$reserved_ip\", \"nb_id\": \"$nb_id\"}" - create_cm=$(kubectl -n $NAMESPACE create configmap nb-config --from-literal=Reserved_IP=$reserved_ip --from-literal=NB_ID=$nb_id -o yaml --dry-run=client | kubectl apply -f -) - if [[ "$create_cm" != "configmap/nb-config created" ]]; then - echo "Unable to create configmap. Error: $create_cm" + if ! kubectl -n $NAMESPACE patch configmap nb-config --type merge -p "{\"data\":{\"NB_ID\":\"$nb_id\"}}" >/dev/null; then + echo "Unable to update configmap" fi check: ($error == null): true (contains($stdout, 'Unable to create configmap')): false + (contains($stdout, 'Unable to update configmap')): false (contains($stdout, 'failed to create a reserved ip')): false (contains($stdout, 'failed to create nodebalancer')): false outputs: @@ -119,10 +123,7 @@ spec: fi primary_ip=$(kubectl get configmap nb-config -o=jsonpath='{.data.Reserved_IP}' -n $NAMESPACE 2>/dev/null || true) if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then - echo "Releasing reserved IP $primary_ip (best-effort)" - curl -s -o /dev/null --request DELETE \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true + ../scripts/cleanup-reserved-ips.sh release "$primary_ip" fi delete_cm=$(kubectl delete configmap nb-config -n $NAMESPACE) diff --git a/e2e/test/lb-created-with-reserved-ip-attached-to-nb/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-attached-to-nb/chainsaw-test.yaml index 068cb3c2..35d7889a 100644 --- a/e2e/test/lb-created-with-reserved-ip-attached-to-nb/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-attached-to-nb/chainsaw-test.yaml @@ -53,9 +53,13 @@ spec: -H "Content-Type: application/json" --fail-early --retry 3 \ -H "accept: application/json" \ "${LINODE_URL}/v4beta/networking/reserved/ips" \ - --data "{\"region\": \"$LINODE_REGION\"}" | jq .address) + --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address) reserved_ip=$(echo "$reserved_ip" | tr -d '"') + create_cm=$(kubectl -n $NAMESPACE create configmap nb-config --from-literal=Reserved_IP=$reserved_ip -o yaml --dry-run=client | kubectl apply -f -) + if [[ "$create_cm" != "configmap/nb-config created" ]]; then + echo "Unable to create configmap. Error: $create_cm" + fi LABEL="ccm-$(head /dev/urandom | tr -dc 'a-z0-9' | head -c 5)" data=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN LABEL=$LABEL REGION=$LINODE_REGION ../scripts/get-nb-create-data.sh svc-dummy) @@ -77,13 +81,13 @@ spec: echo "{\"reserved_ip\": \"$reserved_ip\", \"nb_id\": \"$nb_id\"}" - create_cm=$(kubectl -n $NAMESPACE create configmap nb-config --from-literal=Reserved_IP=$reserved_ip --from-literal=NB_ID=$nb_id -o yaml --dry-run=client | kubectl apply -f -) - if [[ "$create_cm" != "configmap/nb-config created" ]]; then - echo "Unable to create configmap. Error: $create_cm" + if ! kubectl -n $NAMESPACE patch configmap nb-config --type merge -p "{\"data\":{\"NB_ID\":\"$nb_id\"}}" >/dev/null; then + echo "Unable to update configmap" fi check: ($error == null): true (contains($stdout, 'Unable to create configmap')): false + (contains($stdout, 'Unable to update configmap')): false (contains($stdout, 'failed to create a reserved ip')): false (contains($stdout, 'failed to create nodebalancer')): false outputs: @@ -127,10 +131,7 @@ spec: # the reserved IP; release it here (best-effort) so it does not leak. primary_ip=$(kubectl get configmap nb-config -o=jsonpath='{.data.Reserved_IP}' -n $NAMESPACE 2>/dev/null || true) if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then - echo "Releasing reserved IP $primary_ip (best-effort)" - curl -s -o /dev/null --request DELETE \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true + ../scripts/cleanup-reserved-ips.sh release "$primary_ip" fi delete_cm=$(kubectl delete configmap nb-config -n $NAMESPACE) diff --git a/e2e/test/lb-created-with-reserved-ip-change-ip-concurrently/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-change-ip-concurrently/chainsaw-test.yaml index be2475ce..fb6252e8 100644 --- a/e2e/test/lb-created-with-reserved-ip-change-ip-concurrently/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-change-ip-concurrently/chainsaw-test.yaml @@ -35,7 +35,7 @@ spec: -H "Content-Type: application/json" --fail-early --retry 3 \ -H "accept: application/json" \ "${LINODE_URL}/v4beta/networking/reserved/ips" \ - --data "{\"region\": \"$LINODE_REGION\"}" | jq .address) + --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address) reserved_ip=$(echo "$reserved_ip" | tr -d '"') @@ -44,12 +44,17 @@ spec: exit 1 fi + create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip -o yaml --dry-run=client | kubectl apply -f -) + if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then + echo "Unable to create configmap. Error: $create_cm" + fi + reserved_ip2=$(curl -s --request POST \ -H "Authorization: Bearer $LINODE_TOKEN" \ -H "Content-Type: application/json" --fail-early --retry 3 \ -H "accept: application/json" \ "${LINODE_URL}/v4beta/networking/reserved/ips" \ - --data "{\"region\": \"$LINODE_REGION\"}" | jq .address) + --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address) reserved_ip2=$(echo "$reserved_ip2" | tr -d '"') @@ -58,9 +63,8 @@ spec: exit 1 fi - create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip --from-literal=ReservedIP2=$reserved_ip2 -o yaml --dry-run=client | kubectl apply -f -) - if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then - echo "Unable to create configmap. Error: $create_cm" + if ! kubectl -n $NAMESPACE patch configmap reserved-ip-config --type merge -p "{\"data\":{\"ReservedIP2\":\"$reserved_ip2\"}}" >/dev/null; then + echo "Unable to update configmap" fi echo "{ \"reserved_ip\": \"$reserved_ip\" , @@ -89,34 +93,14 @@ spec: content: | set -euo pipefail - reserved_ip2=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP2}' -n $NAMESPACE) - if [ -z "$reserved_ip2" ]; then - echo "Error: No reserved ip found in configmap" - exit 1 - fi - - delete_rip2=$(curl -s --request DELETE \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - -H "Content-Type: application/json" --fail-early --retry 3 \ - "${LINODE_URL}/v4beta/networking/reserved/ips/$reserved_ip2" ) - - if [[ "$delete_rip2" == "{}" ]]; then - echo "Reserved IP $reserved_ip2 deleted successfully" - else - echo "Unable to delete reserved ip: $reserved_ip2. Error: $delete_rip2" - fi - - # Release the primary reserved IP if it still exists. On the happy - # path CCM already released it when svc-test was torn down, so this - # is a best-effort, idempotent no-op; on failure paths it prevents - # the reserved IP from leaking (chainsaw does not track reserved IPs - # created via the API). primary_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE 2>/dev/null || true) if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then - echo "Releasing reserved IP $primary_ip (best-effort)" - curl -s -o /dev/null --request DELETE \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true + ../scripts/cleanup-reserved-ips.sh release "$primary_ip" + fi + + reserved_ip2=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP2}' -n $NAMESPACE 2>/dev/null || true) + if [[ "$reserved_ip2" =~ ^[0-9.]+$ ]]; then + ../scripts/cleanup-reserved-ips.sh release "$reserved_ip2" fi delete_cm=$(kubectl delete configmap reserved-ip-config -n $NAMESPACE) @@ -129,7 +113,7 @@ spec: ($error == null): true (contains($stdout, 'No reserved ip found in configmap')): false (contains($stdout, 'Unable to delete the configmap')): false - (contains($stdout, 'Unable to delete reserved ip')): false + (contains($stdout, 'Unable to update configmap')): false - name: Check that loadbalancer ip is assigned try: - assert: diff --git a/e2e/test/lb-created-with-reserved-ip-change-ip-unreserved/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-change-ip-unreserved/chainsaw-test.yaml index dbbc2622..4c3a072c 100644 --- a/e2e/test/lb-created-with-reserved-ip-change-ip-unreserved/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-change-ip-unreserved/chainsaw-test.yaml @@ -35,7 +35,7 @@ spec: -H "Content-Type: application/json" --fail-early --retry 3 \ -H "accept: application/json" \ "${LINODE_URL}/v4beta/networking/reserved/ips" \ - --data "{\"region\": \"$LINODE_REGION\"}" | jq .address) + --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address) reserved_ip=$(echo "$reserved_ip" | tr -d '"') @@ -83,10 +83,7 @@ spec: # reservation, so it is intentionally left alone. primary_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE 2>/dev/null || true) if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then - echo "Releasing reserved IP $primary_ip (best-effort)" - curl -s -o /dev/null --request DELETE \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true + ../scripts/cleanup-reserved-ips.sh release "$primary_ip" fi delete_cm=$(kubectl delete configmap reserved-ip-config -n $NAMESPACE) diff --git a/e2e/test/lb-created-with-reserved-ip-change-ip/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-change-ip/chainsaw-test.yaml index b6a12194..17305bf9 100644 --- a/e2e/test/lb-created-with-reserved-ip-change-ip/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-change-ip/chainsaw-test.yaml @@ -35,7 +35,7 @@ spec: -H "Content-Type: application/json" --fail-early --retry 3 \ -H "accept: application/json" \ "${LINODE_URL}/v4beta/networking/reserved/ips" \ - --data "{\"region\": \"$LINODE_REGION\"}" | jq .address) + --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address) reserved_ip=$(echo "$reserved_ip" | tr -d '"') @@ -43,12 +43,17 @@ spec: echo "Reserved ip [$reserved_ip] is incorrect, failed to create a reserved ip" fi + create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip -o yaml --dry-run=client | kubectl apply -f -) + if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then + echo "Unable to create configmap. Error: $create_cm" + fi + reserved_ip2=$(curl -s --request POST \ -H "Authorization: Bearer $LINODE_TOKEN" \ -H "Content-Type: application/json" --fail-early --retry 3 \ -H "accept: application/json" \ "${LINODE_URL}/v4beta/networking/reserved/ips" \ - --data "{\"region\": \"$LINODE_REGION\"}" | jq .address) + --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address) reserved_ip2=$(echo "$reserved_ip2" | tr -d '"') @@ -56,9 +61,8 @@ spec: echo "Reserved ip [$reserved_ip2] is incorrect, failed to create a reserved ip" fi - create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip --from-literal=ReservedIP2=$reserved_ip2 -o yaml --dry-run=client | kubectl apply -f -) - if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then - echo "Unable to create configmap. Error: $create_cm" + if ! kubectl -n $NAMESPACE patch configmap reserved-ip-config --type merge -p "{\"data\":{\"ReservedIP2\":\"$reserved_ip2\"}}" >/dev/null; then + echo "Unable to update configmap" fi echo "{ \"reserved_ip\": \"$reserved_ip\" , @@ -88,33 +92,14 @@ spec: content: | set -euo pipefail - reserved_ip2=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP2}' -n $NAMESPACE) - if [ -z "$reserved_ip2" ]; then - echo "Error: No reserved ip found in configmap" - fi - - delete_rip2=$(curl -s --request DELETE \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - -H "Content-Type: application/json" --fail-early --retry 3 \ - "${LINODE_URL}/v4beta/networking/reserved/ips/$reserved_ip2" ) - - if [[ "$delete_rip2" == "{}" ]]; then - echo "Reserved IP $reserved_ip2 deleted successfully" - else - echo "Unable to delete reserved ip: $reserved_ip2. Error: $delete_rip2" - fi - - # Release the primary reserved IP if it still exists. On the happy - # path CCM already released it when svc-test was torn down, so this - # is a best-effort, idempotent no-op; on failure paths it prevents - # the reserved IP from leaking (chainsaw does not track reserved IPs - # created via the API). primary_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE 2>/dev/null || true) if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then - echo "Releasing reserved IP $primary_ip (best-effort)" - curl -s -o /dev/null --request DELETE \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true + ../scripts/cleanup-reserved-ips.sh release "$primary_ip" + fi + + reserved_ip2=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP2}' -n $NAMESPACE 2>/dev/null || true) + if [[ "$reserved_ip2" =~ ^[0-9.]+$ ]]; then + ../scripts/cleanup-reserved-ips.sh release "$reserved_ip2" fi delete_cm=$(kubectl delete configmap reserved-ip-config -n $NAMESPACE) @@ -126,7 +111,7 @@ spec: check: ($error == null): true (contains($stdout, 'Unable to delete the configmap')): false - (contains($stdout, 'Unable to delete reserved ip')): false + (contains($stdout, 'Unable to update configmap')): false - name: Check that loadbalancer ip is assigned try: - assert: diff --git a/e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml index 7823b1a6..fc5edc58 100644 --- a/e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml @@ -34,13 +34,18 @@ spec: -H "Content-Type: application/json" --fail-early --retry 3 \ -H "accept: application/json" \ "${LINODE_URL}/v4beta/networking/reserved/ips" \ - --data "{\"region\": \"$LINODE_REGION\"}" | jq .address) + --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address) reserved_ip=$(echo "$reserved_ip" | tr -d '"') if ! [[ $reserved_ip =~ $re ]]; then echo "Reserved ip [$reserved_ip] is incorrect, failed to create a reserved ip" fi + + create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip -o yaml --dry-run=client | kubectl apply -f -) + if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then + echo "Unable to create configmap. Error: $create_cm" + fi fw_label="ccm-fwtest-$RANDOM" @@ -77,13 +82,13 @@ spec: echo "{\"reserved_ip\": \"$reserved_ip\", \"fw_id\": \"$fw_id\"}" - create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip --from-literal=FW_ID=$fw_id -o yaml --dry-run=client | kubectl apply -f -) - if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then - echo "Unable to create configmap. Error: $create_cm" + if ! kubectl -n $NAMESPACE patch configmap reserved-ip-config --type merge -p "{\"data\":{\"FW_ID\":\"$fw_id\"}}" >/dev/null; then + echo "Unable to update configmap" fi check: ($error == null): true (contains($stdout, 'Unable to create configmap')): false + (contains($stdout, 'Unable to update configmap')): false (contains($stdout, 'failed to create a reserved ip')): false (contains($stdout, 'failed to create firewall')): false outputs: @@ -128,10 +133,7 @@ spec: # created via the API). primary_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE 2>/dev/null || true) if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then - echo "Releasing reserved IP $primary_ip (best-effort)" - curl -s -o /dev/null --request DELETE \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true + ../scripts/cleanup-reserved-ips.sh release "$primary_ip" fi delete_cm=$(kubectl delete configmap reserved-ip-config -n $NAMESPACE) diff --git a/e2e/test/lb-created-with-reserved-ip-multiple-change-ip/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-multiple-change-ip/chainsaw-test.yaml index fb74fbcc..84eb0019 100644 --- a/e2e/test/lb-created-with-reserved-ip-multiple-change-ip/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-multiple-change-ip/chainsaw-test.yaml @@ -35,7 +35,7 @@ spec: -H "Content-Type: application/json" --fail-early --retry 3 \ -H "accept: application/json" \ "${LINODE_URL}/v4beta/networking/reserved/ips" \ - --data "{\"region\": \"$LINODE_REGION\"}" | jq .address) + --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address) reserved_ip=$(echo "$reserved_ip" | tr -d '"') @@ -44,12 +44,18 @@ spec: exit 1 fi + create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip -o yaml --dry-run=client | kubectl apply -f -) + if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then + echo "Unable to create configmap. Error: $create_cm" + exit 1 + fi + reserved_ip2=$(curl -s --request POST \ -H "Authorization: Bearer $LINODE_TOKEN" \ -H "Content-Type: application/json" --fail-early --retry 3 \ -H "accept: application/json" \ "${LINODE_URL}/v4beta/networking/reserved/ips" \ - --data "{\"region\": \"$LINODE_REGION\"}" | jq .address) + --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address) reserved_ip2=$(echo "$reserved_ip2" | tr -d '"') @@ -58,9 +64,8 @@ spec: exit 1 fi - create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip --from-literal=ReservedIP2=$reserved_ip2 -o yaml --dry-run=client | kubectl apply -f -) - if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then - echo "Unable to create configmap. Error: $create_cm" + if ! kubectl -n $NAMESPACE patch configmap reserved-ip-config --type merge -p "{\"data\":{\"ReservedIP2\":\"$reserved_ip2\"}}" >/dev/null; then + echo "Unable to update configmap" exit 1 fi @@ -90,34 +95,14 @@ spec: content: | set -euo pipefail - reserved_ip2=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP2}' -n $NAMESPACE) - if [ -z "$reserved_ip2" ]; then - echo "Error: No reserved ip found in configmap" - exit 1 - fi - - delete_rip2=$(curl -s --request DELETE \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - -H "Content-Type: application/json" --fail-early --retry 3 \ - "${LINODE_URL}/v4beta/networking/reserved/ips/$reserved_ip2" ) - - if [[ "$delete_rip2" == "{}" ]]; then - echo "Reserved IP $reserved_ip2 deleted successfully" - else - echo "Unable to delete reserved ip: $reserved_ip2. Error: $delete_rip2" - fi - - # Release the primary reserved IP if it still exists. On the happy - # path CCM already released it when svc-test was torn down, so this - # is a best-effort, idempotent no-op; on failure paths it prevents - # the reserved IP from leaking (chainsaw does not track reserved IPs - # created via the API). primary_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE 2>/dev/null || true) if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then - echo "Releasing reserved IP $primary_ip (best-effort)" - curl -s -o /dev/null --request DELETE \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true + ../scripts/cleanup-reserved-ips.sh release "$primary_ip" + fi + + reserved_ip2=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP2}' -n $NAMESPACE 2>/dev/null || true) + if [[ "$reserved_ip2" =~ ^[0-9.]+$ ]]; then + ../scripts/cleanup-reserved-ips.sh release "$reserved_ip2" fi delete_cm=$(kubectl delete configmap reserved-ip-config -n $NAMESPACE) @@ -130,7 +115,7 @@ spec: ($error == null): true (contains($stdout, 'No reserved ip found in configmap')): false (contains($stdout, 'Unable to delete the configmap')): false - (contains($stdout, 'Unable to delete reserved ip')): false + (contains($stdout, 'Unable to update configmap')): false - name: Check that loadbalancer ip is assigned try: - assert: diff --git a/e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml index 3ade0e58..c3b65867 100644 --- a/e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml @@ -62,6 +62,21 @@ spec: if [ "$(echo "$update_ip" | jq 'length')" -eq 0 ]; then echo "Error: Unable to update the IP: $ip status to reserved" fi + + reserved_ip=$ip + create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip -o yaml --dry-run=client | kubectl apply -f -) + if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then + echo "Unable to create configmap. Error: $create_cm" + fi + + tagged_ip=$(curl -s --request PUT \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" --fail-early --retry 3 \ + "${LINODE_URL}/v4beta/networking/reserved/ips/$reserved_ip" \ + --data "{\"tags\": [\"$E2E_RESERVED_IP_TAG\"]}") + if [[ "$(echo "$tagged_ip" | jq -r --arg tag "$E2E_RESERVED_IP_TAG" '.tags | index($tag) != null')" != "true" ]]; then + echo "Unable to tag reserved ip: $reserved_ip" + fi del_nb=$(curl -s --no-keepalive --request DELETE \ -H "Authorization: Bearer $LINODE_TOKEN" \ @@ -106,13 +121,11 @@ spec: echo "failed to create firewall" fi - reserved_ip=$ip echo "{\"reserved_ip\": \"$reserved_ip\", \"fw_id\": \"$fw_id\"}" - create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip --from-literal=FW_ID=$fw_id -o yaml --dry-run=client | kubectl apply -f -) - if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then - echo "Unable to create configmap. Error: $create_cm" + if ! kubectl -n $NAMESPACE patch configmap reserved-ip-config --type merge -p "{\"data\":{\"FW_ID\":\"$fw_id\"}}" >/dev/null; then + echo "Unable to update configmap" fi check: ($error == null): true @@ -120,6 +133,8 @@ spec: (contains($stdout, 'Unable to update the IP')): false (contains($stdout, 'Error deleting the nodebalancer')): false (contains($stdout, 'Unable to create configmap')): false + (contains($stdout, 'Unable to update configmap')): false + (contains($stdout, 'Unable to tag reserved ip')): false (contains($stdout, 'failed to create firewall')): false - apply: file: create-pods-services.yaml @@ -161,10 +176,7 @@ spec: # created via the API). primary_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE 2>/dev/null || true) if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then - echo "Releasing reserved IP $primary_ip (best-effort)" - curl -s -o /dev/null --request DELETE \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true + ../scripts/cleanup-reserved-ips.sh release "$primary_ip" fi delete_cm=$(kubectl delete configmap reserved-ip-config -n $NAMESPACE) diff --git a/e2e/test/lb-created-with-specified-nb-id-reserved/chainsaw-test.yaml b/e2e/test/lb-created-with-specified-nb-id-reserved/chainsaw-test.yaml index ad8b3c52..8227ef3d 100644 --- a/e2e/test/lb-created-with-specified-nb-id-reserved/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-specified-nb-id-reserved/chainsaw-test.yaml @@ -52,9 +52,13 @@ spec: -H "Content-Type: application/json" --fail-early --retry 3 \ -H "accept: application/json" \ "${LINODE_URL}/v4beta/networking/reserved/ips" \ - --data "{\"region\": \"$LINODE_REGION\"}" | jq .address) + --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address) reserved_ip=$(echo "$reserved_ip" | tr -d '"') + create_cm=$(kubectl -n $NAMESPACE create configmap nb-config --from-literal=Reserved_IP=$reserved_ip -o yaml --dry-run=client | kubectl apply -f -) + if [[ "$create_cm" != "configmap/nb-config created" ]]; then + echo "Unable to create configmap. Error: $create_cm" + fi LABEL="ccm-$(head /dev/urandom | tr -dc 'a-z0-9' | head -c 5)" data=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN LABEL=$LABEL REGION=$LINODE_REGION ../scripts/get-nb-create-data.sh svc-dummy) @@ -76,13 +80,13 @@ spec: echo "{\"reserved_ip\": \"$reserved_ip\", \"nb_id\": \"$nb_id\"}" - create_cm=$(kubectl -n $NAMESPACE create configmap nb-config --from-literal=Reserved_IP=$reserved_ip --from-literal=NB_ID=$nb_id -o yaml --dry-run=client | kubectl apply -f -) - if [[ "$create_cm" != "configmap/nb-config created" ]]; then - echo "Unable to create configmap. Error: $create_cm" + if ! kubectl -n $NAMESPACE patch configmap nb-config --type merge -p "{\"data\":{\"NB_ID\":\"$nb_id\"}}" >/dev/null; then + echo "Unable to update configmap" fi check: ($error == null): true (contains($stdout, 'Unable to create configmap')): false + (contains($stdout, 'Unable to update configmap')): false (contains($stdout, 'failed to create a reserved ip')): false (contains($stdout, 'failed to create nodebalancer')): false outputs: @@ -120,10 +124,7 @@ spec: fi primary_ip=$(kubectl get configmap nb-config -o=jsonpath='{.data.Reserved_IP}' -n $NAMESPACE 2>/dev/null || true) if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then - echo "Releasing reserved IP $primary_ip (best-effort)" - curl -s -o /dev/null --request DELETE \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true + ../scripts/cleanup-reserved-ips.sh release "$primary_ip" fi delete_cm=$(kubectl delete configmap nb-config -n $NAMESPACE) diff --git a/e2e/test/scripts/cleanup-reserved-ips.sh b/e2e/test/scripts/cleanup-reserved-ips.sh new file mode 100755 index 00000000..750c5092 --- /dev/null +++ b/e2e/test/scripts/cleanup-reserved-ips.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -euo pipefail + +release_reserved_ip() { + local reserved_ip=$1 + local attempt + local http_status + local response + local response_file + + for attempt in {1..12}; do + response_file=$(mktemp) + http_status=$(curl -sS -o "$response_file" -w '%{http_code}' --request DELETE \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" \ + --retry 3 --retry-all-errors \ + "${LINODE_URL}/v4beta/networking/reserved/ips/$reserved_ip" || printf '000') + response=$(<"$response_file") + rm -f "$response_file" + + case "$http_status" in + 2??|404) + echo "Reserved IP $reserved_ip released (HTTP $http_status)" + return 0 + ;; + esac + + echo "Reserved IP $reserved_ip release attempt $attempt failed (HTTP $http_status): $response" >&2 + if [[ $attempt -lt 12 ]]; then + sleep 5 + fi + done + + return 1 +} + +sweep_reserved_ips() { + local response_file + + : "${E2E_RESERVED_IP_TAG:?E2E_RESERVED_IP_TAG must be set}" + + response_file=$(mktemp) + trap 'rm -f "$response_file"' RETURN + + curl -sS --fail-with-body \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" \ + "${LINODE_URL}/v4beta/networking/reserved/ips" >"$response_file" + + while IFS= read -r reserved_ip; do + release_reserved_ip "$reserved_ip" + done < <(jq -r --arg tag "$E2E_RESERVED_IP_TAG" '.data[] | select(.tags | index($tag)) | .address' "$response_file") +} + +case "${1:-}" in + release) + if [[ $# -ne 2 ]]; then + echo "usage: $0 release " >&2 + exit 2 + fi + release_reserved_ip "$2" + ;; + sweep) + if [[ $# -ne 1 ]]; then + echo "usage: $0 sweep" >&2 + exit 2 + fi + sweep_reserved_ips + ;; + *) + echo "usage: $0 {release |sweep}" >&2 + exit 2 + ;; +esac