linode · moshevayner · Aug 4, 2026 · Aug 3, 2026
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -103,6 +103,7 @@ jobs:
       LINODE_CONTROL_PLANE_MACHINE_TYPE: g6-standard-2
       LINODE_MACHINE_TYPE: g6-standard-2
       WORKER_NODES: '2'
+      E2E_RESERVED_IP_TAG: ccm-e2e-${{ github.run_id }}-${{ github.run_attempt }}
     steps:
       - name: Harden Runner
         uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0

diff --git a/Makefile b/Makefile
@@ -38,6 +38,7 @@ LINODE_FIREWALL_ENABLED ?= true
 LINODE_REGION           ?= us-lax
 LINODE_OS               ?= linode/ubuntu22.04
 LINODE_URL              ?= https://api.linode.com
+E2E_RESERVED_IP_TAG     ?= ccm-e2e-$(shell git rev-parse --short HEAD)
 KUBECONFIG_PATH         ?= $(CURDIR)/test-cluster-kubeconfig.yaml
 SUBNET_KUBECONFIG_PATH	?= $(CURDIR)/subnet-testing-kubeconfig.yaml
 MGMT_KUBECONFIG_PATH    ?= $(CURDIR)/mgmt-cluster-kubeconfig.yaml
@@ -202,11 +203,12 @@ mgmt-cluster:
 
 .PHONY: cleanup-cluster
 cleanup-cluster:
-	KUBECONFIG=$(KUBECONFIG_PATH) kubectl delete svc -A --field-selector spec.type=LoadBalancer
-	kubectl delete cluster -A --all --timeout=180s
-	kubectl delete linodefirewalls -A --all --timeout=180s
-	kubectl delete lvpc -A --all --timeout=180s
-	kind delete cluster -n caplccm
+	-KUBECONFIG=$(KUBECONFIG_PATH) kubectl delete svc -A --field-selector spec.type=LoadBalancer
+	-kubectl delete cluster -A --all --timeout=180s
+	-kubectl delete linodefirewalls -A --all --timeout=180s
+	-kubectl delete lvpc -A --all --timeout=180s
+	-kind delete cluster -n caplccm
+	E2E_RESERVED_IP_TAG=$(E2E_RESERVED_IP_TAG) LINODE_TOKEN=$(LINODE_TOKEN) LINODE_URL=$(LINODE_URL) ./e2e/test/scripts/cleanup-reserved-ips.sh sweep
 
 .PHONY: e2e-test
 e2e-test:
@@ -218,6 +220,7 @@ e2e-test:
 	REGION=$(LINODE_REGION) \
 	LINODE_TOKEN=$(LINODE_TOKEN) \
 	LINODE_URL=$(LINODE_URL) \
+	E2E_RESERVED_IP_TAG=$(E2E_RESERVED_IP_TAG) \
 	chainsaw test e2e/test --parallel 2 --selector all $(E2E_FLAGS)
 
 .PHONY: e2e-test-ipv6-backends

diff --git a/e2e/test/lb-created-with-reserved-ip-and-nb-id-annotations/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-and-nb-id-annotations/chainsaw-test.yaml
@@ -52,9 +52,13 @@ spec:
                 -H "Content-Type: application/json" --fail-early --retry 3 \
                 -H "accept: application/json" \
                 "${LINODE_URL}/v4beta/networking/reserved/ips" \
-                --data "{\"region\": \"$LINODE_REGION\"}" | jq .address)
+                --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address)
 
               reserved_ip=$(echo "$reserved_ip" | tr -d '"')
+              create_cm=$(kubectl -n $NAMESPACE create configmap nb-config --from-literal=Reserved_IP=$reserved_ip -o yaml --dry-run=client | kubectl apply -f -)
+              if [[ "$create_cm" != "configmap/nb-config created" ]]; then
+                echo "Unable to create configmap. Error: $create_cm"
+              fi
               LABEL="ccm-$(head /dev/urandom | tr -dc 'a-z0-9' | head -c 5)"
               data=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN LABEL=$LABEL REGION=$LINODE_REGION ../scripts/get-nb-create-data.sh svc-dummy)
 
@@ -76,13 +80,13 @@ spec:
               echo "{\"reserved_ip\": \"$reserved_ip\",
                       \"nb_id\": \"$nb_id\"}"
 
-              create_cm=$(kubectl -n $NAMESPACE create configmap nb-config --from-literal=Reserved_IP=$reserved_ip --from-literal=NB_ID=$nb_id -o yaml --dry-run=client | kubectl apply -f -)
-              if [[ "$create_cm" != "configmap/nb-config created" ]]; then
-                echo "Unable to create configmap. Error: $create_cm"
+              if ! kubectl -n $NAMESPACE patch configmap nb-config --type merge -p "{\"data\":{\"NB_ID\":\"$nb_id\"}}" >/dev/null; then
+                echo "Unable to update configmap"
               fi
             check:
               ($error == null): true
               (contains($stdout, 'Unable to create configmap')): false
+              (contains($stdout, 'Unable to update configmap')): false
               (contains($stdout, 'failed to create a reserved ip')): false
               (contains($stdout, 'failed to create nodebalancer')): false
             outputs:
@@ -119,10 +123,7 @@ spec:
               fi
               primary_ip=$(kubectl get configmap nb-config -o=jsonpath='{.data.Reserved_IP}' -n $NAMESPACE 2>/dev/null || true)
               if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then
-                echo "Releasing reserved IP $primary_ip (best-effort)"
-                curl -s -o /dev/null --request DELETE \
-                  -H "Authorization: Bearer $LINODE_TOKEN" \
-                  "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true
+                ../scripts/cleanup-reserved-ips.sh release "$primary_ip"
               fi
 
               delete_cm=$(kubectl delete configmap nb-config -n $NAMESPACE)

diff --git a/e2e/test/lb-created-with-reserved-ip-attached-to-nb/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-attached-to-nb/chainsaw-test.yaml
@@ -53,9 +53,13 @@ spec:
                 -H "Content-Type: application/json" --fail-early --retry 3 \
                 -H "accept: application/json" \
                 "${LINODE_URL}/v4beta/networking/reserved/ips" \
-                --data "{\"region\": \"$LINODE_REGION\"}" | jq .address)
+                --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address)
 
               reserved_ip=$(echo "$reserved_ip" | tr -d '"')
+              create_cm=$(kubectl -n $NAMESPACE create configmap nb-config --from-literal=Reserved_IP=$reserved_ip -o yaml --dry-run=client | kubectl apply -f -)
+              if [[ "$create_cm" != "configmap/nb-config created" ]]; then
+                echo "Unable to create configmap. Error: $create_cm"
+              fi
               LABEL="ccm-$(head /dev/urandom | tr -dc 'a-z0-9' | head -c 5)"
               data=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN LABEL=$LABEL REGION=$LINODE_REGION ../scripts/get-nb-create-data.sh svc-dummy)
 
@@ -77,13 +81,13 @@ spec:
               echo "{\"reserved_ip\": \"$reserved_ip\",
                       \"nb_id\": \"$nb_id\"}"
 
-              create_cm=$(kubectl -n $NAMESPACE create configmap nb-config --from-literal=Reserved_IP=$reserved_ip --from-literal=NB_ID=$nb_id -o yaml --dry-run=client | kubectl apply -f -)
-              if [[ "$create_cm" != "configmap/nb-config created" ]]; then
-                echo "Unable to create configmap. Error: $create_cm"
+              if ! kubectl -n $NAMESPACE patch configmap nb-config --type merge -p "{\"data\":{\"NB_ID\":\"$nb_id\"}}" >/dev/null; then
+                echo "Unable to update configmap"
               fi
             check:
               ($error == null): true
               (contains($stdout, 'Unable to create configmap')): false
+              (contains($stdout, 'Unable to update configmap')): false
               (contains($stdout, 'failed to create a reserved ip')): false
               (contains($stdout, 'failed to create nodebalancer')): false
             outputs:
@@ -127,10 +131,7 @@ spec:
               # the reserved IP; release it here (best-effort) so it does not leak.
               primary_ip=$(kubectl get configmap nb-config -o=jsonpath='{.data.Reserved_IP}' -n $NAMESPACE 2>/dev/null || true)
               if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then
-                echo "Releasing reserved IP $primary_ip (best-effort)"
-                curl -s -o /dev/null --request DELETE \
-                  -H "Authorization: Bearer $LINODE_TOKEN" \
-                  "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true
+                ../scripts/cleanup-reserved-ips.sh release "$primary_ip"
               fi
 
               delete_cm=$(kubectl delete configmap nb-config -n $NAMESPACE)

diff --git a/e2e/test/lb-created-with-reserved-ip-change-ip-concurrently/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-change-ip-concurrently/chainsaw-test.yaml
@@ -35,7 +35,7 @@ spec:
                 -H "Content-Type: application/json" --fail-early --retry 3 \
                 -H "accept: application/json" \
                 "${LINODE_URL}/v4beta/networking/reserved/ips" \
-                --data "{\"region\": \"$LINODE_REGION\"}" | jq .address)
+                --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address)
 
               reserved_ip=$(echo "$reserved_ip" | tr -d '"')
 
@@ -44,12 +44,17 @@ spec:
                   exit 1
               fi
 
+              create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip -o yaml --dry-run=client | kubectl apply -f -)
+              if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then
+                echo "Unable to create configmap. Error: $create_cm"
+              fi
+
               reserved_ip2=$(curl -s --request POST \
                 -H "Authorization: Bearer $LINODE_TOKEN" \
                 -H "Content-Type: application/json" --fail-early --retry 3 \
                 -H "accept: application/json" \
                 "${LINODE_URL}/v4beta/networking/reserved/ips" \
-                --data "{\"region\": \"$LINODE_REGION\"}" | jq .address)
+                --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address)
 
               reserved_ip2=$(echo "$reserved_ip2" | tr -d '"')
 
@@ -58,9 +63,8 @@ spec:
                   exit 1
               fi
 
-              create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip --from-literal=ReservedIP2=$reserved_ip2 -o yaml --dry-run=client | kubectl apply -f -)
-              if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then
-                echo "Unable to create configmap. Error: $create_cm"
+              if ! kubectl -n $NAMESPACE patch configmap reserved-ip-config --type merge -p "{\"data\":{\"ReservedIP2\":\"$reserved_ip2\"}}" >/dev/null; then
+                echo "Unable to update configmap"
               fi
 
               echo "{ \"reserved_ip\": \"$reserved_ip\" ,
@@ -89,34 +93,14 @@ spec:
             content: |
               set -euo pipefail
 
-              reserved_ip2=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP2}' -n $NAMESPACE)
-              if [ -z "$reserved_ip2" ]; then
-                  echo "Error: No reserved ip found in configmap"
-                  exit 1
-              fi
-
-              delete_rip2=$(curl -s --request DELETE \
-                -H "Authorization: Bearer $LINODE_TOKEN" \
-                -H "Content-Type: application/json" --fail-early --retry 3 \
-                "${LINODE_URL}/v4beta/networking/reserved/ips/$reserved_ip2" )
-
-              if [[ "$delete_rip2" == "{}" ]]; then
-                echo "Reserved IP $reserved_ip2 deleted successfully"
-              else
-                echo "Unable to delete reserved ip: $reserved_ip2. Error: $delete_rip2"
-              fi
-
-              # Release the primary reserved IP if it still exists. On the happy
-              # path CCM already released it when svc-test was torn down, so this
-              # is a best-effort, idempotent no-op; on failure paths it prevents
-              # the reserved IP from leaking (chainsaw does not track reserved IPs
-              # created via the API).
               primary_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE 2>/dev/null || true)
               if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then
-                echo "Releasing reserved IP $primary_ip (best-effort)"
-                curl -s -o /dev/null --request DELETE \
-                  -H "Authorization: Bearer $LINODE_TOKEN" \
-                  "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true
+                ../scripts/cleanup-reserved-ips.sh release "$primary_ip"
+              fi
+
+              reserved_ip2=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP2}' -n $NAMESPACE 2>/dev/null || true)
+              if [[ "$reserved_ip2" =~ ^[0-9.]+$ ]]; then
+                ../scripts/cleanup-reserved-ips.sh release "$reserved_ip2"
               fi
 
               delete_cm=$(kubectl delete configmap reserved-ip-config -n $NAMESPACE)
@@ -129,7 +113,7 @@ spec:
               ($error == null): true
               (contains($stdout, 'No reserved ip found in configmap')): false
               (contains($stdout, 'Unable to delete the configmap')): false
-              (contains($stdout, 'Unable to delete reserved ip')): false
+              (contains($stdout, 'Unable to update configmap')): false
     - name: Check that loadbalancer ip is assigned
       try:
       - assert:

diff --git a/e2e/test/lb-created-with-reserved-ip-change-ip-unreserved/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-change-ip-unreserved/chainsaw-test.yaml
@@ -35,7 +35,7 @@ spec:
                 -H "Content-Type: application/json" --fail-early --retry 3 \
                 -H "accept: application/json" \
                 "${LINODE_URL}/v4beta/networking/reserved/ips" \
-                --data "{\"region\": \"$LINODE_REGION\"}" | jq .address)
+                --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address)
 
               reserved_ip=$(echo "$reserved_ip" | tr -d '"')
 
@@ -83,10 +83,7 @@ spec:
               # reservation, so it is intentionally left alone.
               primary_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE 2>/dev/null || true)
               if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then
-                echo "Releasing reserved IP $primary_ip (best-effort)"
-                curl -s -o /dev/null --request DELETE \
-                  -H "Authorization: Bearer $LINODE_TOKEN" \
-                  "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true
+                ../scripts/cleanup-reserved-ips.sh release "$primary_ip"
               fi
 
               delete_cm=$(kubectl delete configmap reserved-ip-config -n $NAMESPACE)

diff --git a/e2e/test/lb-created-with-reserved-ip-change-ip/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-change-ip/chainsaw-test.yaml
@@ -35,30 +35,34 @@ spec:
                 -H "Content-Type: application/json" --fail-early --retry 3 \
                 -H "accept: application/json" \
                 "${LINODE_URL}/v4beta/networking/reserved/ips" \
-                --data "{\"region\": \"$LINODE_REGION\"}" | jq .address)
+                --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address)
 
               reserved_ip=$(echo "$reserved_ip" | tr -d '"')
 
               if ! [[ $reserved_ip =~ $re ]]; then
                   echo "Reserved ip [$reserved_ip] is incorrect, failed to create a reserved ip"
               fi
 
+              create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip -o yaml --dry-run=client | kubectl apply -f -)
+              if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then
+                echo "Unable to create configmap. Error: $create_cm"
+              fi
+
               reserved_ip2=$(curl -s --request POST \
                 -H "Authorization: Bearer $LINODE_TOKEN" \
                 -H "Content-Type: application/json" --fail-early --retry 3 \
                 -H "accept: application/json" \
                 "${LINODE_URL}/v4beta/networking/reserved/ips" \
-                --data "{\"region\": \"$LINODE_REGION\"}" | jq .address)
+                --data "{\"region\": \"$LINODE_REGION\", \"tags\": [\"$E2E_RESERVED_IP_TAG\"]}" | jq .address)
 
               reserved_ip2=$(echo "$reserved_ip2" | tr -d '"')
 
               if ! [[ $reserved_ip2 =~ $re ]]; then
                   echo "Reserved ip [$reserved_ip2] is incorrect, failed to create a reserved ip"
               fi
 
-              create_cm=$(kubectl -n $NAMESPACE create configmap reserved-ip-config --from-literal=ReservedIP=$reserved_ip --from-literal=ReservedIP2=$reserved_ip2 -o yaml --dry-run=client | kubectl apply -f -)
-              if [[ "$create_cm" != "configmap/reserved-ip-config created" ]]; then
-                echo "Unable to create configmap. Error: $create_cm"
+              if ! kubectl -n $NAMESPACE patch configmap reserved-ip-config --type merge -p "{\"data\":{\"ReservedIP2\":\"$reserved_ip2\"}}" >/dev/null; then
+                echo "Unable to update configmap"
               fi
 
               echo "{ \"reserved_ip\": \"$reserved_ip\" ,
@@ -88,33 +92,14 @@ spec:
             content: |
               set -euo pipefail
 
-              reserved_ip2=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP2}' -n $NAMESPACE)
-              if [ -z "$reserved_ip2" ]; then
-                  echo "Error: No reserved ip found in configmap"
-              fi
-
-              delete_rip2=$(curl -s --request DELETE \
-                -H "Authorization: Bearer $LINODE_TOKEN" \
-                -H "Content-Type: application/json" --fail-early --retry 3 \
-                "${LINODE_URL}/v4beta/networking/reserved/ips/$reserved_ip2" )
-
-              if [[ "$delete_rip2" == "{}" ]]; then
-                echo "Reserved IP $reserved_ip2 deleted successfully"
-              else
-                echo "Unable to delete reserved ip: $reserved_ip2. Error: $delete_rip2"
-              fi
-
-              # Release the primary reserved IP if it still exists. On the happy
-              # path CCM already released it when svc-test was torn down, so this
-              # is a best-effort, idempotent no-op; on failure paths it prevents
-              # the reserved IP from leaking (chainsaw does not track reserved IPs
-              # created via the API).
               primary_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE 2>/dev/null || true)
               if [[ "$primary_ip" =~ ^[0-9.]+$ ]]; then
-                echo "Releasing reserved IP $primary_ip (best-effort)"
-                curl -s -o /dev/null --request DELETE \
-                  -H "Authorization: Bearer $LINODE_TOKEN" \
-                  "${LINODE_URL}/v4beta/networking/reserved/ips/$primary_ip" || true
+                ../scripts/cleanup-reserved-ips.sh release "$primary_ip"
+              fi
+
+              reserved_ip2=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP2}' -n $NAMESPACE 2>/dev/null || true)
+              if [[ "$reserved_ip2" =~ ^[0-9.]+$ ]]; then
+                ../scripts/cleanup-reserved-ips.sh release "$reserved_ip2"
               fi
 
               delete_cm=$(kubectl delete configmap reserved-ip-config -n $NAMESPACE)
@@ -126,7 +111,7 @@ spec:
             check:
               ($error == null): true
               (contains($stdout, 'Unable to delete the configmap')): false
-              (contains($stdout, 'Unable to delete reserved ip')): false
+              (contains($stdout, 'Unable to update configmap')): false
     - name: Check that loadbalancer ip is assigned
       try:
       - assert: