HYPERFLEET-1185 - feat: add performance tests and tooling

.env.example

@@ -0,0 +1,2 @@
+QUAY_USER=
+HYPERFLEET_API_URL=

.gitignore

@@ -48,6 +48,9 @@ vendor/
 .env.*.local
 deploy-scripts/.env
 
+# Perf test results
+perf/results/
+
 # Claude Code files
 .claude/
 

AGENTS.md

@@ -51,7 +51,7 @@ Pre-flight order: `make check` then `make build`.
 
 - **IMPORTANT:** Test files use `.go` extension, NOT `_test.go`. E2E tests are compiled into the binary, not run via `go test`.
 - Location: `e2e/{suite}/descriptive-name.go` (package matches directory name)
-- Test name format: `[Suite: component][category] Description` (e.g., `[Suite: cluster][baseline] Cluster Resource Type Lifecycle`). Known categories: `baseline`, `update`, `delete`, `concurrent`, `negative`.
+- Test name format: `[Suite: component][category] Description` (e.g., `[Suite: cluster][baseline] Cluster Resource Type Lifecycle`). Known categories: `baseline`, `update`, `delete`, `concurrent`, `negative`, `perf`.
 - Test suites auto-register via blank import in `e2e/e2e.go`
 
 ### Labels

README.md

@@ -48,6 +48,12 @@ export HYPERFLEET_API_URL=https://api.hyperfleet.example.com
 
 Run `./bin/hyperfleet-e2e test --help` for all options.
 
+### Performance Tests
+
+Performance tests are labeled `perf` and measure baseline latencies for core operations. They run inside the cluster for production-representative numbers.
+
+See [perf/README.md](perf/README.md).
+
 ## Configuration
 
 Configuration priority (highest to lowest):

configs/config.yaml

@@ -47,6 +47,10 @@ timeouts:
     # Can be overridden by: HYPERFLEET_TIMEOUTS_NODEPOOL_RECONCILED
     reconciled: 2m
 
+    # Maximum time to wait for nodepool hard-delete (404 response)
+    # Can be overridden by: HYPERFLEET_TIMEOUTS_NODEPOOL_DELETED
+    deleted: 2m
+
   adapter:
     # Maximum time to wait for adapter processing
     #

e2e/cluster/perf_cascade_delete_latency.go

@@ -0,0 +1,71 @@
+package cluster
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability
+
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels"
+)
+
+var _ = ginkgo.Describe("[Suite: cluster][perf] Cascade delete-to-hard-delete latency",
+	ginkgo.Label(labels.Tier1, labels.Performance),
+	func() {
+		var h *helper.Helper
+		var clusterID string
+		var nodepoolID string
+
+		ginkgo.BeforeEach(func(ctx context.Context) {
+			h = helper.New()
+
+			cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json"))
+			Expect(err).NotTo(HaveOccurred())
+			clusterID = *cluster.Id
+
+			ginkgo.DeferCleanup(func(ctx context.Context) {
+				if err := h.CleanupTestCluster(ctx, clusterID); err != nil {
+					ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup cluster %s: %v\n", clusterID, err)
+				}
+			})
+
+			ginkgo.By("waiting for cluster to reach Reconciled")
+			Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval).
+				Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue))
+
+			ginkgo.By("creating a nodepool on the cluster")
+			nodepool, err := h.Client.CreateNodePoolFromPayload(ctx, clusterID, h.TestDataPath("payloads/nodepools/nodepool-request.json"))
+			Expect(err).NotTo(HaveOccurred())
+			nodepoolID = *nodepool.Id
+
+			ginkgo.DeferCleanup(func(ctx context.Context) {
+				if err := h.CleanupTestNodePool(ctx, clusterID, nodepoolID); err != nil {
+					ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup nodepool %s: %v\n", nodepoolID, err)
+				}
+			})
+
+			ginkgo.By("waiting for nodepool to reach Reconciled")
+			Eventually(h.PollNodePool(ctx, clusterID, nodepoolID), h.Cfg.Timeouts.NodePool.Reconciled, h.Cfg.Polling.Interval).
+				Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue))
+		})
+
+		ginkgo.It("should cascade-delete a cluster with nodepools and reach hard-delete within acceptable latency", func(ctx context.Context) {
+			ginkgo.By("deleting cluster (with attached nodepool) and timing until hard-delete (404)")
+			start := time.Now()
+
+			_, err := h.Client.DeleteCluster(ctx, clusterID)
+			Expect(err).NotTo(HaveOccurred())
+
+			Eventually(h.PollClusterHTTPStatus(ctx, clusterID), h.Cfg.Timeouts.Cluster.Deleted, h.Cfg.Polling.Interval).
+				Should(Equal(http.StatusNotFound))
+
+			elapsed := time.Since(start)
+			ginkgo.GinkgoWriter.Printf("[PERF] Cluster cascade delete-to-hard-delete latency: %v\n", elapsed)
+		})
+	},
+)

e2e/cluster/perf_create_latency.go

@@ -0,0 +1,47 @@
+package cluster
+
+import (
+	"context"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability
+
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels"
+)
+
+var _ = ginkgo.Describe("[Suite: cluster][perf] Create-to-reconciled latency",
+	ginkgo.Label(labels.Tier1, labels.Performance),
+	func() {
+		var h *helper.Helper
+		var clusterID string
+
+		ginkgo.BeforeEach(func(ctx context.Context) {
+			h = helper.New()
+		})
+
+		ginkgo.It("should create a cluster and reach Reconciled within acceptable latency", func(ctx context.Context) {
+			ginkgo.By("creating a cluster and timing until Reconciled")
+			start := time.Now()
+
+			cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json"))
+			Expect(err).NotTo(HaveOccurred())
+			clusterID = *cluster.Id
+
+			ginkgo.DeferCleanup(func(ctx context.Context) {
+				if err := h.CleanupTestCluster(ctx, clusterID); err != nil {
+					ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup cluster %s: %v\n", clusterID, err)
+				}
+			})
+
+			Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval).
+				Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue))
+
+			elapsed := time.Since(start)
+			ginkgo.GinkgoWriter.Printf("[PERF] Cluster create-to-reconciled latency: %v\n", elapsed)
+		})
+	},
+)

e2e/cluster/perf_delete_latency.go

@@ -0,0 +1,55 @@
+package cluster
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability
+
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels"
+)
+
+var _ = ginkgo.Describe("[Suite: cluster][perf] Delete-to-hard-delete latency",
+	ginkgo.Label(labels.Tier1, labels.Performance),
+	func() {
+		var h *helper.Helper
+		var clusterID string
+
+		ginkgo.BeforeEach(func(ctx context.Context) {
+			h = helper.New()
+
+			cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json"))
+			Expect(err).NotTo(HaveOccurred())
+			clusterID = *cluster.Id
+
+			ginkgo.DeferCleanup(func(ctx context.Context) {
+				if err := h.CleanupTestCluster(ctx, clusterID); err != nil {
+					ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup cluster %s: %v\n", clusterID, err)
+				}
+			})
+
+			ginkgo.By("waiting for cluster to reach Reconciled before delete")
+			Eventually(h.PollCluster(ctx, clusterID), h.Cfg.Timeouts.Cluster.Reconciled, h.Cfg.Polling.Interval).
+				Should(helper.HaveResourceCondition(client.ConditionTypeReconciled, openapi.ResourceConditionStatusTrue))
+		})
+
+		ginkgo.It("should delete a cluster and reach hard-delete within acceptable latency", func(ctx context.Context) {
+			ginkgo.By("deleting cluster and timing until hard-delete (404)")
+			start := time.Now()
+
+			_, err := h.Client.DeleteCluster(ctx, clusterID)
+			Expect(err).NotTo(HaveOccurred())
+
+			Eventually(h.PollClusterHTTPStatus(ctx, clusterID), h.Cfg.Timeouts.Cluster.Deleted, h.Cfg.Polling.Interval).
+				Should(Equal(http.StatusNotFound))
+
+			elapsed := time.Since(start)
+			ginkgo.GinkgoWriter.Printf("[PERF] Cluster delete-to-hard-delete latency: %v\n", elapsed)
+		})
+	},
+)

e2e/cluster/perf_list_filtered_latency.go

@@ -0,0 +1,73 @@
+package cluster
+
+import (
+	"context"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability
+
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels"
+)
+
+var _ = ginkgo.Describe("[Suite: cluster][perf] API list latency with filters and pagination",
+	ginkgo.Label(labels.Tier1, labels.Performance),
+	func() {
+		var h *helper.Helper
+		var clusterID string
+
+		ginkgo.BeforeEach(func(ctx context.Context) {
+			h = helper.New()
+
+			cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json"))
+			Expect(err).NotTo(HaveOccurred())
+			clusterID = *cluster.Id
+
+			ginkgo.DeferCleanup(func(ctx context.Context) {
+				if err := h.CleanupTestCluster(ctx, clusterID); err != nil {
+					ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup cluster %s: %v\n", clusterID, err)
+				}
+			})
+		})
+
+		ginkgo.It("should list clusters with search filter within acceptable latency", func(ctx context.Context) {
+			ginkgo.By("measuring GET /clusters?search=... response time")
+			filter := openapi.SearchParams("labels.environment='test'")
+			start := time.Now()
+			_, err := h.Client.ListClustersWithParams(ctx, &openapi.GetClustersParams{
+				Search: &filter,
+			})
+			Expect(err).NotTo(HaveOccurred())
+			elapsed := time.Since(start)
+			ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters (search filter) latency: %v\n", elapsed)
+		})
+
+		ginkgo.It("should list clusters with page size limit within acceptable latency", func(ctx context.Context) {
+			ginkgo.By("measuring GET /clusters?pageSize=10 response time")
+			pageSize := openapi.QueryParamsPageSize(10)
+			start := time.Now()
+			_, err := h.Client.ListClustersWithParams(ctx, &openapi.GetClustersParams{
+				PageSize: &pageSize,
+			})
+			Expect(err).NotTo(HaveOccurred())
+			elapsed := time.Since(start)
+			ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters (pageSize=10) latency: %v\n", elapsed)
+		})
+
+		ginkgo.It("should list clusters with pagination within acceptable latency", func(ctx context.Context) {
+			ginkgo.By("measuring GET /clusters?page=1&pageSize=10 response time")
+			page := openapi.QueryParamsPage(1)
+			pageSize := openapi.QueryParamsPageSize(10)
+			start := time.Now()
+			_, err := h.Client.ListClustersWithParams(ctx, &openapi.GetClustersParams{
+				Page:     &page,
+				PageSize: &pageSize,
+			})
+			Expect(err).NotTo(HaveOccurred())
+			elapsed := time.Since(start)
+			ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters (page=1, pageSize=10) latency: %v\n", elapsed)
+		})
+	},
+)

e2e/cluster/perf_list_latency.go

@@ -0,0 +1,43 @@
+package cluster
+
+import (
+	"context"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability
+
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels"
+)
+
+var _ = ginkgo.Describe("[Suite: cluster][perf] API list latency",
+	ginkgo.Label(labels.Tier1, labels.Performance),
+	func() {
+		var h *helper.Helper
+		var clusterID string
+
+		ginkgo.BeforeEach(func(ctx context.Context) {
+			h = helper.New()
+
+			cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json"))
+			Expect(err).NotTo(HaveOccurred())
+			clusterID = *cluster.Id
+
+			ginkgo.DeferCleanup(func(ctx context.Context) {
+				if err := h.CleanupTestCluster(ctx, clusterID); err != nil {
+					ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup cluster %s: %v\n", clusterID, err)
+				}
+			})
+		})
+
+		ginkgo.It("should list clusters within acceptable latency", func(ctx context.Context) {
+			ginkgo.By("measuring GET /clusters response time")
+			start := time.Now()
+			_, err := h.Client.ListClusters(ctx)
+			Expect(err).NotTo(HaveOccurred())
+			elapsed := time.Since(start)
+			ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters latency: %v\n", elapsed)
+		})
+	},
+)

e2e/cluster/perf_read_entity_size_latency.go

@@ -0,0 +1,54 @@
+package cluster
+
+import (
+	"context"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega" //nolint:staticcheck // dot import for test readability
+
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/helper"
+	"github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/labels"
+)
+
+var _ = ginkgo.Describe("[Suite: cluster][perf] API read latency by entity size",
+	ginkgo.Label(labels.Tier1, labels.Performance),
+	func() {
+		var h *helper.Helper
+
+		ginkgo.BeforeEach(func(ctx context.Context) {
+			h = helper.New()
+		})
+
+		sizes := []struct {
+			name    string
+			payload string
+		}{
+			{"small", "payloads/clusters/cluster-request-small.json"},
+			{"medium", "payloads/clusters/cluster-request.json"},
+			{"large", "payloads/clusters/cluster-request-large.json"},
+		}
+
+		for _, size := range sizes {
+			ginkgo.It("should read a "+size.name+" cluster within acceptable latency", func(ctx context.Context) {
+				ginkgo.By("creating a " + size.name + " cluster")
+				cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath(size.payload))
+				Expect(err).NotTo(HaveOccurred())
+				clusterID := *cluster.Id
+
+				ginkgo.DeferCleanup(func(ctx context.Context) {
+					if err := h.CleanupTestCluster(ctx, clusterID); err != nil {
+						ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup cluster %s: %v\n", clusterID, err)
+					}
+				})
+
+				ginkgo.By("measuring GET /clusters/{id} response time for " + size.name + " entity")
+				start := time.Now()
+				_, err = h.Client.GetCluster(ctx, clusterID)
+				Expect(err).NotTo(HaveOccurred())
+				elapsed := time.Since(start)
+				ginkgo.GinkgoWriter.Printf("[PERF] GET /clusters/%s (%s entity) latency: %v\n", clusterID, size.name, elapsed)
+			})
+		}
+	},
+)