Fix node replacement test timestamp filter and allow job retry

test/extended/edge_topologies/tnf_node_replacement.go

@@ -18,6 +18,7 @@ import (
 	"github.com/openshift/origin/test/extended/edge_topologies/utils/core"
 	"github.com/openshift/origin/test/extended/etcd/helpers"
 	exutil "github.com/openshift/origin/test/extended/util"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	e2e "k8s.io/kubernetes/test/e2e/framework"
 )
 
@@ -161,6 +162,14 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][Suite:openshift/two
 		e2e.Logf("[stage timing] Waiting for Machine nodeRef: %v (timeout cap: %v, poll: %v)", time.Since(stageStart), machineNodeRefWaitTimeout, machineNodeRefPollInterval)
 		stageStart = time.Now()
 
+		// Capture Node creation time for update-setup job pod timestamp filter.
+		// CEO may create the job as soon as the Node object exists (even before Ready),
+		// so we use creation time rather than Ready time as the minimum pod timestamp.
+		node, err := oc.AdminKubeClient().CoreV1().Nodes().Get(context.Background(), testConfig.TargetNode.Name, metav1.GetOptions{})
+		o.Expect(err).To(o.BeNil(), "Expected to get replacement Node %s after Machine has nodeRef", testConfig.TargetNode.Name)
+		nodeCreatedTime := node.CreationTimestamp.Time
+		e2e.Logf("Replacement Node %s created at %v (will use as minimum pod creation time for update-setup job)", testConfig.TargetNode.Name, nodeCreatedTime.UTC())
+
 		// cluster-machine-approver may leave Pending kube-apiserver-client-kubelet CSRs for same-node-name
 		// replacements (known product bug; add OCPBUGS-xxxx to apis.WaitForAndApproveNodeBootstrapperCSR when filed).
 		g.By("Waiting for node CSR to be approved (approving node-bootstrapper CSR if machine-approver has not)")
@@ -172,6 +181,7 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][Suite:openshift/two
 		readyTime, err := waitForNodeRecovery(&testConfig, oc, nodeReadyAfterCSRTimeout, utils.ThirtySecondPollInterval)
 		o.Expect(err).To(o.BeNil(), "Expected replacement node %s to appear and become Ready", testConfig.TargetNode.Name)
 		e2e.Logf("[stage timing] Node Ready: %v (timeout cap: %v, poll: %v)", time.Since(stageStart), nodeReadyAfterCSRTimeout, utils.ThirtySecondPollInterval)
+		e2e.Logf("Node created at %v, became Ready at %v (delta: %v)", nodeCreatedTime.UTC(), readyTime.UTC(), readyTime.Sub(nodeCreatedTime))
 		stageStart = time.Now()
 
 		g.By("Bumping kube-apiserver / KCM / scheduler revision so static pod installers re-run on the replaced node")
@@ -211,7 +221,7 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][Suite:openshift/two
 		stageStart = time.Now()
 
 		g.By("Restoring pacemaker cluster configuration")
-		restorePacemakerCluster(&testConfig, oc, readyTime)
+		restorePacemakerCluster(&testConfig, oc, nodeCreatedTime)
 		e2e.Logf("[stage timing] Restoring pacemaker cluster (total): %v (see sub-lines from restorePacemakerCluster for CEO job vs pcs online caps)", time.Since(stageStart))
 		stageStart = time.Now()
 

test/extended/edge_topologies/tnf_node_replacement_finish.go

@@ -26,7 +26,7 @@ import (
 	e2e "k8s.io/kubernetes/test/e2e/framework"
 )
 
-func restorePacemakerCluster(testConfig *TNFTestConfig, oc *exutil.CLI, nodeReadyTime time.Time) {
+func restorePacemakerCluster(testConfig *TNFTestConfig, oc *exutil.CLI, nodeCreatedTime time.Time) {
 	restorePCMStart := time.Now()
 	// Prepare known hosts file for the target node now that it has been reprovisioned
 	// The SSH key changed during reprovisioning, so we need to scan it again
@@ -42,12 +42,14 @@ func restorePacemakerCluster(testConfig *TNFTestConfig, oc *exutil.CLI, nodeRead
 	// - Old CEO: creates 2 per-node jobs; target node's job exits early, survivor does the work
 	// - New CEO: creates 1 cluster-wide job that runs on the survivor node
 	// Only wait for the survivor job since that's where the work happens in both cases.
-	minPodCreationTime := nodeReadyTime
-	e2e.Logf("Waiting for CEO update-setup job on survivor node %s", testConfig.SurvivingNode.Name)
+	// Use replacement Node creation time (not Ready time) as minimum pod timestamp because CEO
+	// may create the job as soon as the Node object appears, before it reaches Ready status.
+	minPodCreationTime := nodeCreatedTime
+	e2e.Logf("Waiting for CEO update-setup job on any node (pod created after replacement Node object created at %v)", nodeCreatedTime.UTC())
 	ceoJobsStart := time.Now()
-	errSurvivor := services.WaitForSurvivorUpdateSetupJobCompletionByNode(oc, services.EtcdNamespace, testConfig.SurvivingNode.Name, minPodCreationTime, ceoUpdateSetupJobWaitTimeout, utils.ThirtySecondPollInterval)
+	errSurvivor := services.WaitForUpdateSetupJobCompletion(oc, services.EtcdNamespace, minPodCreationTime, ceoUpdateSetupJobWaitTimeout, utils.ThirtySecondPollInterval)
 	e2e.Logf("[stage timing] restorePacemakerCluster: CEO update-setup job: %v (timeout cap: %v, poll: %v)", time.Since(ceoJobsStart), ceoUpdateSetupJobWaitTimeout, utils.ThirtySecondPollInterval)
-	o.Expect(errSurvivor).To(o.BeNil(), "Expected survivor update-setup job for node %s to complete (run after replacement node Ready)", testConfig.SurvivingNode.Name)
+	o.Expect(errSurvivor).To(o.BeNil(), "Expected update-setup job to complete on any node (run after replacement Node created)")
 
 	// Verify both nodes are online in the pacemaker cluster
 	e2e.Logf("Verifying both nodes are online in pacemaker cluster")

test/extended/edge_topologies/utils/services/etcd.go

@@ -300,14 +300,15 @@ func WaitForJobCompletion(jobName, namespace string, timeout, pollInterval time.
 	return err
 }
 
-// getUpdateSetupJobNameForNode returns the name of the TNF update-setup job that targets the given node,
-// or "" if no such job exists. CEO creates these jobs with a hash suffix in the name, so the test discovers
-// the actual name by listing jobs with the update-setup label and matching by node.
+// getUpdateSetupJobNameForNode returns the name of the TNF update-setup job targeting the given node,
+// or "" if no such job exists. If nodeName is empty, returns the newest job on any node (for round-robin retry).
+// CEO creates these jobs with a hash suffix in the name, so the test discovers the actual name by listing jobs
+// with the update-setup label and optionally matching by node.
 // When multiple jobs match (e.g. after node replacement), returns the newest by CreationTimestamp so we
 // wait for the job created after the node was recreated.
 // If minCreationTime is non-zero, only jobs with CreationTimestamp.After(minCreationTime) are considered,
-// ensuring we wait for a job created after the node replacement event (e.g. node Ready time).
-func getUpdateSetupJobNameForNode(oc *exutil.CLI, namespace, nodeName string, minCreationTime time.Time) (string, error) {
+// ensuring we wait for a job created after the node replacement event (e.g. Node creation time).
+func getUpdateSetupJobNameForNode(oc *exutil.CLI, namespace string, minCreationTime time.Time) (string, error) {
 	list, err := oc.AdminKubeClient().BatchV1().Jobs(namespace).List(context.Background(), metav1.ListOptions{
 		LabelSelector: tnfUpdateSetupJobLabelSelector,
 	})
@@ -317,9 +318,6 @@ func getUpdateSetupJobNameForNode(oc *exutil.CLI, namespace, nodeName string, mi
 	var newest *batchv1.Job
 	for i := range list.Items {
 		job := &list.Items[i]
-		if job.Spec.Template.Spec.NodeName != nodeName {
-			continue
-		}
 		if !minCreationTime.IsZero() && !job.CreationTimestamp.Time.After(minCreationTime) {
 			continue // require job created after node replacement event
 		}
@@ -333,21 +331,19 @@ func getUpdateSetupJobNameForNode(oc *exutil.CLI, namespace, nodeName string, mi
 	return newest.Name, nil
 }
 
-// WaitForUpdateSetupJobCompletionByNode waits for the TNF update-setup job targeting the given node to complete.
-// It discovers the job by label and node name so it works regardless of CEO's job naming (e.g. hash suffix).
-// If minJobCreationTime is non-zero, only a job created after that time is considered (ensures we wait for the
-// job created after the node was recreated, not a stale pre-replacement job).
-func WaitForUpdateSetupJobCompletionByNode(oc *exutil.CLI, namespace, nodeName string, minJobCreationTime time.Time, timeout, pollInterval time.Duration) error {
-	e2e.Logf("Waiting for update-setup job for node %s in namespace %s to complete (timeout: %v)", nodeName, namespace, timeout)
+// WaitForUpdateSetupJobCompletion waits for any TNF update-setup job to complete in a
+// run that started after minPodCreationTime, regardless of which node it ran on (round-robin retry may succeed on either node).
+func WaitForUpdateSetupJobCompletion(oc *exutil.CLI, namespace string, minPodCreationTime time.Time, timeout, pollInterval time.Duration) error {
+	e2e.Logf("Waiting for update-setup job (run after %v) to complete on any node (timeout: %v)", minPodCreationTime.UTC(), timeout)
 
 	var resolvedJobName string
 	err := core.PollUntil(func() (bool, error) {
-		name, err := getUpdateSetupJobNameForNode(oc, namespace, nodeName, minJobCreationTime)
+		name, err := getUpdateSetupJobNameForNode(oc, namespace, time.Time{}) // accept job on any node
 		if err != nil {
 			return false, err
 		}
 		if name == "" {
-			e2e.Logf("Update-setup job for node %s not found yet, waiting...", nodeName)
+			e2e.Logf("Update-setup job not found yet, waiting...")
 			return false, nil
 		}
 		resolvedJobName = name
@@ -360,55 +356,6 @@ func WaitForUpdateSetupJobCompletionByNode(oc *exutil.CLI, namespace, nodeName s
 			e2e.Logf("Job %s has no conditions yet, waiting...", name)
 			return false, nil
 		}
-		for _, cond := range job.Status.Conditions {
-			if cond.Type == batchv1.JobComplete && cond.Status == "True" {
-				e2e.Logf("Update-setup job %s for node %s completed successfully", name, nodeName)
-				return true, nil
-			}
-			if cond.Type == batchv1.JobFailed && cond.Status == "True" {
-				return false, core.NewError(fmt.Sprintf("job %s", name), fmt.Sprintf("failed: %s - %s", cond.Reason, cond.Message))
-			}
-		}
-		e2e.Logf("Job %s still running...", name)
-		return false, nil
-	}, timeout, pollInterval, fmt.Sprintf("update-setup job for node %s completion", nodeName))
-
-	if resolvedJobName != "" {
-		DumpJobPodLogs(resolvedJobName, namespace, oc)
-	}
-	return err
-}
-
-// WaitForSurvivorUpdateSetupJobCompletionByNode waits for the survivor's TNF update-setup job to complete in a
-// run that started after minPodCreationTime. It discovers the job by label and node name (see WaitForUpdateSetupJobCompletionByNode).
-func WaitForSurvivorUpdateSetupJobCompletionByNode(oc *exutil.CLI, namespace, nodeName string, minPodCreationTime time.Time, timeout, pollInterval time.Duration) error {
-	e2e.Logf("Waiting for survivor update-setup job for node %s (run after %v) to complete (timeout: %v)", nodeName, minPodCreationTime.UTC(), timeout)
-
-	var resolvedJobName string
-	err := core.PollUntil(func() (bool, error) {
-		name, err := getUpdateSetupJobNameForNode(oc, namespace, nodeName, time.Time{}) // survivor: any job name, we filter by pod time below
-		if err != nil {
-			return false, err
-		}
-		if name == "" {
-			e2e.Logf("Survivor update-setup job for node %s not found yet, waiting...", nodeName)
-			return false, nil
-		}
-		resolvedJobName = name
-		job, err := oc.AdminKubeClient().BatchV1().Jobs(namespace).Get(context.Background(), name, metav1.GetOptions{})
-		if err != nil {
-			e2e.Logf("Job %s not found, waiting...", name)
-			return false, nil
-		}
-		for _, cond := range job.Status.Conditions {
-			if cond.Type == batchv1.JobFailed && cond.Status == "True" {
-				return false, core.NewError(fmt.Sprintf("job %s", name), fmt.Sprintf("failed: %s - %s", cond.Reason, cond.Message))
-			}
-		}
-		if len(job.Status.Conditions) == 0 {
-			e2e.Logf("Job %s has no conditions yet, waiting...", name)
-			return false, nil
-		}
 		for _, cond := range job.Status.Conditions {
 			if cond.Type == batchv1.JobComplete && cond.Status == "True" {
 				selector := "job-name=" + name
@@ -419,17 +366,17 @@ func WaitForSurvivorUpdateSetupJobCompletionByNode(oc *exutil.CLI, namespace, no
 				}
 				for i := range podList.Items {
 					if !podList.Items[i].CreationTimestamp.Time.Before(minPodCreationTime) {
-						e2e.Logf("Survivor job %s completed in a run after target Ready (pod %s created at %v)", name, podList.Items[i].Name, podList.Items[i].CreationTimestamp.UTC())
+						e2e.Logf("Update-setup job %s completed in a run after replacement Node created (pod %s on node %s created at %v)", name, podList.Items[i].Name, podList.Items[i].Spec.NodeName, podList.Items[i].CreationTimestamp.UTC())
 						return true, nil
 					}
 				}
-				e2e.Logf("Job %s completed but run started before replacement node was Ready; waiting for a fresh run", name)
+				e2e.Logf("Job %s completed but run started before replacement Node was created; waiting for a fresh run", name)
 				return false, nil
 			}
 		}
 		e2e.Logf("Job %s still running...", name)
 		return false, nil
-	}, timeout, pollInterval, fmt.Sprintf("survivor update-setup job for node %s completion (run after %v)", nodeName, minPodCreationTime.UTC()))
+	}, timeout, pollInterval, fmt.Sprintf("update-setup job completion on any node (run after %v)", minPodCreationTime.UTC()))
 
 	if resolvedJobName != "" {
 		DumpJobPodLogs(resolvedJobName, namespace, oc)