Merge pull request kubernetes#21586 from janetkuo/retry-update-e2e

Retry all Update in deployment e2e tests
bnookala · Feb 19, 2016 · 6766f11 · 6766f11
2 parents 6a1156d + 98a1eaf
commit 6766f11
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 17 deletions.
diff --git a/test/e2e/deployment.go b/test/e2e/deployment.go
@@ -484,10 +484,10 @@ func testRolloverDeployment(f *Framework) {
 	// If the deployment already finished here, the test would fail. When this happens, increase its minReadySeconds or replicas to prevent it.
 	Expect(newRS.Spec.Replicas).Should(BeNumerically("<", deploymentReplicas))
 	updatedDeploymentImage := "redis"
-	newDeployment.Spec.Template.Spec.Containers[0].Name = updatedDeploymentImage
-	newDeployment.Spec.Template.Spec.Containers[0].Image = updatedDeploymentImage
-	Logf("updating deployment %s", deploymentName)
-	_, err = c.Extensions().Deployments(ns).Update(newDeployment)
+	_, err = updateDeploymentWithRetries(c, ns, newDeployment.Name, func(update *extensions.Deployment) {
+		update.Spec.Template.Spec.Containers[0].Name = updatedDeploymentImage
+		update.Spec.Template.Spec.Containers[0].Image = updatedDeploymentImage
+	})
 	Expect(err).NotTo(HaveOccurred())
 
 	err = waitForDeploymentStatus(c, ns, deploymentName, deploymentReplicas, deploymentReplicas-1, deploymentReplicas+1, deploymentMinReadySeconds)
@@ -524,8 +524,9 @@ func testPausedDeployment(f *Framework) {
 	}
 
 	// Update the deployment to run
-	deployment.Spec.Paused = false
-	deployment, err = c.Extensions().Deployments(ns).Update(deployment)
+	deployment, err = updateDeploymentWithRetries(c, ns, d.Name, func(update *extensions.Deployment) {
+		update.Spec.Paused = false
+	})
 	Expect(err).NotTo(HaveOccurred())
 
 	selector, err := unversioned.LabelSelectorAsSelector(deployment.Spec.Selector)
@@ -546,9 +547,9 @@ func testPausedDeployment(f *Framework) {
 
 	// Pause the deployment and delete the replica set.
 	// The paused deployment shouldn't recreate a new one.
-	deployment.Spec.Paused = true
-	deployment.ResourceVersion = ""
-	deployment, err = c.Extensions().Deployments(ns).Update(deployment)
+	deployment, err = updateDeploymentWithRetries(c, ns, d.Name, func(update *extensions.Deployment) {
+		update.Spec.Paused = true
+	})
 	Expect(err).NotTo(HaveOccurred())
 
 	newRS, err := deploymentutil.GetNewReplicaSet(*deployment, c)
@@ -613,10 +614,10 @@ func testRollbackDeployment(f *Framework) {
 	// Update the deployment to create redis pods.
 	updatedDeploymentImage := "redis"
 	updatedDeploymentImageName := "redis"
-	d.Spec.Template.Spec.Containers[0].Name = updatedDeploymentImageName
-	d.Spec.Template.Spec.Containers[0].Image = updatedDeploymentImage
-	Logf("updating deployment %s", deploymentName)
-	_, err = c.Extensions().Deployments(ns).Update(d)
+	_, err = updateDeploymentWithRetries(c, ns, d.Name, func(update *extensions.Deployment) {
+		update.Spec.Template.Spec.Containers[0].Name = updatedDeploymentImageName
+		update.Spec.Template.Spec.Containers[0].Image = updatedDeploymentImage
+	})
 	Expect(err).NotTo(HaveOccurred())
 
 	err = waitForDeploymentStatus(c, ns, deploymentName, deploymentReplicas, deploymentReplicas-1, deploymentReplicas+1, 0)
@@ -731,10 +732,10 @@ func testRollbackDeploymentRSNoRevision(f *Framework) {
 	// Update the deployment to create redis pods.
 	updatedDeploymentImage := "redis"
 	updatedDeploymentImageName := "redis"
-	d.Spec.Template.Spec.Containers[0].Name = updatedDeploymentImageName
-	d.Spec.Template.Spec.Containers[0].Image = updatedDeploymentImage
-	Logf("updating deployment %s", deploymentName)
-	_, err = c.Extensions().Deployments(ns).Update(d)
+	_, err = updateDeploymentWithRetries(c, ns, d.Name, func(update *extensions.Deployment) {
+		update.Spec.Template.Spec.Containers[0].Name = updatedDeploymentImageName
+		update.Spec.Template.Spec.Containers[0].Image = updatedDeploymentImage
+	})
 	Expect(err).NotTo(HaveOccurred())
 
 	err = waitForDeploymentStatus(c, ns, deploymentName, deploymentReplicas, deploymentReplicas-1, deploymentReplicas+1, 0)

diff --git a/test/e2e/util.go b/test/e2e/util.go
@@ -2204,6 +2204,25 @@ func waitForPartialEvents(c *client.Client, ns string, objOrRef runtime.Object,
 	})
 }
 
+type updateDeploymentFunc func(d *extensions.Deployment)
+
+func updateDeploymentWithRetries(c *clientset.Clientset, namespace, name string, applyUpdate updateDeploymentFunc) (deployment *extensions.Deployment, err error) {
+	deployments := c.Extensions().Deployments(namespace)
+	err = wait.Poll(10*time.Millisecond, 1*time.Minute, func() (bool, error) {
+		if deployment, err = deployments.Get(name); err != nil {
+			return false, err
+		}
+		// Apply the update, then attempt to push it to the apiserver.
+		applyUpdate(deployment)
+		if deployment, err = deployments.Update(deployment); err == nil {
+			Logf("updating deployment %s", name)
+			return true, nil
+		}
+		return false, nil
+	})
+	return deployment, err
+}
+
 // FailedContainers inspects all containers in a pod and returns failure
 // information for containers that have failed or been restarted.
 // A map is returned where the key is the containerID and the value is a