diff --git a/terratest/test/k8gb_failover_playground_test.go b/terratest/test/k8gb_failover_playground_test.go index 193a68035d..cf3b006d47 100644 --- a/terratest/test/k8gb_failover_playground_test.go +++ b/terratest/test/k8gb_failover_playground_test.go @@ -72,11 +72,13 @@ func TestFailoverPlayground(t *testing.T) { t.Run("stop podinfo on eu cluster", func(t *testing.T) { instanceEU.StopTestApp() + require.NoError(t, instanceEU.WaitForAppIsStopped()) actAndAssert(t.Name(), usGeoTag, usLocalTargets) }) t.Run("start podinfo again on eu cluster", func(t *testing.T) { instanceEU.StartTestApp() + require.NoError(t, instanceEU.WaitForAppIsRunning()) actAndAssert(t.Name(), euGeoTag, euLocalTargets) }) } diff --git a/terratest/utils/extensions.go b/terratest/utils/extensions.go index c6d6463df9..28e0d4d966 100644 --- a/terratest/utils/extensions.go +++ b/terratest/utils/extensions.go @@ -42,7 +42,7 @@ import ( type Workflow struct { error error namespace string - cluster string + cluster string k8sOptions *k8s.KubectlOptions t *testing.T settings struct { @@ -92,10 +92,10 @@ type InstanceStatus struct { GslbHealthStatus string `json:"gslb-status"` Cluster string `json:"cluster"` Namespace string `json:"namespace"` - Endpoint0DNSName string `json:"ep0-dns-name"` - Endpoint0Targets string `json:"ep0-dns-targets"` - Endpoint1DNSName string `json:"ep1-dns-name"` - Endpoint1Targets string `json:"ep1-dns-targets"` + Endpoint0DNSName string `json:"ep0-dns-name"` + Endpoint0Targets string `json:"ep0-dns-targets"` + Endpoint1DNSName string `json:"ep1-dns-name"` + Endpoint1Targets string `json:"ep1-dns-targets"` } func NewWorkflow(t *testing.T, cluster string, port int) *Workflow { @@ -297,24 +297,41 @@ func (i *Instance) WaitForGSLB(instances ...*Instance) ([]string, error) { func (i *Instance) WaitForExpected(expectedIPs []string) (err error) { _, err = waitForLocalGSLBNew(i.w.t, i.w.state.gslb.host, i.w.state.gslb.port, expectedIPs) if err != nil { - fmt.Println(i.GetStatus(fmt.Sprintf("expected IPs: %s",expectedIPs)).String()) + fmt.Println(i.GetStatus(fmt.Sprintf("expected IPs: %s", expectedIPs)).String()) } return } // WaitForAppIsRunning waits until app has one pod running func (i *Instance) WaitForAppIsRunning() (err error) { - f := func()([]string, error){ - r := i.GetStatus("").AppReplicas - return []string{r}, nil + return i.waitForApp(func(instances int) bool { return instances > 0 }) +} + +// WaitForAppIsStopped waits until app has 0 pods running +func (i *Instance) WaitForAppIsStopped() (err error) { + return i.waitForApp(func(instances int) bool { return instances == 0 }) +} + +// WaitForAppIsRunning waits until app has one pod running +func (i *Instance) waitForApp(action func(instances int) bool) (err error) { + const description = "Wait for app is running" + for n := 0; n < 25; n++ { + var r int + rep := i.GetStatus("").AppReplicas + // r == 0 + if rep != "" { + r, err = strconv.Atoi(rep) + if err != nil { + i.w.t.Logf("%s returned an error: %s.", description, err) + return err + } + } + if action(r) { + i.w.t.Logf("%s found match: Expected:(%v)", description, r) + return nil + } + i.w.t.Logf("Application %s is not in expected state. Waiting...", i.w.state.testApp.name) } - _, err = DoWithRetryWaitingForValueE( - i.w.t, - "Wait for failover to happen and coredns to pickup new values...", - 100, - time.Second*1, - func() ([]string, error) { return f() }, - []string{"1"}) return } @@ -359,7 +376,7 @@ func (i *Instance) HitTestApp() (result *TestAppResult) { // GetStatus reads overall status about instance. Status can be used for assertion as well as printed to test output // Annotation argument is just free text which might be used in various test scenarios -func (i *Instance) GetStatus(annotation string) (s *InstanceStatus){ +func (i *Instance) GetStatus(annotation string) (s *InstanceStatus) { const na = "n/a" var err error s = new(InstanceStatus) @@ -372,8 +389,8 @@ func (i *Instance) GetStatus(annotation string) (s *InstanceStatus){ s.CoreDNS = i.GetCoreDNSIP() s.AppMessage = i.w.state.testApp.message s.AppRunning = i.w.state.testApp.isRunning - s.AppReplicas, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "deployments","frontend-podinfo", - "-o","custom-columns=STATUS:.status.replicas", "--no-headers") + s.AppReplicas, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "deployments", "frontend-podinfo", + "-o", "custom-columns=STATUS:.status.readyReplicas", "--no-headers") if err != nil { s.AppReplicas = na } @@ -382,22 +399,22 @@ func (i *Instance) GetStatus(annotation string) (s *InstanceStatus){ if err != nil { s.GslbHealthStatus = na } - s.Endpoint0DNSName,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o", + s.Endpoint0DNSName, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "dnsendpoints.externaldns.k8s.io", "test-gslb", "-o", "custom-columns=SERVICESTATUS:.spec.endpoints[0].dnsName", "--no-headers") if err != nil { s.Endpoint0DNSName = na } - s.Endpoint0Targets,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o", + s.Endpoint0Targets, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "dnsendpoints.externaldns.k8s.io", "test-gslb", "-o", "custom-columns=SERVICESTATUS:.spec.endpoints[0].targets", "--no-headers") if err != nil { s.Endpoint0Targets = na } - s.Endpoint1DNSName,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o", + s.Endpoint1DNSName, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "dnsendpoints.externaldns.k8s.io", "test-gslb", "-o", "custom-columns=SERVICESTATUS:.spec.endpoints[1].dnsName", "--no-headers") if err != nil { s.Endpoint1DNSName = na } - s.Endpoint1Targets,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o", + s.Endpoint1Targets, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "dnsendpoints.externaldns.k8s.io", "test-gslb", "-o", "custom-columns=SERVICESTATUS:.spec.endpoints[1].targets", "--no-headers") if err != nil { s.Endpoint1Targets = na