Skip to content

Commit

Permalink
http failover, wait for app (#546)
Browse files Browse the repository at this point in the history 
Small change, the test waits until the application starts or is stopped.

I also changed the implementation of the wait for app function. Now it can wait until the application starts as well as it can wait for it to stop.

Signed-off-by: kuritka <[email protected]>
  • Loading branch information
@kuritka
kuritka authored Jul 22, 2021
1 parent 8253842 commit f84bfe0
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 23 deletions.
2 changes: 2 additions & 0 deletions terratest/test/k8gb_failover_playground_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,13 @@ func TestFailoverPlayground(t *testing.T) {

t.Run("stop podinfo on eu cluster", func(t *testing.T) {
instanceEU.StopTestApp()
require.NoError(t, instanceEU.WaitForAppIsStopped())
actAndAssert(t.Name(), usGeoTag, usLocalTargets)
})

t.Run("start podinfo again on eu cluster", func(t *testing.T) {
instanceEU.StartTestApp()
require.NoError(t, instanceEU.WaitForAppIsRunning())
actAndAssert(t.Name(), euGeoTag, euLocalTargets)
})
}
63 changes: 40 additions & 23 deletions terratest/utils/extensions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import (
type Workflow struct {
error error
namespace string
cluster string
cluster string
k8sOptions *k8s.KubectlOptions
t *testing.T
settings struct {
Expand Down Expand Up @@ -92,10 +92,10 @@ type InstanceStatus struct {
GslbHealthStatus string `json:"gslb-status"`
Cluster string `json:"cluster"`
Namespace string `json:"namespace"`
Endpoint0DNSName string `json:"ep0-dns-name"`
Endpoint0Targets string `json:"ep0-dns-targets"`
Endpoint1DNSName string `json:"ep1-dns-name"`
Endpoint1Targets string `json:"ep1-dns-targets"`
Endpoint0DNSName string `json:"ep0-dns-name"`
Endpoint0Targets string `json:"ep0-dns-targets"`
Endpoint1DNSName string `json:"ep1-dns-name"`
Endpoint1Targets string `json:"ep1-dns-targets"`
}

func NewWorkflow(t *testing.T, cluster string, port int) *Workflow {
Expand Down Expand Up @@ -297,24 +297,41 @@ func (i *Instance) WaitForGSLB(instances ...*Instance) ([]string, error) {
func (i *Instance) WaitForExpected(expectedIPs []string) (err error) {
_, err = waitForLocalGSLBNew(i.w.t, i.w.state.gslb.host, i.w.state.gslb.port, expectedIPs)
if err != nil {
fmt.Println(i.GetStatus(fmt.Sprintf("expected IPs: %s",expectedIPs)).String())
fmt.Println(i.GetStatus(fmt.Sprintf("expected IPs: %s", expectedIPs)).String())
}
return
}

// WaitForAppIsRunning waits until app has one pod running
func (i *Instance) WaitForAppIsRunning() (err error) {
f := func()([]string, error){
r := i.GetStatus("").AppReplicas
return []string{r}, nil
return i.waitForApp(func(instances int) bool { return instances > 0 })
}

// WaitForAppIsStopped waits until app has 0 pods running
func (i *Instance) WaitForAppIsStopped() (err error) {
return i.waitForApp(func(instances int) bool { return instances == 0 })
}

// WaitForAppIsRunning waits until app has one pod running
func (i *Instance) waitForApp(action func(instances int) bool) (err error) {
const description = "Wait for app is running"
for n := 0; n < 25; n++ {
var r int
rep := i.GetStatus("").AppReplicas
// r == 0
if rep != "<none>" {
r, err = strconv.Atoi(rep)
if err != nil {
i.w.t.Logf("%s returned an error: %s.", description, err)
return err
}
}
if action(r) {
i.w.t.Logf("%s found match: Expected:(%v)", description, r)
return nil
}
i.w.t.Logf("Application %s is not in expected state. Waiting...", i.w.state.testApp.name)
}
_, err = DoWithRetryWaitingForValueE(
i.w.t,
"Wait for failover to happen and coredns to pickup new values...",
100,
time.Second*1,
func() ([]string, error) { return f() },
[]string{"1"})
return
}

Expand Down Expand Up @@ -359,7 +376,7 @@ func (i *Instance) HitTestApp() (result *TestAppResult) {

// GetStatus reads overall status about instance. Status can be used for assertion as well as printed to test output
// Annotation argument is just free text which might be used in various test scenarios
func (i *Instance) GetStatus(annotation string) (s *InstanceStatus){
func (i *Instance) GetStatus(annotation string) (s *InstanceStatus) {
const na = "n/a"
var err error
s = new(InstanceStatus)
Expand All @@ -372,8 +389,8 @@ func (i *Instance) GetStatus(annotation string) (s *InstanceStatus){
s.CoreDNS = i.GetCoreDNSIP()
s.AppMessage = i.w.state.testApp.message
s.AppRunning = i.w.state.testApp.isRunning
s.AppReplicas, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "deployments","frontend-podinfo",
"-o","custom-columns=STATUS:.status.replicas", "--no-headers")
s.AppReplicas, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "deployments", "frontend-podinfo",
"-o", "custom-columns=STATUS:.status.readyReplicas", "--no-headers")
if err != nil {
s.AppReplicas = na
}
Expand All @@ -382,22 +399,22 @@ func (i *Instance) GetStatus(annotation string) (s *InstanceStatus){
if err != nil {
s.GslbHealthStatus = na
}
s.Endpoint0DNSName,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o",
s.Endpoint0DNSName, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "dnsendpoints.externaldns.k8s.io", "test-gslb", "-o",
"custom-columns=SERVICESTATUS:.spec.endpoints[0].dnsName", "--no-headers")
if err != nil {
s.Endpoint0DNSName = na
}
s.Endpoint0Targets,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o",
s.Endpoint0Targets, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "dnsendpoints.externaldns.k8s.io", "test-gslb", "-o",
"custom-columns=SERVICESTATUS:.spec.endpoints[0].targets", "--no-headers")
if err != nil {
s.Endpoint0Targets = na
}
s.Endpoint1DNSName,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o",
s.Endpoint1DNSName, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "dnsendpoints.externaldns.k8s.io", "test-gslb", "-o",
"custom-columns=SERVICESTATUS:.spec.endpoints[1].dnsName", "--no-headers")
if err != nil {
s.Endpoint1DNSName = na
}
s.Endpoint1Targets,err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions,"get","dnsendpoints.externaldns.k8s.io","test-gslb","-o",
s.Endpoint1Targets, err = k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "dnsendpoints.externaldns.k8s.io", "test-gslb", "-o",
"custom-columns=SERVICESTATUS:.spec.endpoints[1].targets", "--no-headers")
if err != nil {
s.Endpoint1Targets = na
Expand Down

0 comments on commit f84bfe0

Please sign in to comment.