Expose CoreDNS over tcp + make Dig in terratests go via tcp

Makefile

@@ -146,6 +146,7 @@ deploy-test-version: ## Upgrade k8gb to the test version on existing clusters
 
 	@for c in $(CLUSTER_IDS); do \
 		$(MAKE) deploy-local-cluster CLUSTER_ID=$$c VERSION=$(SEMVER)-amd64 CHART='./chart/k8gb' ;\
+		kubectl apply -n k8gb -f ./deploy/test/coredns-tcp-svc.yaml ;\
 	done
 
 .PHONY: list-running-pods

controllers/gslb_controller_test.go

@@ -30,7 +30,7 @@ import (
 
 	"github.com/golang/mock/gomock"
 
-	str "github.com/AbsaOSS/gopkg/strings"
+	str "github.com/AbsaOSS/gopkg/string"
 	k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1"
 	"github.com/k8gb-io/k8gb/controllers/depresolver"
 	"github.com/k8gb-io/k8gb/controllers/internal/utils"

deploy/test/coredns-tcp-svc.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: k8gb-coredns-tcp
+spec:
+  type: NodePort
+  selector:
+    app.kubernetes.io/instance: k8gb
+    app.kubernetes.io/name: coredns
+  ports:
+  - name: tcp-5353
+    port: 53
+    protocol: TCP
+    targetPort: 5353
+    nodePort: 30053

docs/local.md

@@ -71,7 +71,7 @@ Cluster [edgedns](https://github.com/k8gb-io/k8gb/tree/master/k3d/edge-dns.yaml)
 on port `:1053`.
 
 ```sh
-dig @localhost -p 1053 roundrobin.cloud.example.com +short
+dig @localhost -p 1053 roundrobin.cloud.example.com +short +tcp
 ```
 Should return ***two A records*** from both clusters (IP addresses and order may differ):
 ```
@@ -98,7 +98,8 @@ k3d-test-gslb2-agent-0    172.20.0.5
 
 Or you can ask specific CoreDNS instance for its local targets:
 ```sh
-dig @localhost localtargets-roundrobin.cloud.example.com -p 5053 && dig -p 5054 @localhost localtargets-roundrobin.cloud.example.com
+dig -p 5053 +tcp @localhost localtargets-roundrobin.cloud.example.com && \
+dig -p 5054 +tcp @localhost localtargets-roundrobin.cloud.example.com
 ```
 As expected result you should see **two A records** divided between both clusters.
 ```sh

go.mod

@@ -4,7 +4,7 @@ go 1.17
 
 require (
 	github.com/AbsaOSS/env-binder v1.0.0
-	github.com/AbsaOSS/gopkg v0.1.2
+	github.com/AbsaOSS/gopkg v0.1.3
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-logr/logr v1.2.0
 	github.com/golang/mock v1.6.0

go.sum

@@ -56,8 +56,8 @@ git.lukeshu.com/go/libsystemd v0.5.3/go.mod h1:FfDoP0i92r4p5Vn4NCLxvjkd7rCOe6otP
 github.com/0x4c6565/genie v1.0.0/go.mod h1:fDOjW0hFamMWOIkh4irf2D/TZpXXWMFtpP8MfgK0N3c=
 github.com/AbsaOSS/env-binder v1.0.0 h1:TwDcooL87BdR7NTsG2XbQpIj7KrhPU7Es7vxDKHJUB8=
 github.com/AbsaOSS/env-binder v1.0.0/go.mod h1:B48qTr35jCMHNqghPQxqaXdom2s9O3aOk3dOaWI7OQU=
-github.com/AbsaOSS/gopkg v0.1.2 h1:jYLOqZC23vDi5cH4G2dnaQSzFO3RNb2V7NXa4yQ3vVY=
-github.com/AbsaOSS/gopkg v0.1.2/go.mod h1:6bJ9NNcX+xvAAv2c31HPGe+P04RKdm/NV/V+qqyJrlQ=
+github.com/AbsaOSS/gopkg v0.1.3 h1:pTeIn3u9cFsHo5PFUGYvKUu9kIetLwa0ldhzEdZRUhU=
+github.com/AbsaOSS/gopkg v0.1.3/go.mod h1:6bJ9NNcX+xvAAv2c31HPGe+P04RKdm/NV/V+qqyJrlQ=
 github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go v56.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go v61.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=

k3d/gslb.yaml.tmpl

@@ -14,6 +14,9 @@ ports:
   - port: $PORT_PROM:30090
     nodeFilters:
       - agent:0:direct
+  - port: $PORT_DNS:30053/tcp
+    nodeFilters:
+      - agent:0:direct
   - port: $PORT_DNS:53/udp
     nodeFilters:
       - agent:0:direct

k3d/test-gslb1.yaml

@@ -17,6 +17,9 @@ ports:
   - port: 9080:30090
     nodeFilters:
       - agent:0:direct
+  - port: 5053:30053/tcp
+    nodeFilters:
+      - agent:0:direct
   - port: 5053:53/udp
     nodeFilters:
       - agent:0:direct

k3d/test-gslb2.yaml

@@ -14,6 +14,9 @@ ports:
   - port: 9081:30090
     nodeFilters:
       - agent:0:direct
+  - port: 5054:30053/tcp
+    nodeFilters:
+      - agent:0:direct
   - port: 5054:53/udp
     nodeFilters:
       - agent:0:direct

k3d/test-gslb3.yaml

@@ -14,6 +14,9 @@ ports:
   - port: 9082:30090
     nodeFilters:
       - agent:0:direct
+  - port: 5055:30053/tcp
+    nodeFilters:
+      - agent:0:direct
   - port: 5055:53/udp
     nodeFilters:
       - agent:0:direct

terratest/go.mod

@@ -3,7 +3,7 @@ module k8gbterratest
 go 1.17
 
 require (
-	github.com/AbsaOSS/gopkg v0.1.2
+	github.com/AbsaOSS/gopkg v0.1.3
 	github.com/gruntwork-io/terratest v0.38.6
 	github.com/stretchr/testify v1.7.0
 	gopkg.in/yaml.v2 v2.4.0

terratest/go.sum

@@ -41,6 +41,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/AbsaOSS/gopkg v0.1.2 h1:jYLOqZC23vDi5cH4G2dnaQSzFO3RNb2V7NXa4yQ3vVY=
 github.com/AbsaOSS/gopkg v0.1.2/go.mod h1:6bJ9NNcX+xvAAv2c31HPGe+P04RKdm/NV/V+qqyJrlQ=
+github.com/AbsaOSS/gopkg v0.1.3 h1:pTeIn3u9cFsHo5PFUGYvKUu9kIetLwa0ldhzEdZRUhU=
+github.com/AbsaOSS/gopkg v0.1.3/go.mod h1:6bJ9NNcX+xvAAv2c31HPGe+P04RKdm/NV/V+qqyJrlQ=
 github.com/Azure/azure-sdk-for-go v50.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=

terratest/test/init.go

@@ -32,7 +32,7 @@ func init() {
 	p1, _ := env.GetEnvAsIntOrFallback("DNS_SERVER1_PORT", 5053)
 	p2, _ := env.GetEnvAsIntOrFallback("DNS_SERVER2_PORT", 5054)
 	p3, _ := env.GetEnvAsIntOrFallback("DNS_SERVER3_PORT", 5055)
-	clNum, _ :=  env.GetEnvAsIntOrFallback("CLUSTERS_NUMBER", 2)
+	clNum, _ := env.GetEnvAsIntOrFallback("CLUSTERS_NUMBER", 2)
 	settings = utils.TestSettings{
 		DNSZone:         env.GetEnvAsStringOrFallback("GSLB_DOMAIN", "cloud.example.com"),
 		PrimaryGeoTag:   env.GetEnvAsStringOrFallback("PRIMARY_GEO_TAG", "eu"),
@@ -48,5 +48,6 @@ func init() {
 		Cluster3:        env.GetEnvAsStringOrFallback("K8GB_CLUSTER3", "k3d-test-gslb3"),
 		PodinfoImage:    env.GetEnvAsStringOrFallback("PODINFO_IMAGE_REPO", "ghcr.io/stefanprodan/podinfo"),
 		ClustersNumber:  clNum,
+		DigUsingUDP:     env.GetEnvAsBoolOrFallback("DIG_USING_UDP", false),
 	}
 }

terratest/test/k8gb_abstract_full_roundrobin_test.go

@@ -39,8 +39,8 @@ func abstractTestFullRoundRobin(t *testing.T, n int) {
 	const gslbPath = "../examples/roundrobin2.yaml"
 
 	// start all the test apps on all the clusters
-	for i := 0; i < n; i+=1 {
-		instance, er := utils.NewWorkflow(t, fmt.Sprintf("k3d-test-gslb%d", i+1), 5053 + i).
+	for i := 0; i < n; i += 1 {
+		instance, er := utils.NewWorkflow(t, fmt.Sprintf("k3d-test-gslb%d", i+1), 5053+i).
 			WithGslb(gslbPath, host).
 			WithTestApp(tags[i]).
 			Start()
@@ -50,15 +50,15 @@ func abstractTestFullRoundRobin(t *testing.T, n int) {
 	}
 	var err error
 	t.Run(fmt.Sprintf("round-robin on %d concurrent clusters with podinfo running", n), func(t *testing.T) {
-		for _, ins  := range instances {
+		for _, ins := range instances {
 			err = ins.WaitForAppIsRunning()
 			require.NoError(t, err)
 		}
 	})
 
 	// at the beginning, it should contain all the targets
 	var workingTargets []string
-	for _, ins  := range instances {
+	for _, ins := range instances {
 		workingTargets = append(workingTargets, ins.GetLocalTargets()...)
 	}
 	t.Run(fmt.Sprintf("all %d clusters should be interconnected", n), func(t *testing.T) {
@@ -67,7 +67,7 @@ func abstractTestFullRoundRobin(t *testing.T, n int) {
 
 	// kill the apps on clusters one by one and expect less and less targets to be available
 	for i, instance := range instances {
-		t.Run(fmt.Sprintf("kill podinfo on cluster %d (%s)", i + 1, tags[i]), func(t *testing.T) {
+		t.Run(fmt.Sprintf("kill podinfo on cluster %d (%s)", i+1, tags[i]), func(t *testing.T) {
 			workingTargets = distinct(subtract(workingTargets, instance.GetLocalTargets()))
 			t.Logf("New expected targets: %v", workingTargets)
 			instance.StopTestApp()
@@ -77,7 +77,7 @@ func abstractTestFullRoundRobin(t *testing.T, n int) {
 
 	// start the test apps again on each cluster and check if the targets start appearing
 	for i, instance := range instances {
-		t.Run(fmt.Sprintf("start podinfo on cluster %d (%s)", i + 1, tags[i]), func(t *testing.T) {
+		t.Run(fmt.Sprintf("start podinfo on cluster %d (%s)", i+1, tags[i]), func(t *testing.T) {
 			instance.StartTestApp()
 			workingTargets = distinct(append(workingTargets, instance.GetLocalTargets()...))
 			t.Logf("New expected targets: %v", workingTargets)

terratest/test/k8gb_basic_app_test.go

@@ -1,3 +1,4 @@
+//go:build basic || all
 // +build basic all
 
 package test

terratest/test/k8gb_failover_playground_test.go

@@ -1,3 +1,4 @@
+//go:build failover || all
 // +build failover all
 
 package test

terratest/test/k8gb_full_failover_test.go

@@ -1,3 +1,4 @@
+//go:build failover || all
 // +build failover all
 
 package test

terratest/test/k8gb_full_roundrobin_test.go

@@ -1,3 +1,4 @@
+//go:build rr || rr_multicluster
 // +build rr rr_multicluster
 
 package test

terratest/test/k8gb_ingress_annotation_failover_test.go

@@ -1,3 +1,4 @@
+//go:build failover || all
 // +build failover all
 
 package test

terratest/test/k8gb_ingress_annotation_rr_test.go

@@ -1,3 +1,4 @@
+//go:build rr || all
 // +build rr all
 
 package test

terratest/test/k8gb_lifecycle_test.go

@@ -1,3 +1,4 @@
+//go:build lifecycle || all
 // +build lifecycle all
 
 package test

terratest/test/k8gb_split_failover_test.go

@@ -1,3 +1,4 @@
+//go:build failover || all
 // +build failover all
 
 package test
@@ -74,12 +75,12 @@ func TestK8gbSplitFailoverExample(t *testing.T) {
 	expectedIPsCluster2 := utils.GetIngressIPs(t, optionsContext2, gslbName)
 
 	t.Run("Each cluster resolves its own set of IP addresses", func(t *testing.T) {
-		beforeFailoverResponseCluster1, err := utils.WaitForLocalGSLB(t, settings.DNSServer1, settings.Port1, "terratest-failover-split."+settings.DNSZone, expectedIPsCluster1)
+		beforeFailoverResponseCluster1, err := utils.WaitForLocalGSLB(t, settings.DNSServer1, settings.Port1, settings, "terratest-failover-split.", expectedIPsCluster1)
 		require.NoError(t, err)
 
 		assert.Equal(t, beforeFailoverResponseCluster1, expectedIPsCluster1)
 
-		beforeFailoverResponseCluster2, err := utils.WaitForLocalGSLB(t, settings.DNSServer2, settings.Port2, "terratest-failover-split."+settings.DNSZone, expectedIPsCluster2)
+		beforeFailoverResponseCluster2, err := utils.WaitForLocalGSLB(t, settings.DNSServer2, settings.Port2, settings, "terratest-failover-split.", expectedIPsCluster2)
 		require.NoError(t, err)
 
 		assert.Equal(t, beforeFailoverResponseCluster2, expectedIPsCluster2)
@@ -93,14 +94,14 @@ func TestK8gbSplitFailoverExample(t *testing.T) {
 	})
 
 	t.Run("Cluster 1 failovers to Cluster 2", func(t *testing.T) {
-		afterFailoverResponse, err := utils.WaitForLocalGSLB(t, settings.DNSServer1, settings.Port1, "terratest-failover-split."+settings.DNSZone, expectedIPsCluster2)
+		afterFailoverResponse, err := utils.WaitForLocalGSLB(t, settings.DNSServer1, settings.Port1, settings, "terratest-failover-split.", expectedIPsCluster2)
 		require.NoError(t, err)
 
 		assert.Equal(t, afterFailoverResponse, expectedIPsCluster2)
 	})
 
 	t.Run("Cluster 2 still returns own entries", func(t *testing.T) {
-		afterFailoverResponse, err := utils.WaitForLocalGSLB(t, settings.DNSServer2, settings.Port2, "terratest-failover-split."+settings.DNSZone, expectedIPsCluster2)
+		afterFailoverResponse, err := utils.WaitForLocalGSLB(t, settings.DNSServer2, settings.Port2, settings, "terratest-failover-split.", expectedIPsCluster2)
 		require.NoError(t, err)
 
 		assert.Equal(t, afterFailoverResponse, expectedIPsCluster2)
@@ -114,7 +115,7 @@ func TestK8gbSplitFailoverExample(t *testing.T) {
 	})
 
 	t.Run("Cluster 1 returns own entries again", func(t *testing.T) {
-		afterFailoverResponse, err := utils.WaitForLocalGSLB(t, settings.DNSServer1, settings.Port1, "terratest-failover-split."+settings.DNSZone, expectedIPsCluster1)
+		afterFailoverResponse, err := utils.WaitForLocalGSLB(t, settings.DNSServer1, settings.Port1, settings, "terratest-failover-split.", expectedIPsCluster1)
 		require.NoError(t, err)
 
 		assert.Equal(t, afterFailoverResponse, expectedIPsCluster1)

terratest/utils/extensions.go

@@ -29,7 +29,7 @@ import (
 	"time"
 
 	"github.com/AbsaOSS/gopkg/dns"
-	gopkgstr "github.com/AbsaOSS/gopkg/strings"
+	gopkgstr "github.com/AbsaOSS/gopkg/string"
 	"github.com/gruntwork-io/terratest/modules/helm"
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/random"
@@ -50,6 +50,7 @@ type Workflow struct {
 		ingressResourcePath string
 		gslbResourcePath    string
 		ingressName         string
+		digUsingUDP         bool
 	}
 	state struct {
 		namespaceCreated bool
@@ -148,6 +149,11 @@ func (w *Workflow) WithGslb(path, host string) *Workflow {
 	return w
 }
 
+func (w *Workflow) WithDigUsingUDP(digUsingUDP bool) *Workflow {
+	w.settings.digUsingUDP = digUsingUDP
+	return w
+}
+
 func (w *Workflow) WithTestApp(uiMessage string) *Workflow {
 	w.state.testApp.isInstalled = true
 	w.state.testApp.name = "frontend-podinfo"
@@ -287,12 +293,12 @@ func (i *Instance) WaitForGSLB(instances ...*Instance) ([]string, error) {
 			expectedIPs = append(expectedIPs, ip...)
 		}
 	}
-	return waitForLocalGSLBNew(i.w.t, i.w.state.gslb.host, i.w.state.gslb.port, expectedIPs)
+	return waitForLocalGSLBNew(i.w.t, i.w.state.gslb.host, i.w.state.gslb.port, expectedIPs, i.w.settings.digUsingUDP)
 }
 
 // WaitForExpected waits until GSLB dig doesnt return list of expected IP's
 func (i *Instance) WaitForExpected(expectedIPs []string) (err error) {
-	_, err = waitForLocalGSLBNew(i.w.t, i.w.state.gslb.host, i.w.state.gslb.port, expectedIPs)
+	_, err = waitForLocalGSLBNew(i.w.t, i.w.state.gslb.host, i.w.state.gslb.port, expectedIPs, i.w.settings.digUsingUDP)
 	if err != nil {
 		fmt.Println(i.GetStatus(fmt.Sprintf("expected IPs: %s", expectedIPs)).String())
 	}
@@ -339,15 +345,15 @@ func (i *Instance) String() (out string) {
 
 // Dig  returns a list of IP addresses from CoreDNS that belong to the instance
 func (i *Instance) Dig() []string {
-	dig, err := dns.Dig("localhost:"+strconv.Itoa(i.w.state.gslb.port), i.w.state.gslb.host)
+	dig, err := dns.Dig("localhost:"+strconv.Itoa(i.w.state.gslb.port), i.w.state.gslb.host, i.w.settings.digUsingUDP)
 	require.NoError(i.w.t, err)
 	return dig
 }
 
 // GetLocalTargets returns instance local targets
 func (i *Instance) GetLocalTargets() []string {
 	dnsName := fmt.Sprintf("localtargets-%s", i.w.state.gslb.host)
-	dig, err := dns.Dig("localhost:"+strconv.Itoa(i.w.state.gslb.port), dnsName)
+	dig, err := dns.Dig("localhost:"+strconv.Itoa(i.w.state.gslb.port), dnsName, i.w.settings.digUsingUDP)
 	require.NoError(i.w.t, err)
 	return dig
 }
@@ -432,12 +438,12 @@ func (s *InstanceStatus) String() string {
 	return gopkgstr.ToString(s)
 }
 
-func waitForLocalGSLBNew(t *testing.T, host string, port int, expectedResult []string) (output []string, err error) {
+func waitForLocalGSLBNew(t *testing.T, host string, port int, expectedResult []string, isUdp bool) (output []string, err error) {
 	return DoWithRetryWaitingForValueE(
 		t,
 		"Wait for failover to happen and coredns to pickup new values...",
 		100,
 		time.Second*1,
-		func() ([]string, error) { return dns.Dig("localhost:"+strconv.Itoa(port), host) },
+		func() ([]string, error) { return dns.Dig("localhost:"+strconv.Itoa(port), host, isUdp) },
 		expectedResult)
 }

terratest/utils/test_settings.go

@@ -33,4 +33,5 @@ type TestSettings struct {
 	Cluster3        string
 	PodinfoImage    string
 	ClustersNumber  int
+	DigUsingUDP     bool
 }

terratest/utils/utils.go

@@ -220,13 +220,17 @@ func AssertGslbDeleted(t *testing.T, options *k8s.KubectlOptions, gslbName strin
 	assert.Equal(t, deletionExpected, deletionActual)
 }
 
-func WaitForLocalGSLB(t *testing.T, dnsServer string, dnsPort int, host string, expectedResult []string) (output []string, err error) {
+func WaitForLocalGSLB(t *testing.T, dnsServer string, dnsPort int, settings TestSettings, host string, expectedResult []string) (output []string, err error) {
+	var additionalArgs []string
+	if !settings.DigUsingUDP {
+		additionalArgs = append(additionalArgs, "+tcp")
+	}
 	return DoWithRetryWaitingForValueE(
 		t,
 		"Wait for failover to happen and coredns to pickup new values...",
 		300,
 		time.Second*1,
-		func() ([]string, error) { return Dig(t, dnsServer, dnsPort, host) },
+		func() ([]string, error) { return Dig(t, dnsServer, dnsPort, host+settings.DNSZone, additionalArgs...) },
 		expectedResult)
 }