diff --git a/controllers/dnsupdate.go b/controllers/dnsupdate.go index 3fe70766b5..92fd7c4683 100644 --- a/controllers/dnsupdate.go +++ b/controllers/dnsupdate.go @@ -20,10 +20,10 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic import ( "fmt" - "sort" "strings" "github.com/k8gb-io/k8gb/controllers/depresolver" + "github.com/k8gb-io/k8gb/controllers/providers/assistant" k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,13 +31,6 @@ import ( externaldns "sigs.k8s.io/external-dns/endpoint" ) -func sortTargets(targets []string) []string { - sort.Slice(targets, func(i, j int) bool { - return targets[i] < targets[j] - }) - return targets -} - func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.DNSEndpoint, error) { var gslbHosts []*externaldns.Endpoint var ttl = externaldns.TTL(gslb.Spec.Strategy.DNSTtlSeconds) @@ -53,7 +46,7 @@ func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.D } for host, health := range serviceHealth { - var finalTargets []string + var finalTargets = assistant.NewTargets() if !strings.Contains(host, r.Config.EdgeDNSZone) { return nil, fmt.Errorf("ingress host %s does not match delegated zone %s", host, r.Config.EdgeDNSZone) @@ -63,7 +56,7 @@ func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.D isHealthy := health == k8gbv1beta1.Healthy if isHealthy { - finalTargets = append(finalTargets, localTargets...) + finalTargets.Append(r.Config.ClusterGeoTag, localTargets) localTargetsHost := fmt.Sprintf("localtargets-%s", host) dnsRecord := &externaldns.Endpoint{ DNSName: localTargetsHost, @@ -75,14 +68,13 @@ func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.D } // Check if host is alive on external Gslb - externalTargets := r.DNSProvider.GetExternalTargets(host).GetIPs() - - sortTargets(externalTargets) + externalTargets := r.DNSProvider.GetExternalTargets(host) + externalTargets.Sort() if len(externalTargets) > 0 { switch gslb.Spec.Strategy.Type { case depresolver.RoundRobinStrategy, depresolver.GeoStrategy: - finalTargets = append(finalTargets, externalTargets...) + finalTargets.AppendTargets(externalTargets) case depresolver.FailoverStrategy: // If cluster is Primary if isPrimary { @@ -93,7 +85,7 @@ func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.D log.Info(). Str("gslb", gslb.Name). Str("cluster", gslb.Spec.Strategy.PrimaryGeoTag). - Strs("targets", finalTargets). + Strs("targets", finalTargets.GetIPs()). Str("workload", k8gbv1beta1.Unhealthy.String()). Msg("Executing failover strategy for primary cluster") } @@ -105,7 +97,7 @@ func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.D log.Info(). Str("gslb", gslb.Name). Str("cluster", gslb.Spec.Strategy.PrimaryGeoTag). - Strs("targets", finalTargets). + Strs("targets", finalTargets.GetIPs()). Str("workload", k8gbv1beta1.Healthy.String()). Msg("Executing failover strategy for secondary cluster") } @@ -116,10 +108,10 @@ func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.D Msg("No external targets have been found for host") } - r.updateRuntimeStatus(gslb, isPrimary, health, finalTargets) + r.updateRuntimeStatus(gslb, isPrimary, health, finalTargets.GetIPs()) log.Info(). Str("gslb", gslb.Name). - Strs("targets", finalTargets). + Strs("targets", finalTargets.GetIPs()). Msg("Final target list") if len(finalTargets) > 0 { @@ -127,11 +119,14 @@ func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.D DNSName: host, RecordTTL: ttl, RecordType: "A", - Targets: finalTargets, + Targets: finalTargets.GetIPs(), Labels: externaldns.Labels{ "strategy": gslb.Spec.Strategy.Type, }, } + for k, v := range r.getLabels(gslb, finalTargets) { + dnsRecord.Labels[k] = v + } gslbHosts = append(gslbHosts, dnsRecord) } } @@ -166,3 +161,19 @@ func (r *GslbReconciler) updateRuntimeStatus(gslb *k8gbv1beta1.Gslb, isPrimary b m.UpdateFailoverStatus(gslb, isPrimary, isHealthy, finalTargets) } } + +// getLabels map of where key identifies region and weight, value identifies IP. +func (r *GslbReconciler) getLabels(gslb *k8gbv1beta1.Gslb, targets assistant.Targets) (labels map[string]string) { + labels = make(map[string]string, 0) + for k, v := range gslb.Spec.Strategy.Weight { + t, found := targets[k] + if !found { + continue + } + for i, ip := range t.IPs { + l := fmt.Sprintf("weight-%s-%v-%v", k, i, v.Int()) + labels[l] = ip + } + } + return labels +} diff --git a/controllers/providers/assistant/gslb.go b/controllers/providers/assistant/gslb.go index 4e87bd8e67..6797098f38 100644 --- a/controllers/providers/assistant/gslb.go +++ b/controllers/providers/assistant/gslb.go @@ -297,8 +297,8 @@ func dnsQuery(host string, nameservers utils.DNSList) (*dns.Msg, error) { } func (r *Gslb) GetExternalTargets(host string, extClusterNsNames map[string]string) (targets Targets) { - targets = Targets{} - for _, cluster := range extClusterNsNames { + targets = NewTargets() + for tag, cluster := range extClusterNsNames { // Use edgeDNSServer for resolution of NS names and fallback to local nameservers log.Info(). Str("cluster", cluster). @@ -327,7 +327,7 @@ func (r *Gslb) GetExternalTargets(host string, extClusterNsNames map[string]stri } clusterTargets := getARecords(a) if len(clusterTargets) > 0 { - targets = append(targets, Target{cluster, clusterTargets}) + targets[tag] = &Target{clusterTargets} log.Info(). Strs("clusterTargets", clusterTargets). Str("cluster", cluster). diff --git a/controllers/providers/assistant/target.go b/controllers/providers/assistant/target.go index ca8d523ed2..ef7d7b6bdd 100644 --- a/controllers/providers/assistant/target.go +++ b/controllers/providers/assistant/target.go @@ -1,5 +1,7 @@ package assistant +import "sort" + /* Copyright 2022 The k8gb Contributors. @@ -19,17 +21,46 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic */ type Target struct { - Region string - IPs []string + IPs []string } -type Targets []Target +type Targets map[string]*Target + +func NewTargets() Targets { + return make(map[string]*Target, 0) +} -func (t Targets) GetIPs() (targets []string) { +func (t Targets) GetIPs() (ips []string) { // initializing targets to avoid possible nil reference errors (serialization etc.) - targets = []string{} + ips = []string{} + for _, v := range t { + ips = append(ips, v.IPs...) + } + return ips +} + +func (t Targets) Append(tag string, ips []string) { + if target, found := t[tag]; found { + target.IPs = append(target.IPs, ips...) + return + } + t[tag] = &Target{IPs: ips} +} + +func (t Targets) AppendTargets(targets Targets) { + for k, v := range targets { + t.Append(k, v.IPs) + } +} + +func (t Targets) Sort() { + sort := func(targets []string) []string { + sort.Slice(targets, func(i, j int) bool { + return targets[i] < targets[j] + }) + return targets + } for _, v := range t { - targets = append(targets, v.IPs...) + v.IPs = sort(v.IPs) } - return targets } diff --git a/controllers/weight_test.go b/controllers/weight_test.go new file mode 100644 index 0000000000..7af62d030a --- /dev/null +++ b/controllers/weight_test.go @@ -0,0 +1,195 @@ +package controllers + +import ( + "context" + "fmt" + "testing" + + "github.com/golang/mock/gomock" + k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" + "github.com/k8gb-io/k8gb/controllers/depresolver" + "github.com/k8gb-io/k8gb/controllers/providers/assistant" + "github.com/k8gb-io/k8gb/controllers/providers/dns" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/client" + externaldns "sigs.k8s.io/external-dns/endpoint" +) + +/* +Copyright 2022 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +func TestWeight(t *testing.T) { + // arrange + type wrr struct { + weight string + targets []string + } + var tests = []struct { + name string + data map[string]wrr + injectWeights bool + expectedLabels map[string]string + }{ + { + name: "eu35-us50-za15", + injectWeights: true, + data: map[string]wrr{ + "eu": {weight: "35%", targets: []string{"10.10.0.1", "10.10.0.2"}}, + "us": {weight: "50%", targets: []string{"10.0.0.1", "10.0.0.2"}}, + "za": {weight: "15%", targets: []string{"10.22.0.1", "10.22.0.2", "10.22.1.1"}}, + }, + expectedLabels: map[string]string{ + "strategy": depresolver.RoundRobinStrategy, + "weight-us-0-50": "10.0.0.1", + "weight-us-1-50": "10.0.0.2", + "weight-eu-0-35": "10.10.0.1", + "weight-eu-1-35": "10.10.0.2", + "weight-za-0-15": "10.22.0.1", + "weight-za-1-15": "10.22.0.2", + "weight-za-2-15": "10.22.1.1", + }, + }, + { + name: "eu100-us0-za0", + injectWeights: true, + data: map[string]wrr{ + "eu": {weight: "100%", targets: []string{"10.10.0.1", "10.10.0.2"}}, + "us": {weight: "0%", targets: []string{"10.0.0.1", "10.0.0.2"}}, + "za": {weight: "0%", targets: []string{"10.22.0.1", "10.22.0.2", "10.22.1.1"}}, + }, + expectedLabels: map[string]string{ + "strategy": depresolver.RoundRobinStrategy, + "weight-us-0-0": "10.0.0.1", + "weight-us-1-0": "10.0.0.2", + "weight-eu-0-100": "10.10.0.1", + "weight-eu-1-100": "10.10.0.2", + "weight-za-0-0": "10.22.0.1", + "weight-za-1-0": "10.22.0.2", + "weight-za-2-0": "10.22.1.1", + }, + }, + { + name: "weights-without-external-targets", + injectWeights: true, + data: map[string]wrr{ + "eu": {weight: "25%", targets: []string{}}, + "us": {weight: "75%", targets: []string{}}, + "za": {weight: "0%", targets: []string{}}, + }, + expectedLabels: map[string]string{ + "strategy": depresolver.RoundRobinStrategy, + }, + }, + { + name: "weights-without-external-targets", + injectWeights: true, + data: map[string]wrr{ + "eu": {weight: "25%", targets: []string{}}, + "us": {weight: "75%", targets: []string{}}, + "za": {weight: "0%", targets: []string{}}, + }, + expectedLabels: map[string]string{ + "strategy": depresolver.RoundRobinStrategy, + }, + }, + { + name: "no weights without external targets", + injectWeights: false, + data: map[string]wrr{}, + expectedLabels: map[string]string{ + "strategy": depresolver.RoundRobinStrategy, + }, + }, + { + name: "no weights with external targets", + injectWeights: false, + data: map[string]wrr{ + "eu": {weight: "100%", targets: []string{"10.10.0.1", "10.10.0.2"}}, + "us": {weight: "0%", targets: []string{"10.0.0.1", "10.0.0.2"}}, + "za": {weight: "0%", targets: []string{"10.22.0.1", "10.22.0.2", "10.22.1.1"}}, + }, + expectedLabels: map[string]string{ + "strategy": depresolver.RoundRobinStrategy, + }, + }, + { + name: "empty weights", + injectWeights: true, + data: map[string]wrr{}, + expectedLabels: map[string]string{ + "strategy": depresolver.RoundRobinStrategy, + }, + }, + } + + // act + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + injectWeight := func(ctx context.Context, gslb *k8gbv1beta1.Gslb, client client.Client) error { + if !test.injectWeights { + return nil + } + gslb.Spec.Strategy.Weight = make(map[string]k8gbv1beta1.Percentage, 0) + for k, w := range test.data { + gslb.Spec.Strategy.Weight[k] = k8gbv1beta1.Percentage(w.weight) + } + return nil + } + + assertAnnotation := func(gslb *k8gbv1beta1.Gslb, ep *externaldns.DNSEndpoint) error { + require.NotNil(t, ep) + require.NotNil(t, gslb) + // annotation is equal to tested value + for _, e := range ep.Spec.Endpoints { + for k, v := range e.Labels { + assert.Equal(t, test.expectedLabels[k], v) + } + assert.Equal(t, len(test.expectedLabels), len(e.Labels)) + } + return nil + } + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + // settings := provideSettings(t, predefinedConfig) + m := dns.NewMockProvider(ctrl) + r := depresolver.NewMockResolver(ctrl) + m.EXPECT().GslbIngressExposedIPs(gomock.Any()).Return([]string{}, nil).Times(1) + m.EXPECT().SaveDNSEndpoint(gomock.Any(), gomock.Any()).Do(assertAnnotation).Return(fmt.Errorf("save DNS error")).Times(1) + m.EXPECT().CreateZoneDelegationForExternalDNS(gomock.Any()).Return(nil).AnyTimes() + r.EXPECT().ResolveGslbSpec(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(injectWeight).AnyTimes() + + ts := assistant.Targets{} + for k, w := range test.data { + ts[k] = &assistant.Target{IPs: w.targets} + } + m.EXPECT().GetExternalTargets("roundrobin.cloud.example.com").Return(ts).Times(1) + m.EXPECT().GetExternalTargets("notfound.cloud.example.com").Return(assistant.Targets{}).Times(1) + m.EXPECT().GetExternalTargets("unhealthy.cloud.example.com").Return(assistant.Targets{}).Times(1) + + settings := provideSettings(t, predefinedConfig) + settings.reconciler.DNSProvider = m + settings.reconciler.DepResolver = r + + // act, assert + _, _ = settings.reconciler.Reconcile(context.TODO(), settings.request) + }) + } +}