OpenTelemetry for k8gb (#934)

* OpenTelemetry for k8gb - by default, it is noop - iff enabled (`tracing.enabled`): * helm chart deploys otel collector as a sidecar container on k8gb pod * iff enabled (`tracing.deployJaeger`), it deploys also Jaeger and configure the otel collector to feed it with traces * if enabled, collect and send following traces: Reconcile, gslbDNSEndpoint, SaveDNSEndpoint, CreateZoneDelegationForExternalDNS, finalize - docs * Add missing license * otel: using the otlp protocol instead of Jaeger's proprietary one + updating the versions Signed-off-by: Jirka Kremser <[email protected]>
k8gb-io · Aug 12, 2022 · 5db15df · 5db15df
1 parent 8820bba
commit 5db15df
Show file tree

Hide file tree

Showing 25 changed files with 553 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -108,6 +108,7 @@ Internal k8gb architecture and its components are described [here](/docs/compone
 * [Local playground for testing and development](/docs/local.md)
 * [Local playground with Kuar web app](/docs/local-kuar.md)
 * [Metrics](/docs/metrics.md)
+* [Traces](/docs/traces.md)
 * [Ingress annotations](/docs/ingress_annotations.md)
 * [Integration with Admiralty](/docs/admiralty.md)
 * [Integration with Liqo](/docs/liqo.md)

diff --git a/chart/k8gb/templates/NOTES.txt b/chart/k8gb/templates/NOTES.txt
@@ -6,6 +6,16 @@ done
   |_|\_\___/ \__, |_.__/  & all dependencies are installed
              |___/
 
+{{- if and .Values.tracing.enabled .Values.tracing.deployJaeger }}
+
+
+To check the OpenTelemetry (tracing) data:
+------------------------------------------
+kubectl port-forward svc/jaeger-collector -n k8gb 16686
+open http://localhost:16686
+
+{{- end }}
+
 1. Check if your DNS Zone is served by K8GB CoreDNS
   $ kubectl -n {{ .Release.Namespace }} run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools --command -- /usr/bin/dig @{{ .Release.Name }}-coredns SOA . +short
 

diff --git a/chart/k8gb/templates/infoblox-cm.yaml b/chart/k8gb/templates/infoblox-cm.yaml
@@ -9,4 +9,6 @@ data:
 kind: ConfigMap
 metadata:
   name: infoblox
+  labels:
+{{ include "chart.labels" . | indent 4  }}
 {{- end }}
diff --git a/chart/k8gb/templates/operator.yaml b/chart/k8gb/templates/operator.yaml
@@ -14,6 +14,8 @@ spec:
     metadata:
       labels:
         name: k8gb
+      annotations:
+        kubectl.kubernetes.io/default-container: k8gb
     spec:
       serviceAccountName: k8gb
       containers:
@@ -103,6 +105,18 @@ spec:
             - name: COREDNS_EXPOSED
               value: "true"
             {{- end }}
+            {{- if .Values.tracing.enabled }}
+              {{- with .Values.tracing }}
+            - name: TRACING_ENABLED
+              value: {{ .enabled | quote }}
+            - name: OTEL_EXPORTER_OTLP_ENDPOINT
+              value: {{ .endpoint | quote }}
+                {{- with .samplingRatio }}
+            - name: TRACING_SAMPLING_RATIO
+              value: {{ . | quote }}
+                {{- end }}
+              {{- end }}
+            {{- end }}
             - name: LOG_FORMAT
               value: {{ quote .Values.k8gb.log.format }}
             - name: LOG_LEVEL
@@ -113,3 +127,20 @@ spec:
               value: {{ quote .Values.k8gb.splitBrainCheck }}
             - name: METRICS_ADDRESS
               value: {{ .Values.k8gb.metricsAddress }}
+      {{- if .Values.tracing.enabled }}
+        - image: {{ .Values.tracing.sidecarImage.repository }}:{{ .Values.tracing.sidecarImage.tag }}
+          name: otel-collector
+          imagePullPolicy: {{ .Values.tracing.sidecarImage.pullPolicy }}
+          args:
+          - --config=/conf/agent.yaml
+          volumeMounts:
+          - mountPath: /conf
+            name: agent-config
+      volumes:
+      - configMap:
+          items:
+          - key: agent.yaml
+            path: agent.yaml
+          name: agent-config
+        name: agent-config
+      {{- end }}
diff --git a/chart/k8gb/templates/otel/jaeger-deployment.yaml b/chart/k8gb/templates/otel/jaeger-deployment.yaml
@@ -0,0 +1,39 @@
+{{- if .Values.tracing.enabled }}
+{{- if .Values.tracing.deployJaeger }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: jaeger
+  labels:
+{{ include "chart.labels" . | indent 4  }}
+spec:
+  selector:
+    matchLabels:
+      id: jaeger
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        id: jaeger
+    spec:
+      containers:
+        - name: jaeger
+          image: {{ .Values.tracing.jaegerImage.repository }}:{{ .Values.tracing.jaegerImage.tag }}
+          imagePullPolicy: {{ .Values.tracing.jaegerImage.pullPolicy }}
+          readinessProbe:
+            httpGet:
+              path: "/"
+              port: 14269
+            initialDelaySeconds: 5
+          env:
+            - name: COLLECTOR_OTLP_ENABLED
+              value: "true"
+          resources:
+            limits:
+              cpu: 500m
+              memory: 128Mi
+            requests:
+              cpu: 10m
+              memory: 64Mi
+{{- end }}
+{{- end }}
diff --git a/chart/k8gb/templates/otel/jaeger-svc.yaml b/chart/k8gb/templates/otel/jaeger-svc.yaml
@@ -0,0 +1,27 @@
+{{- if .Values.tracing.enabled }}
+{{- if .Values.tracing.deployJaeger }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: jaeger-collector
+  labels:
+{{ include "chart.labels" . | indent 4  }}
+spec:
+  ports:
+    - name: dashboard
+      port: 16686
+      protocol: TCP
+      targetPort: 16686
+    - name: grpc-otlp-collector
+      port: 4317
+      protocol: TCP
+      targetPort: 4317
+    - name: http-otlp-collector
+      port: 4318
+      protocol: TCP
+      targetPort: 4318
+  selector:
+    id: jaeger
+  type: ClusterIP
+{{- end }}
+{{- end }}
diff --git a/chart/k8gb/templates/otel/otel-config.yaml b/chart/k8gb/templates/otel/otel-config.yaml
@@ -0,0 +1,36 @@
+{{- if .Values.tracing.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: agent-config
+  namespace: k8gb
+  labels:
+{{ include "chart.labels" . | indent 4  }}
+data:
+{{- if or .Values.tracing.deployJaeger (not .Values.tracing.otelConfig ) }}
+  agent.yaml: |  
+    receivers:
+      otlp:
+        protocols:
+          http:
+          grpc:
+    processors:
+    exporters:
+      otlp:
+        endpoint: jaeger-collector.k8gb:4317
+        tls:
+          insecure: true
+        retry_on_failure:
+          enabled: true
+      logging:
+    service:
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: []
+          exporters: [otlp]
+{{- else -}}
+  agent.yaml: |  
+{{ toYaml .Values.tracing.otelConfig | indent 4  }}
+{{- end }}
+{{- end }}
diff --git a/chart/k8gb/templates/otel/otel-svc.yaml b/chart/k8gb/templates/otel/otel-svc.yaml
@@ -0,0 +1,21 @@
+{{- if .Values.tracing.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: otel-collector
+  labels:
+{{ include "chart.labels" . | indent 4  }}
+spec:
+  ports:
+    - name: http-otlp
+      port: 4318
+      protocol: TCP
+      targetPort: 4318
+    - name: grpc-otlp
+      port: 4317
+      protocol: TCP
+      targetPort: 4317
+  selector:
+    name: k8gb
+  type: ClusterIP
+{{- end }}
diff --git a/chart/k8gb/templates/role.yaml b/chart/k8gb/templates/role.yaml
@@ -4,6 +4,8 @@ kind: ClusterRole
 metadata:
   creationTimestamp: null
   name: k8gb
+  labels:
+{{ include "chart.labels" . | indent 4  }}
 rules:
 - apiGroups:
   - ""

diff --git a/chart/k8gb/templates/role_binding.yaml b/chart/k8gb/templates/role_binding.yaml
@@ -3,6 +3,8 @@ kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: k8gb
+  labels:
+{{ include "chart.labels" . | indent 4  }}
 subjects:
 - kind: ServiceAccount
   name: k8gb

diff --git a/chart/k8gb/templates/service_account.yaml b/chart/k8gb/templates/service_account.yaml
@@ -4,5 +4,7 @@ kind: ServiceAccount
 metadata:
   name: k8gb
   namespace: {{ .Release.Namespace }}
+  labels:
+{{ include "chart.labels" . | indent 4  }}
 imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | nindent 2 }}
 {{- end }}
diff --git a/chart/k8gb/values.schema.json b/chart/k8gb/values.schema.json
@@ -32,6 +32,9 @@
                 },
                 "rfc2136": {
                     "$ref": "#/definitions/Rfc2136"
+                },
+                "tracing": {
+                    "$ref": "#/definitions/Tracing"
                 }
             },
             "required": []
@@ -155,6 +158,27 @@
             "required": [],
             "title": "Global"
         },
+        "Image": {
+            "type": "object",
+            "required": [],
+            "additionalProperties": false,
+            "properties": {
+                "repository": {
+                    "type": "string",
+                    "minLength": 1
+                },
+                "pullPolicy": {
+                    "enum": [
+                        "Always",
+                        "IfNotPresent",
+                        "Never"
+                    ]
+                },
+                "tag": {
+                    "type": "string"
+                }
+            }
+        },
         "Infoblox": {
             "type": "object",
             "additionalProperties": false,
@@ -389,6 +413,43 @@
                 "irsaRole"
             ],
             "title": "Route53"
+        },
+        "Tracing": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "enabled": {
+                    "type": "boolean"
+                },
+                "deployJaeger": {
+                    "type": "boolean"
+                },
+                "endpoint": {
+                    "type": "string",
+                    "pattern": "^.{2,256}:\\d{2,5}$"
+                },
+                "samplingRatio": {
+                    "type": [
+                        "string",
+                        "null"
+                    ],
+                    "pattern": "^(0(\\.\\d{1,3})?|1(\\.0)?)$"
+                },
+                "otelConfig": {
+                    "type": ["object", "null"],
+                    "additionalProperties": true
+                },
+                "sidecarImage": {
+                    "$ref": "#/definitions/Image"
+                },
+                "jaegerImage": {
+                    "$ref": "#/definitions/Image"
+                }
+            },
+            "required": [
+                "enabled"
+            ],
+            "title": "Tracing"
         }
     }
 }
diff --git a/chart/k8gb/values.yaml b/chart/k8gb/values.yaml
@@ -115,3 +115,35 @@ rfc2136:
     - port: 1053
     - tsig-secret-alg: hmac-sha256
     - tsig-keyname: externaldns-key
+
+tracing:
+  # -- if the application should be sending the traces to OTLP collector (env var `TRACING_ENABLED`)
+  enabled: false
+
+  # -- should the Jaeger be deployed together with the k8gb operator? In case of using another OpenTracing solution,
+  # make sure that configmap for OTEL agent has the correct exporters set up (`tracing.otelConfig`).
+  deployJaeger: false
+
+  # -- `host:port` where the spans from the applications (traces) should be sent, sets the `OTEL_EXPORTER_OTLP_ENDPOINT` env var
+  # This is not the final destination where all the traces are going. Otel collector has its configuration in the associated configmap (`tracing.otelConfig`).
+  endpoint: localhost:4318
+
+  # -- float representing the ratio of how often the span should be kept/dropped (env var `TRACING_SAMPLING_RATIO`)
+  # if not specified, the AlwaysSample will be used which is the same as 1.0. `0.1` would mean that 10% of samples will be kept
+  samplingRatio: null
+
+  # -- configuration for OTEL collector, this will be represented as configmap called `agent-config`
+  otelConfig: null
+
+  sidecarImage:
+    # -- OpenTelemetry collector into which the k8gb operator sends the spans. It can be further configured to send its data
+    # to somewhere else using exporters (Jaeger for instance)
+    repository: otel/opentelemetry-collector
+    tag: 0.57.2
+    pullPolicy: Always
+
+  jaegerImage:
+    # -- if `tracing.deployJaeger==true` this image will be used in the deployment for Jaeger
+    repository: jaegertracing/all-in-one
+    tag: 1.37.0
+    pullPolicy: Always
diff --git a/controllers/depresolver/depresolver.go b/controllers/depresolver/depresolver.go
@@ -150,6 +150,13 @@ type Config struct {
 	extDNSEnabled bool `env:"EXTDNS_ENABLED, default=false"`
 	// SplitBrainCheck flag decides whether split brain TXT records will be stored in edge DNS
 	SplitBrainCheck bool `env:"SPLIT_BRAIN_CHECK, default=false"`
+	// TracingEnabled flag decides whether to use a real otlp tracer or a noop one
+	TracingEnabled bool `env:"TRACING_ENABLED, default=false"`
+	// TracingSamplingRatio how many traces should be kept and sent (1.0 - all, 0.0 - none)
+	TracingSamplingRatio float64 `env:"TRACING_SAMPLING_RATIO, default=1.0"`
+	// OtelExporterOtlpEndpoint where the traces should be sent to (in case of otel collector deployed on the same pod as sidecar -> localhost:4318)
+	// otel collector itself can be configured via a configmap to send it somewhere else
+	OtelExporterOtlpEndpoint string `env:"OTEL_EXPORTER_OTLP_ENDPOINT, default=localhost:4318"`
 }
 
 // DependencyResolver resolves configuration for GSLB

diff --git a/controllers/depresolver/depresolver_config.go b/controllers/depresolver/depresolver_config.go
@@ -53,6 +53,9 @@ const (
 	LogFormatKey                   = "LOG_FORMAT"
 	LogNoColorKey                  = "NO_COLOR"
 	SplitBrainCheckKey             = "SPLIT_BRAIN_CHECK"
+	TracingEnabled                 = "TRACING_ENABLED"
+	OtelExporterOtlpEndpoint       = "OTEL_EXPORTER_OTLP_ENDPOINT"
+	TracingSamplingRatio           = "TRACING_SAMPLING_RATIO"
 	MetricsAddressKey              = "METRICS_ADDRESS"
 )
 

diff --git a/controllers/depresolver/depresolver_test.go b/controllers/depresolver/depresolver_test.go
@@ -1522,7 +1522,8 @@ func cleanup() {
 	for _, s := range []string{ReconcileRequeueSecondsKey, ClusterGeoTagKey, ExtClustersGeoTagsKey, EdgeDNSZoneKey, DNSZoneKey, EdgeDNSServersKey,
 		ExtDNSEnabledKey, InfobloxGridHostKey, InfobloxVersionKey, InfobloxPortKey, InfobloxUsernameKey,
 		InfobloxPasswordKey, K8gbNamespaceKey, CoreDNSExposedKey, InfobloxHTTPRequestTimeoutKey,
-		InfobloxHTTPPoolConnectionsKey, LogLevelKey, LogFormatKey, LogNoColorKey, MetricsAddressKey, SplitBrainCheckKey} {
+		InfobloxHTTPPoolConnectionsKey, LogLevelKey, LogFormatKey, LogNoColorKey, MetricsAddressKey, SplitBrainCheckKey, TracingEnabled,
+		TracingSamplingRatio, OtelExporterOtlpEndpoint} {
 		if os.Unsetenv(s) != nil {
 			panic(fmt.Errorf("cleanup %s", s))
 		}
@@ -1553,6 +1554,9 @@ func configureEnvVar(config Config) {
 	_ = os.Setenv(LogNoColorKey, strconv.FormatBool(config.Log.NoColor))
 	_ = os.Setenv(MetricsAddressKey, config.MetricsAddress)
 	_ = os.Setenv(SplitBrainCheckKey, strconv.FormatBool(config.SplitBrainCheck))
+	_ = os.Setenv(TracingEnabled, strconv.FormatBool(config.TracingEnabled))
+	_ = os.Setenv(TracingSamplingRatio, strconv.FormatFloat(config.TracingSamplingRatio, 'f', 2, 64))
+	_ = os.Setenv(OtelExporterOtlpEndpoint, config.OtelExporterOtlpEndpoint)
 }
 
 func getTestContext(testData string) (client.Client, *k8gbv1beta1.Gslb) {