Prepare development cluster for OpenTelemetry (#1685)

This change implements an operator-based telemetry ingestion stack for
development purposes. It's a somewhat large stack to deploy, so we keep
it out of CI and exclusively add it to the `deploy/dev` setup.

For convenience we also adjust the nativelink K8s deployment to set the
`OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_COMPRESSION`
environment variables as per https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/.
These are currently a no-op and will be the only required change for
existing deployments once we land the the corresponding Rust changes.

Author: aaronmondal

deploy/dev/kustomization.yaml

@@ -3,6 +3,7 @@ kind: Kustomization
 
 resources:
 - ../../kubernetes/resources/flux
+- ../../kubernetes/resources/flux-dev
 - ../../kubernetes/resources/nativelink-core
 - ../../kubernetes/resources/gateway-routes
 - ../../kubernetes/workers/resources/worker-init

kubernetes/components/worker/worker.yaml

@@ -30,6 +30,10 @@ spec:
               value: info
             - name: NATIVELINK_ENDPOINT
               value: nativelink
+            - name: OTEL_EXPORTER_OTLP_ENDPOINT
+              value: http://otel-collector-collector.default.svc:4317
+            - name: OTEL_EXPORTER_OTLP_COMPRESSION
+              value: zstd
           volumeMounts:
             - name: worker-config
               mountPath: /worker.json5

kubernetes/nativelink/nativelink.yaml

@@ -20,6 +20,10 @@ spec:
           env:
             - name: RUST_LOG
               value: info
+            - name: OTEL_EXPORTER_OTLP_ENDPOINT
+              value: http://otel-collector-collector.default.svc:4317
+            - name: OTEL_EXPORTER_OTLP_COMPRESSION
+              value: zstd
           ports:
             - containerPort: 9090
             - containerPort: 50051

kubernetes/resources/cert-manager/cert-manager.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: cert-manager
+  namespace: flux-system
+spec:
+  interval: 24h
+  url: https://charts.jetstack.io
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: cert-manager
+  namespace: flux-system
+spec:
+  interval: 30m
+  targetNamespace: cert-manager
+  chart:
+    spec:
+      chart: cert-manager
+      version: "1.x"
+      sourceRef:
+        kind: HelmRepository
+        name: cert-manager
+      interval: 12h
+  values:
+    installCRDs: true

kubernetes/resources/cert-manager/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - cert-manager.yaml

kubernetes/resources/flux-dev/flux-kustomizations.yaml

@@ -0,0 +1,58 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cert-manager
+  namespace: flux-system
+spec:
+  interval: 2m
+  path: "./kubernetes/resources/cert-manager"
+  prune: true
+  force: true
+  retryInterval: 20s
+  # Don't set targetNamespace for this one as the cert-manager kustomization
+  # needs to create a cert-manager namespace. The flux resources will live in
+  # the flux-system namespace.
+  wait: true
+  sourceRef:
+    kind: GitRepository
+    name: nativelink-infra
+    namespace: flux-system
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: olm
+  namespace: flux-system
+spec:
+  interval: 24h
+  path: "./kubernetes/vendor/olm"
+  prune: true
+  force: true
+  retryInterval: 20s
+  # Don't set targetNamespace for this one as OLM mangages this itself.
+  wait: true
+  sourceRef:
+    kind: GitRepository
+    name: nativelink-infra
+    namespace: flux-system
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: telemetry-operators
+  namespace: flux-system
+spec:
+  interval: 24h
+  path: "./kubernetes/vendor/telemetry"
+  prune: true
+  force: true
+  retryInterval: 20s
+  targetNamespace: operators
+  wait: true
+  sourceRef:
+    kind: GitRepository
+    name: nativelink-infra
+    namespace: flux-system
+  dependsOn:
+    - name: olm

kubernetes/resources/flux-dev/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - flux-kustomizations.yaml

kubernetes/vendor/olm/crds.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- crds.yaml
+- olm.yaml