Skip to content

Commit eadd4a5

Browse files
NickLarsenNZNickLarsenNZ
authored
feat(run-integration-tests): Install OpenTelemetry Operator and Collectors for test observability (#34)
* feat(run-integration-test): Add OpenTelemetry kustomize bases * feat(run-integration-test): Add OpenTelemetry kustomize overlay for replicated clusters * feat(run-integration-test): Add steps to apply kustomize configs to cluster * feat(run-integration-test): Add attributes from env to collectors * temp(run-integration-test): Sleep instead of running tests * temp(run-integration-test): Run the action * Revert "temp(run-integration-test): Sleep instead of running tests" This reverts commit 7d4db60. * Revert "temp(run-integration-test): Run the action" This reverts commit 585d0ea.
1 parent 2d703e5 commit eadd4a5

23 files changed

+685
-1
lines changed

‎run-integration-test/action.yml‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ runs:
3636
INTERU_VERSION: ${{ inputs.interu-version }}
3737
TEST_PROFILE: ${{ inputs.test-profile }}
3838
GITHUB_DEBUG: ${{ runner.debug }}
39+
GITHUB_REPOSITORY: ${{ github.repository }}
40+
GITHUB_RUN_ID: ${{ github.run_id }}"
3941
shell: bash
4042
run: |
4143
set -euo pipefail
@@ -45,6 +47,9 @@ runs:
4547
curl -fsSL -o /tmp/interu "https://github.com/stackabletech/actions/releases/download/interu-$INTERU_VERSION/interu-x86_64-unknown-linux-gnu"
4648
sudo install -m 755 -t /usr/local/bin /tmp/interu
4749
50+
# Generate the cluster name
51+
echo "KUBERNETES_CLUSTER_NAME=integration-test-${GITHUB_REPOSITORY}-${GITHUB_RUN_ID}" | tee -a "$GITHUB_ENV"
52+
4853
# Run interu to expand parameters into GITHUB_ENV
4954
interu --instances "$GITHUB_ACTION_PATH/instances.yml" --check-test-definitions "$TEST_PROFILE" --output "$GITHUB_ENV"
5055
@@ -105,7 +110,7 @@ runs:
105110
with:
106111
# See: https://github.com/replicatedhq/replicated-actions/tree/main/create-cluster#inputs
107112
api-token: ${{ inputs.replicated-api-token }}
108-
cluster-name: integration-test-${{ github.repository }}-${{ github.run_id }}
113+
cluster-name: ${{ env.KUBERNETES_CLUSTER_NAME }}
109114
kubernetes-distribution: ${{ env.INTERU_KUBERNETES_DISTRIBUTION }}
110115
kubernetes-version: ${{ env.INTERU_KUBERNETES_VERSION }}
111116
ttl: ${{ env.INTERU_CLUSTER_TTL }}
@@ -140,6 +145,43 @@ runs:
140145
OPERATOR_NAME=$(echo "$REPOSITORY" | cut -d / -f 2 | sed 's/-operator//g')
141146
echo "OPERATOR_NAME=$OPERATOR_NAME" | tee -a "$GITHUB_ENV"
142147
148+
- name: Install OpenTelemetry Operator
149+
shell: bash
150+
run: |
151+
set -euo pipefail
152+
153+
echo "::group::kubectl apply"
154+
kubectl kustomize --enable-helm "${GITHUB_ACTION_PATH}/kustomize/bases/opentelemetry-operator" | kubectl apply -f -
155+
kubectl -n opentelemetry-operator wait --for condition=Progressing deploy/opentelemetry-operator --timeout=300s
156+
kubectl -n opentelemetry-operator wait --for condition=Available deploy/opentelemetry-operator --timeout=300s
157+
kubectl -n opentelemetry-operator get pods
158+
echo "::endgroup::"
159+
160+
- name: Apply OpenTelemetry Collectors configurations
161+
shell: bash
162+
env:
163+
GITHUB_TRIGGERED_BY: ${{ github.triggering_actor }}
164+
run: |
165+
set -euo pipefail
166+
167+
echo "::group::get info"
168+
INTEGRATION_TEST_INFO="${GITHUB_ACTION_PATH}/kustomize/overlays/replicated/integration-test-info.env"
169+
echo "KUBERNETES_CLUSTER_NAME=${KUBERNETES_CLUSTER_NAME}" | tee -a "$INTEGRATION_TEST_INFO"
170+
echo "TRIGGERED_BY=${GITHUB_TRIGGERED_BY}" | tee -a "$INTEGRATION_TEST_INFO"
171+
echo "::endgroup::"
172+
173+
echo "::group::kubectl apply"
174+
kubectl apply -k "${GITHUB_ACTION_PATH}/kustomize/overlays/replicated"
175+
echo "Waiting a few seconds for the operator to create the deployment" && sleep 5
176+
kubectl -n opentelemetry-operator wait --for condition=Progressing deploy/replicated-kubernetes-events-collector --timeout=300s
177+
kubectl -n opentelemetry-operator wait --for condition=Available deploy/replicated-kubernetes-events-collector --timeout=300s
178+
kubectl -n opentelemetry-operator rollout status ds replicated-container-log-scrape-collector --timeout=300s
179+
echo "::endgroup::"
180+
echo "::group::kubectl get"
181+
kubectl -n opentelemetry-operator get opentelemetrycollectors
182+
kubectl -n opentelemetry-operator get pods
183+
echo "::endgroup::"
184+
143185
- name: Record Test Start Time
144186
id: start-time
145187
shell: bash

‎run-integration-test/kustomize/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
charts/

‎run-integration-test/kustomize/bases/opentelemetry-collectors/clusterrole-k8sattributes-processor.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
---
2+
apiVersion: rbac.authorization.k8s.io/v1
3+
kind: ClusterRole
4+
metadata:
5+
name: k8sattributes-processor
6+
rules:
7+
- apiGroups: [""]
8+
resources: ["pods", "namespaces", "nodes"]
9+
verbs: ["get", "watch", "list"]
10+
- apiGroups: ["apps"]
11+
resources: ["replicasets"]
12+
verbs: ["get", "list", "watch"]
13+
- apiGroups: ["extensions"]
14+
resources: ["replicasets"]
15+
verbs: ["get", "list", "watch"]

‎run-integration-test/kustomize/bases/opentelemetry-collectors/container-log-scrape/clusterrole-container-log-scrape.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
---
2+
apiVersion: rbac.authorization.k8s.io/v1
3+
kind: ClusterRole
4+
metadata:
5+
name: container-log-scrape
6+
labels:
7+
app: container-log-scrape
8+
rules:
9+
- apiGroups:
10+
- ""
11+
resources:
12+
- events
13+
- namespaces
14+
- namespaces/status
15+
- nodes
16+
- nodes/spec
17+
- pods
18+
- pods/status
19+
- replicationcontrollers
20+
- replicationcontrollers/status
21+
- resourcequotas
22+
- services
23+
verbs:
24+
- get
25+
- list
26+
- watch
27+
- apiGroups:
28+
- apps
29+
resources:
30+
- daemonsets
31+
- deployments
32+
- replicasets
33+
- statefulsets
34+
verbs:
35+
- get
36+
- list
37+
- watch
38+
- apiGroups:
39+
- extensions
40+
resources:
41+
- daemonsets
42+
- deployments
43+
- replicasets
44+
verbs:
45+
- get
46+
- list
47+
- watch
48+
- apiGroups:
49+
- batch
50+
resources:
51+
- jobs
52+
- cronjobs
53+
verbs:
54+
- get
55+
- list
56+
- watch
57+
- apiGroups:
58+
- autoscaling
59+
resources:
60+
- horizontalpodautoscalers
61+
verbs:
62+
- get
63+
- list
64+
- watch

‎run-integration-test/kustomize/bases/opentelemetry-collectors/container-log-scrape/clusterrolebinding-container-log-scrape.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
---
2+
apiVersion: rbac.authorization.k8s.io/v1
3+
kind: ClusterRoleBinding
4+
metadata:
5+
name: container-log-scrape
6+
labels:
7+
app: container-log-scrape
8+
roleRef:
9+
apiGroup: rbac.authorization.k8s.io
10+
kind: ClusterRole
11+
name: container-log-scrape
12+
subjects:
13+
- kind: ServiceAccount
14+
name: container-log-scrape
15+
namespace: opentelemetry-operator

‎run-integration-test/kustomize/bases/opentelemetry-collectors/container-log-scrape/clusterrolebinding-k8sattributes-processor.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
---
2+
apiVersion: rbac.authorization.k8s.io/v1
3+
kind: ClusterRoleBinding
4+
metadata:
5+
name: k8sattributes-processor-container-log-scrape
6+
roleRef:
7+
kind: ClusterRole
8+
name: k8sattributes-processor
9+
apiGroup: rbac.authorization.k8s.io
10+
subjects:
11+
- kind: ServiceAccount
12+
name: container-log-scrape
13+
namespace: opentelemetry-operator

‎run-integration-test/kustomize/bases/opentelemetry-collectors/container-log-scrape/kustomization.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
---
2+
# commonLabels:
3+
4+
resources:
5+
- serviceaccount.yml
6+
- clusterrole-container-log-scrape.yml
7+
- clusterrolebinding-container-log-scrape.yml
8+
- clusterrolebinding-k8sattributes-processor.yml
9+
- opentelemetrycollector.yml

‎run-integration-test/kustomize/bases/opentelemetry-collectors/container-log-scrape/opentelemetrycollector.yml‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
---
2+
# https://github.com/open-telemetry/opentelemetry-operator/blob/main/tests/e2e/daemonset-features/01-install.yaml
3+
apiVersion: opentelemetry.io/v1beta1
4+
kind: OpenTelemetryCollector
5+
metadata:
6+
name: container-log-scrape
7+
spec:
8+
mode: daemonset
9+
# hostNetwork: true
10+
serviceAccount: container-log-scrape
11+
env:
12+
# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/k8sattributesprocessor/README.md#as-an-agent
13+
- name: KUBE_NODE_NAME
14+
valueFrom:
15+
fieldRef:
16+
apiVersion: v1
17+
fieldPath: spec.nodeName
18+
envFrom: []
19+
volumeMounts:
20+
# Mount the volumes to the collector container
21+
- name: varlogpods
22+
mountPath: /var/log/pods
23+
readOnly: true
24+
- name: varlibdockercontainers
25+
mountPath: /var/lib/docker/containers
26+
readOnly: true
27+
volumes:
28+
# Typically the collector will want access to pod logs and container logs
29+
- name: varlogpods
30+
hostPath:
31+
path: /var/log/pods
32+
- name: varlibdockercontainers
33+
hostPath:
34+
path: /var/lib/docker/containers
35+
config:
36+
receivers:
37+
# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/filelogreceiver
38+
filelog/varlogpods:
39+
# A storage extension (eg: redis, or file) can be used for storing log offsets. Otherwise it is held in memory.
40+
# See: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/storage
41+
# storage: {}
42+
include:
43+
- /var/log/pods/*/*/*.log
44+
exclude:
45+
# Exclude logs from all containers named otc-container
46+
# This is to prevent log amplification
47+
- /var/log/pods/*/otc-container/*.log
48+
start_at: end
49+
include_file_path: true
50+
include_file_name: false
51+
# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/stanza/docs/operators/README.md#what-operators-are-available
52+
operators:
53+
# parse container logs
54+
# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/stanza/docs/operators/container.md
55+
- id: container-parser
56+
type: container
57+
format: containerd
58+
add_metadata_from_filepath: true
59+
- id: klog-parser
60+
type: regex_parser
61+
on_error: send_quiet
62+
# Eg: kindnet-cni log:
63+
# I1214 14:13:50.384530 1 main.go:295] Handling node with IPs: map[172.18.0.2:{}]
64+
# https://seankhliao.com/blog/12021-06-17-opentelemetry-collector-logs/
65+
regex: '^(?P<severity>[FEWIDT])(?P<timestamp_field>.{20})\s+(?P<threadid>\d+)\s(?P<file>\w+\.go):(?P<line>\d+)]\s+(?P<message>.*)$'
66+
timestamp:
67+
parse_from: body.timestamp_field
68+
layout: "%m%d %H:%M:%S.%f"
69+
severity:
70+
parse_from: body.severity
71+
mapping:
72+
fatal: F
73+
error: E
74+
warning: W
75+
info: I
76+
debug: D
77+
trace: T
78+
- id: rust-tracing-fmt-plain
79+
type: regex_parser
80+
on_error: send_quiet
81+
# 2024-12-14T13:55:52.550715642+00:00 INFO openobserve::service::alerts::scheduler: Pulled 0 jobs from scheduler
82+
regex: '^(?P<timestamp_field>.{35})\s+(?P<severity>\w+)\s+(?P<target>\w+):\s+(?P<message>.*)$'
83+
timestamp:
84+
parse_from: body.timestamp_field
85+
layout: "%Y-%m-%dT%H:%M:%S.%f%j"
86+
severity:
87+
parse_from: body.severity
88+
# yamllint disable rule:comments-indentation
89+
# mapping:
90+
# fatal: FATAL
91+
# error: ERROR
92+
# warning: WARN
93+
# info: INFO
94+
# debug: DEBUG
95+
# trace: TRACE
96+
# yamllint enable rule:comments-indentation
97+
98+
- type: key_value_parser
99+
on_error: send_quiet
100+
parse_from: body.message
101+
102+
processors:
103+
# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/k8sattributesprocessor
104+
k8sattributes:
105+
filter:
106+
# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/k8sattributesprocessor/README.md#as-an-agent
107+
node_from_env_var: KUBE_NODE_NAME # this should be same as the var set from the downward API further up
108+
# https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/transformprocessor
109+
# transform:
110+
# error_mode: ignore
111+
# log_statements:
112+
# # Currently the body from go app logs contains a timestamp, severity, then log messages.
113+
# # I want to see if we can hoist those up a level.
114+
# - context: log
115+
# statements:
116+
# - set(attributes["body"], body)
117+
resourcedetection/env:
118+
detectors: [env]
119+
timeout: 2s
120+
override: false
121+
# yamllint disable rule:comments-indentation
122+
# memory_limiter:
123+
# check_interval: 1s
124+
# limit_percentage: 75
125+
# spike_limit_percentage: 15
126+
# batch:
127+
# send_batch_size: 10000
128+
# timeout: 10s
129+
# yamllint enable rule:comments-indentation
130+
131+
exporters:
132+
# Configuration done in the overlay
133+
otlp/grpc:
134+
headers: {}
135+
136+
service:
137+
pipelines:
138+
logs:
139+
receivers: [filelog/varlogpods]
140+
# processors: [memory_limiter, batch]
141+
processors: [k8sattributes, resourcedetection/env]
142+
exporters: [otlp/grpc]

‎run-integration-test/kustomize/bases/opentelemetry-collectors/container-log-scrape/serviceaccount.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
apiVersion: v1
3+
kind: ServiceAccount
4+
metadata:
5+
labels:
6+
app: container-log-scrape
7+
name: container-log-scrape

‎run-integration-test/kustomize/bases/opentelemetry-collectors/crds.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
---
2+
nameReference:
3+
- kind: ServiceAccount
4+
fieldSpecs:
5+
- kind: OpenTelemetryCollector
6+
path: spec/serviceAccount
7+
- kind: ClusterRoleBinding
8+
path: subjects/name
9+
- kind: Secret
10+
fieldSpecs:
11+
- kind: OpenTelemetryCollector
12+
path: spec/env/valueFrom/secretKeyRef/name
13+
- kind: ConfigMap
14+
fieldSpecs:
15+
- kind: OpenTelemetryCollector
16+
path: spec/envFrom/configMapRef/name
17+
# - kind: ClusterRole
18+
# fieldSpecs:
19+
# - kind: ClusterRoleBinding
20+
# path: roleRef/name

0 commit comments

Comments
 (0)