Skip to content

Commit 2161d81

Browse files
teeohhemteeohhem
authored
chore: Add otel tests to smoke test (#107)
Fixes: HDX-1869
1 parent 3a36cf3 commit 2161d81

File tree

2 files changed

+150
-93
lines changed

2 files changed

+150
-93
lines changed

‎.github/workflows/chart-test.yml‎

Lines changed: 47 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ jobs:
7474
apiKey: "test-api-key-for-ci"
7575
frontendUrl: "http://localhost:3000"
7676
replicas: 1
77+
service:
78+
type: NodePort
79+
nodePort: 30000
7780
7881
clickhouse:
7982
persistence:
@@ -86,42 +89,70 @@ jobs:
8689
enabled: true
8790
dataSize: 2Gi
8891
89-
# Enable NodePort services for testing
90-
hyperdx:
91-
service:
92-
type: NodePort
93-
nodePort: 30000
94-
9592
otel:
96-
service:
97-
type: NodePort
98-
nodePort: 30001
93+
resources:
94+
requests:
95+
memory: "128Mi"
96+
cpu: "100m"
97+
limits:
98+
memory: "256Mi"
99+
cpu: "200m"
99100
EOF
100101
101102
# Install the chart
102-
helm install hyperdx-test ./charts/hdx-oss-v2 -f test-values.yaml --timeout=2m
103+
helm install hyperdx-test ./charts/hdx-oss-v2 -f test-values.yaml --timeout=5m
103104
104105
# Give services time to initialize after pods are running
105106
echo "Waiting for services to initialize..."
106107
sleep 20
107108
109+
- name: Bootstrap team in MongoDB
110+
run: |
111+
# Wait for MongoDB to be ready
112+
kubectl wait --for=condition=Ready pods -l app=mongodb --timeout=300s
113+
114+
echo "Creating test team in MongoDB..."
115+
kubectl exec -n default deployment/hyperdx-test-hdx-oss-v2-mongodb -- mongosh hyperdx --eval "
116+
db.teams.insertOne({
117+
name: 'CI Test Team',
118+
apiKey: 'test-api-key-for-ci',
119+
collectorAuthenticationEnforced: false,
120+
createdAt: new Date(),
121+
updatedAt: new Date()
122+
})
123+
"
124+
125+
echo "Verifying team creation..."
126+
kubectl exec -n default deployment/hyperdx-test-hdx-oss-v2-mongodb -- mongosh hyperdx --eval "
127+
const team = db.teams.findOne({ apiKey: 'test-api-key-for-ci' });
128+
if (team) {
129+
print('Team created successfully:', team.name);
130+
} else {
131+
print('Team creation failed');
132+
exit(1);
133+
}
134+
"
135+
136+
echo "Waiting for OpAMP server to reconfigure collectors..."
137+
sleep 30
138+
108139
- name: Verify deployment
109140
run: |
110-
# Wait for all pods to be ready
111-
kubectl wait --for=condition=Ready pods --all --timeout=300s
141+
echo "Initial pod status:"
142+
kubectl get pods -o wide
143+
144+
echo "Waiting for all pods to be ready..."
145+
kubectl wait --for=condition=Ready pods --all --timeout=600s
112146
113-
# Check pod status
147+
echo "Final pod status:"
114148
kubectl get pods -o wide
115149
116-
# Check services
117150
kubectl get services
118151
119152
- name: Run comprehensive smoke tests
120153
run: |
121-
# Make smoke test script executable
122154
chmod +x ./scripts/smoke-test.sh
123155
124-
# Run the smoke test with CI-specific environment
125156
RELEASE_NAME=hyperdx-test NAMESPACE=default ./scripts/smoke-test.sh
126157
127158
- name: Collect logs on failure

‎scripts/smoke-test.sh‎

Lines changed: 103 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -70,71 +70,94 @@ check_endpoint "http://localhost:3000" "200" "UI"
7070
kill $pf_pid 2>/dev/null || true
7171
sleep 2
7272

73-
# # Skip OTEL collector metrics test (port 8888 not exposed by HyperDX collector)
74-
# echo "Skipping OTEL collector metrics test (not exposed on port 8888)"
75-
76-
# # Test data ingestion
77-
# echo "Testing data ingestion..."
78-
# kubectl port-forward service/$RELEASE_NAME-hdx-oss-v2-otel-collector 4318:4318 -n $NAMESPACE &
79-
# pf_pid=$!
80-
# sleep 10
81-
82-
# # Send test log
83-
# echo "Sending test log..."
84-
# timestamp=$(date +%s)
85-
# curl -X POST http://localhost:4318/v1/logs \
86-
# -H "Content-Type: application/json" \
87-
# -d '{
88-
# "resourceLogs": [{
89-
# "resource": {
90-
# "attributes": [
91-
# {"key": "service.name", "value": {"stringValue": "test-service"}},
92-
# {"key": "environment", "value": {"stringValue": "test"}}
93-
# ]
94-
# },
95-
# "scopeLogs": [{
96-
# "scope": {"name": "test-scope"},
97-
# "logRecords": [{
98-
# "timeUnixNano": "'${timestamp}'000000000",
99-
# "severityText": "INFO",
100-
# "body": {"stringValue": "Test log from deployment check"}
101-
# }]
102-
# }]
103-
# }]
104-
# }' > /dev/null 2>&1
105-
106-
# echo "Log sent"
107-
108-
# # Send test trace
109-
# echo "Sending test trace..."
110-
# trace_id=$(openssl rand -hex 16)
111-
# span_id=$(openssl rand -hex 8)
112-
# curl -X POST http://localhost:4318/v1/traces \
113-
# -H "Content-Type: application/json" \
114-
# -d '{
115-
# "resourceSpans": [{
116-
# "resource": {
117-
# "attributes": [
118-
# {"key": "service.name", "value": {"stringValue": "test-service"}}
119-
# ]
120-
# },
121-
# "scopeSpans": [{
122-
# "scope": {"name": "test-tracer"},
123-
# "spans": [{
124-
# "traceId": "'$trace_id'",
125-
# "spanId": "'$span_id'",
126-
# "name": "test-operation",
127-
# "kind": 1,
128-
# "startTimeUnixNano": "'${timestamp}'000000000",
129-
# "endTimeUnixNano": "'$((timestamp + 1))'000000000"
130-
# }]
131-
# }]
132-
# }]
133-
# }' > /dev/null 2>&1
134-
135-
# echo "Trace sent"
136-
137-
# kill $pf_pid 2>/dev/null || true
73+
# Test OTEL collector metrics endpoint
74+
echo "Testing OTEL collector metrics endpoint..."
75+
kubectl port-forward service/$RELEASE_NAME-hdx-oss-v2-otel-collector 8888:8888 -n $NAMESPACE &
76+
metrics_pf_pid=$!
77+
sleep 10
78+
79+
wait_for_service "http://localhost:8888/metrics" "OTEL Metrics"
80+
check_endpoint "http://localhost:8888/metrics" "200" "OTEL Metrics endpoint"
81+
82+
kill $metrics_pf_pid 2>/dev/null || true
83+
sleep 2
84+
85+
# Test data ingestion
86+
echo "Testing data ingestion..."
87+
kubectl port-forward service/$RELEASE_NAME-hdx-oss-v2-otel-collector 4318:4318 -n $NAMESPACE &
88+
pf_pid=$!
89+
sleep 10
90+
91+
# Test OTLP endpoint connectivity
92+
if ! nc -z localhost 4318; then
93+
echo "ERROR: OTEL HTTP endpoint not accessible"
94+
exit 1
95+
fi
96+
97+
# Send test log
98+
echo "Sending test log..."
99+
timestamp=$(date +%s)
100+
log_response=$(curl -X POST http://localhost:4318/v1/logs \
101+
-H "Content-Type: application/json" \
102+
-d '{
103+
"resourceLogs": [{
104+
"resource": {
105+
"attributes": [
106+
{"key": "service.name", "value": {"stringValue": "test-service"}},
107+
{"key": "environment", "value": {"stringValue": "test"}}
108+
]
109+
},
110+
"scopeLogs": [{
111+
"scope": {"name": "test-scope"},
112+
"logRecords": [{
113+
"timeUnixNano": "'${timestamp}'000000000",
114+
"severityText": "INFO",
115+
"body": {"stringValue": "Test log from deployment check"}
116+
}]
117+
}]
118+
}]
119+
}' -w "%{http_code}" -s -o /dev/null)
120+
121+
if [ "$log_response" = "200" ] || [ "$log_response" = "202" ]; then
122+
echo "Log sent successfully (status: $log_response)"
123+
else
124+
echo "WARNING: Log send failed with status: $log_response"
125+
fi
126+
127+
# Send test trace
128+
echo "Sending test trace..."
129+
trace_id=$(openssl rand -hex 16)
130+
span_id=$(openssl rand -hex 8)
131+
trace_response=$(curl -X POST http://localhost:4318/v1/traces \
132+
-H "Content-Type: application/json" \
133+
-d '{
134+
"resourceSpans": [{
135+
"resource": {
136+
"attributes": [
137+
{"key": "service.name", "value": {"stringValue": "test-service"}}
138+
]
139+
},
140+
"scopeSpans": [{
141+
"scope": {"name": "test-tracer"},
142+
"spans": [{
143+
"traceId": "'$trace_id'",
144+
"spanId": "'$span_id'",
145+
"name": "test-operation",
146+
"kind": 1,
147+
"startTimeUnixNano": "'${timestamp}'000000000",
148+
"endTimeUnixNano": "'$((timestamp + 1))'000000000"
149+
}]
150+
}]
151+
}]
152+
}' -w "%{http_code}" -s -o /dev/null)
153+
154+
if [ "$trace_response" = "200" ] || [ "$trace_response" = "202" ]; then
155+
echo "Trace sent successfully (status: $trace_response)"
156+
else
157+
echo "WARNING: Trace send failed with status: $trace_response"
158+
fi
159+
160+
kill $pf_pid 2>/dev/null || true
138161

139162
# Test databases
140163
echo "Testing ClickHouse..."
@@ -153,24 +176,27 @@ else
153176
exit 1
154177
fi
155178

156-
# # Check if data got ingested
157-
# echo "Waiting for data ingestion..."
158-
# sleep 30
179+
# Check if data got ingested
180+
echo "Waiting for data ingestion..."
181+
sleep 30
159182

160-
# echo "Checking ingested data..."
161-
# log_count=$(kubectl exec -n $NAMESPACE deployment/$RELEASE_NAME-hdx-oss-v2-clickhouse -- clickhouse-client --query "SELECT count() FROM default.otel_logs WHERE ServiceName = 'test-service'" 2>/dev/null || echo "0")
183+
echo "Checking ingested data..."
184+
log_count=$(kubectl exec -n $NAMESPACE deployment/$RELEASE_NAME-hdx-oss-v2-clickhouse -- clickhouse-client --query "SELECT count() FROM default.otel_logs WHERE ServiceName = 'test-service'" 2>/dev/null || echo "0")
185+
trace_count=$(kubectl exec -n $NAMESPACE deployment/$RELEASE_NAME-hdx-oss-v2-clickhouse -- clickhouse-client --query "SELECT count() FROM default.otel_traces WHERE ServiceName = 'test-service'" 2>/dev/null || echo "0")
162186

163-
# echo "Found $log_count test log records"
187+
echo "Found $log_count test log records"
188+
echo "Found $trace_count test trace records"
164189

165-
# if [ "$log_count" -gt "0" ]; then
166-
# echo "Data ingestion: OK"
167-
# else
168-
# echo "Data ingestion: No data found (may be normal for quick test)"
169-
# fi
190+
if [ "$log_count" -gt "0" ] || [ "$trace_count" -gt "0" ]; then
191+
echo "Data ingestion: OK"
192+
else
193+
echo "Data ingestion: No data found (may be normal for quick test or data processing delay)"
194+
fi
170195

171196
echo ""
172197
echo "Tests completed successfully"
173198
echo "- All components running"
174199
echo "- Endpoints responding"
175-
# echo "- Data ingestion working"
200+
echo "- OTEL collector metrics accessible"
201+
echo "- Data ingestion tested"
176202
echo "- Database connections OK"

0 commit comments

Comments
 (0)