test: add OTEL-to-ClickHouse ingestion assertions to integration smoke test (#200)

dhable · web-flow · commit 1a85f26ec5ea · 2026-03-25T21:42:52.000Z
## Summary - extend `scripts/smoke-test.sh` with ClickHouse query helpers, port-forward lifecycle cleanup, and ingestion polling - generate OTLP traces/logs with `telemetrygen` and send them to the in-cluster collector (`4318`) - assert `default.otel_traces` and `default.otel_logs` row counts increase after telemetry is sent - run smoke tests with `TIMEOUT=300` in the integration workflow so failures surface sooner - set `hyperdx.config.OPAMP_SERVER_URL: ""` in CI test values so the collector runs in standalone mode with built-in ClickHouse export pipelines ## Test - [x] `bash -n scripts/smoke-test.sh` - [x] `Helm Chart Integration Test` passed on this PR (`test-helm-chart`) - [x] `helm-unittest` passed on this PR ## New Test Assertions - fetch `CLICKHOUSE_APP_PASSWORD` from `clickstack-secret` and query ClickHouse over `8123` as `app` - capture baseline counts from `default.otel_traces` and `default.otel_logs` - send synthetic OTLP traces/logs to the collector - poll until each table count is greater than baseline; fail on timeout/query/auth errors ## Why This Would Catch `6f29e730856cb5bcc30138dd168794bfdb17441d` That commit fixed a grant-shape issue where split `app` grants could result in missing `SELECT` privileges (`SELECT ON default.*` / `SELECT ON system.*`) because only the first grant was effectively applied. The new integration assertions actively query `default.otel_traces` and `default.otel_logs` as the `app` user before and after telemetry ingestion. If the old broken grant shape were reintroduced, those `SELECT` checks would fail (or remain unreadable), causing the smoke test and PR checks to fail automatically. Made with [Cursor](https://cursor.com)
diff --git a/.github/workflows/chart-test.yml b/.github/workflows/chart-test.yml
@@ -86,10 +86,12 @@ jobs:
       - name: Deploy ClickStack chart (Phase 2)
         run: |
           # Create test values for faster deployment
-          cat > test-values.yaml << EOF
+          cat > test-values.yaml << 'EOF'
           hyperdx:
             secrets:
               HYPERDX_API_KEY: "test-api-key-for-ci"
+            config:
+              OPAMP_SERVER_URL: ""
             deployment:
               replicas: 1
             service:
@@ -139,11 +141,16 @@ jobs:
 
           kubectl get services
 
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+
       - name: Run comprehensive smoke tests
         run: |
           chmod +x ./scripts/smoke-test.sh
 
-          RELEASE_NAME=hyperdx-test NAMESPACE=default ./scripts/smoke-test.sh
+          TIMEOUT=300 RELEASE_NAME=hyperdx-test NAMESPACE=default ./scripts/smoke-test.sh
 
       - name: Collect logs on failure
         if: failure()
diff --git a/scripts/e2e/package.json b/scripts/e2e/package.json
@@ -0,0 +1,6 @@
+{
+  "private": true,
+  "devDependencies": {
+    "@playwright/test": "^1.58.2"
+  }
+}
diff --git a/scripts/e2e/playwright.config.ts b/scripts/e2e/playwright.config.ts
@@ -0,0 +1,15 @@
+import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+  testDir: '.',
+  timeout: 120_000,
+  retries: 1,
+  use: {
+    baseURL: 'http://localhost:3000',
+    trace: 'on-first-retry',
+    screenshot: 'only-on-failure',
+  },
+  projects: [
+    { name: 'chromium', use: { ...devices['Desktop Chrome'] } },
+  ],
+});
diff --git a/scripts/e2e/search-log.spec.ts b/scripts/e2e/search-log.spec.ts
@@ -0,0 +1,26 @@
+import { test, expect } from '@playwright/test';
+
+const TEST_EMAIL = 'smoke@test.local';
+const TEST_PASSWORD = 'SmokeTest1234!';
+const SEARCH_TERM = 'clickstack smoke test log';
+
+test('register user and verify log appears on search page', async ({ page }) => {
+  await page.goto('/register');
+
+  await page.getByRole('textbox', { name: /email/i }).fill(TEST_EMAIL);
+  await page.locator('input[name="password"]').fill(TEST_PASSWORD);
+  await page.locator('input[name="confirmPassword"]').fill(TEST_PASSWORD);
+  await page.getByRole('button', { name: 'Create' }).click();
+
+  await page.waitForURL('**/search**', { timeout: 60_000 });
+
+  const searchInput = page.getByTestId('search-input');
+  await expect(searchInput).toBeVisible({ timeout: 30_000 });
+  await searchInput.fill(SEARCH_TERM);
+  await page.getByTestId('search-submit-button').click();
+  await page.waitForLoadState('networkidle');
+
+  const resultsTable = page.getByTestId('search-results-table');
+  await expect(resultsTable).toBeVisible({ timeout: 30_000 });
+  await expect(resultsTable).toContainText(SEARCH_TERM, { timeout: 30_000 });
+});
diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh
@@ -1,17 +1,49 @@
 #!/bin/bash
 set -e
+set -o pipefail
 
 # Test script for HyperDX deployment
 NAMESPACE=${NAMESPACE:-default}
 RELEASE_NAME=${RELEASE_NAME:-hyperdx-test}
 CHART_NAME=${CHART_NAME:-clickstack}
 TIMEOUT=${TIMEOUT:-300}
+CLICKHOUSE_SERVICE=${CLICKHOUSE_SERVICE:-$RELEASE_NAME-$CHART_NAME-clickhouse-clickhouse-headless}
+CLICKHOUSE_SECRET_NAME=${CLICKHOUSE_SECRET_NAME:-clickstack-secret}
+CLICKHOUSE_HTTP_USER=${CLICKHOUSE_HTTP_USER:-app}
+CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE:-default}
+CLICKHOUSE_TRACE_TABLE=${CLICKHOUSE_TRACE_TABLE:-otel_traces}
+CLICKHOUSE_LOG_TABLE=${CLICKHOUSE_LOG_TABLE:-otel_logs}
+INGESTION_POLL_INTERVAL=${INGESTION_POLL_INTERVAL:-5}
+OTEL_TELEMETRYGEN_IMAGE=${OTEL_TELEMETRYGEN_IMAGE:-ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest}
+OTEL_SIGNAL_COUNT=${OTEL_SIGNAL_COUNT:-20}
+
+PORT_FORWARD_PIDS=()
+PORT_FORWARD_LOGS=()
+CLICKHOUSE_HTTP_PASSWORD=""
 
 echo "Starting HyperDX tests..."
 echo "Release: $RELEASE_NAME"
 echo "Chart: $CHART_NAME"
 echo "Namespace: $NAMESPACE"
 
+cleanup_port_forwards() {
+    local pid=""
+    local log_file=""
+
+    for pid in "${PORT_FORWARD_PIDS[@]}"; do
+        if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
+            kill "$pid" 2>/dev/null || true
+            wait "$pid" 2>/dev/null || true
+        fi
+    done
+
+    for log_file in "${PORT_FORWARD_LOGS[@]}"; do
+        rm -f "$log_file" 2>/dev/null || true
+    done
+}
+
+trap cleanup_port_forwards EXIT
+
 wait_for_service() {
     local url=$1
     local name=$2
@@ -39,6 +71,7 @@ check_endpoint() {
     local url=$1
     local expected_code=$2
     local desc=$3
+    local code=""
     
     echo "Checking $desc..."
     
@@ -53,6 +86,148 @@ check_endpoint() {
     fi
 }
 
+start_port_forward() {
+    local resource=$1
+    local local_port=$2
+    local remote_port=$3
+    local name=$4
+    local log_file=""
+    local pid=""
+
+    log_file=$(mktemp "/tmp/${name}.XXXXXX.log")
+    echo "Starting port-forward for $name (${resource} ${local_port}:${remote_port})..." >&2
+    kubectl port-forward "$resource" "${local_port}:${remote_port}" -n "$NAMESPACE" >"$log_file" 2>&1 &
+    pid=$!
+
+    PORT_FORWARD_PIDS+=("$pid")
+    PORT_FORWARD_LOGS+=("$log_file")
+
+    sleep 3
+    if ! kill -0 "$pid" 2>/dev/null; then
+        echo "ERROR: Failed to start port-forward for $name" >&2
+        sed -n '1,120p' "$log_file" >&2 || true
+        return 1
+    fi
+
+    echo "$pid"
+}
+
+stop_port_forward() {
+    local pid=$1
+
+    if [ -n "${pid:-}" ] && kill -0 "$pid" 2>/dev/null; then
+        kill "$pid" 2>/dev/null || true
+        wait "$pid" 2>/dev/null || true
+    fi
+}
+
+get_secret_value() {
+    local secret_name=$1
+    local key_name=$2
+
+    kubectl get secret "$secret_name" -n "$NAMESPACE" -o "jsonpath={.data.${key_name}}" | base64 --decode
+}
+
+run_clickhouse_query() {
+    local sql=$1
+
+    curl -sS --fail \
+        -u "${CLICKHOUSE_HTTP_USER}:${CLICKHOUSE_HTTP_PASSWORD}" \
+        --data-binary "$sql" \
+        "http://localhost:8123/?database=${CLICKHOUSE_DATABASE}"
+}
+
+get_table_count() {
+    local table=$1
+    local count=""
+
+    count=$(run_clickhouse_query "SELECT count() FROM \`${CLICKHOUSE_DATABASE}\`.\`${table}\`;")
+    count=$(echo "$count" | tr -d '[:space:]')
+
+    if [[ ! "$count" =~ ^[0-9]+$ ]]; then
+        echo "ERROR: Non-numeric count for table ${table}: ${count}"
+        return 1
+    fi
+
+    echo "$count"
+}
+
+wait_for_table_queryable() {
+    local table=$1
+    local timeout_seconds=$2
+    local start_time=0
+    local now=0
+    local count=""
+
+    start_time=$(date +%s)
+    while true; do
+        count=$(get_table_count "$table" 2>/dev/null || true)
+        if [[ "$count" =~ ^[0-9]+$ ]]; then
+            echo "$count"
+            return 0
+        fi
+
+        now=$(date +%s)
+        if [ $((now - start_time)) -ge "$timeout_seconds" ]; then
+            echo "ERROR: Timed out waiting for table ${CLICKHOUSE_DATABASE}.${table} to become queryable"
+            return 1
+        fi
+
+        sleep "$INGESTION_POLL_INTERVAL"
+    done
+}
+
+wait_for_table_count_increase() {
+    local table=$1
+    local baseline_count=$2
+    local timeout_seconds=$3
+    local start_time=0
+    local now=0
+    local current_count=""
+
+    start_time=$(date +%s)
+    while true; do
+        current_count=$(get_table_count "$table" 2>/dev/null || true)
+        if [[ "$current_count" =~ ^[0-9]+$ ]]; then
+            echo "Current count for ${CLICKHOUSE_DATABASE}.${table}: ${current_count} (baseline ${baseline_count})"
+            if [ "$current_count" -gt "$baseline_count" ]; then
+                echo "Detected new rows in ${CLICKHOUSE_DATABASE}.${table}"
+                return 0
+            fi
+        fi
+
+        now=$(date +%s)
+        if [ $((now - start_time)) -ge "$timeout_seconds" ]; then
+            echo "ERROR: Timed out waiting for row increase in ${CLICKHOUSE_DATABASE}.${table}"
+            return 1
+        fi
+
+        sleep "$INGESTION_POLL_INTERVAL"
+    done
+}
+
+send_telemetrygen_signal() {
+    local signal=$1
+    local count_flag=$2
+    local count=$3
+    local run_id=$4
+    local body_arg=()
+
+    if [ "$signal" = "logs" ]; then
+        body_arg=(--body "clickstack smoke test log ${run_id}")
+    fi
+
+    echo "Sending ${signal} to OTEL collector over OTLP HTTP..."
+    docker run --rm --network host "$OTEL_TELEMETRYGEN_IMAGE" "$signal" \
+        --otlp-http \
+        --otlp-endpoint "localhost:4318" \
+        --otlp-insecure \
+        "$count_flag" "$count" \
+        --rate 5 \
+        --service "clickstack-smoke-test" \
+        "${body_arg[@]}"
+}
+
 # Check pods
 echo "Checking pod status..."
 kubectl wait --for=condition=Ready pods -l app.kubernetes.io/instance=$RELEASE_NAME --timeout=${TIMEOUT}s -n $NAMESPACE
@@ -62,26 +237,24 @@ kubectl get pods -l app.kubernetes.io/instance=$RELEASE_NAME -n $NAMESPACE
 
 # Test UI
 echo "Testing HyperDX UI..."
-kubectl port-forward service/$RELEASE_NAME-$CHART_NAME-app 3000:3000 -n $NAMESPACE &
-pf_pid=$!
-sleep 10
+pf_pid=$(start_port_forward "service/$RELEASE_NAME-$CHART_NAME-app" "3000" "3000" "hyperdx-ui")
+sleep 2
 
 wait_for_service "http://localhost:3000" "HyperDX UI"
 check_endpoint "http://localhost:3000" "200" "UI"
 
-kill $pf_pid 2>/dev/null || true
+stop_port_forward "$pf_pid"
 sleep 2
 
 # Test OTEL collector metrics endpoint
 echo "Testing OTEL collector metrics endpoint..."
-kubectl port-forward service/$RELEASE_NAME-otel-collector 8888:8888 -n $NAMESPACE &
-metrics_pf_pid=$!
-sleep 10
+metrics_pf_pid=$(start_port_forward "service/$RELEASE_NAME-otel-collector" "8888" "8888" "otel-metrics")
+sleep 2
 
 wait_for_service "http://localhost:8888/metrics" "OTEL Metrics"
 check_endpoint "http://localhost:8888/metrics" "200" "OTEL Metrics endpoint"
 
-kill $metrics_pf_pid 2>/dev/null || true
+stop_port_forward "$metrics_pf_pid"
 sleep 2
 
 # Verify OTEL Collector Deployment is Available
@@ -105,11 +278,62 @@ else
     exit 1
 fi
 
+# Verify OTEL data ingestion to ClickHouse
+echo "Verifying OTEL ingestion into ClickHouse..."
+otlp_http_pf_pid=$(start_port_forward "service/$RELEASE_NAME-otel-collector" "4318" "4318" "otel-http")
+clickhouse_pf_pid=$(start_port_forward "service/$CLICKHOUSE_SERVICE" "8123" "8123" "clickhouse-http")
+
+CLICKHOUSE_HTTP_PASSWORD=$(get_secret_value "$CLICKHOUSE_SECRET_NAME" "CLICKHOUSE_APP_PASSWORD")
+if [ -z "${CLICKHOUSE_HTTP_PASSWORD:-}" ]; then
+    echo "ERROR: Could not read CLICKHOUSE_APP_PASSWORD from secret ${CLICKHOUSE_SECRET_NAME}"
+    exit 1
+fi
+
+trace_baseline=$(wait_for_table_queryable "$CLICKHOUSE_TRACE_TABLE" "$TIMEOUT")
+log_baseline=$(wait_for_table_queryable "$CLICKHOUSE_LOG_TABLE" "$TIMEOUT")
+echo "Baseline count ${CLICKHOUSE_DATABASE}.${CLICKHOUSE_TRACE_TABLE}: ${trace_baseline}"
+echo "Baseline count ${CLICKHOUSE_DATABASE}.${CLICKHOUSE_LOG_TABLE}: ${log_baseline}"
+
+if ! command -v docker > /dev/null 2>&1; then
+    echo "ERROR: docker is required to run telemetrygen for OTEL ingestion checks"
+    exit 1
+fi
+
+run_id=$(date +%s)
+send_telemetrygen_signal "traces" "--traces" "$OTEL_SIGNAL_COUNT" "$run_id"
+send_telemetrygen_signal "logs" "--logs" "$OTEL_SIGNAL_COUNT" "$run_id"
+
+echo "Waiting for traces/logs to land in ClickHouse..."
+
+wait_for_table_count_increase "$CLICKHOUSE_TRACE_TABLE" "$trace_baseline" "$TIMEOUT"
+wait_for_table_count_increase "$CLICKHOUSE_LOG_TABLE" "$log_baseline" "$TIMEOUT"
+
+stop_port_forward "$otlp_http_pf_pid"
+stop_port_forward "$clickhouse_pf_pid"
+
+# Verify app works end-to-end with default connection (register + search)
+echo "Running Playwright e2e test..."
+ui_pf_pid=$(start_port_forward "service/$RELEASE_NAME-$CHART_NAME-app" "3000" "3000" "hyperdx-ui-e2e")
+sleep 2
+wait_for_service "http://localhost:3000" "HyperDX UI (e2e)"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+(
+    cd "$SCRIPT_DIR/e2e"
+    npm install
+    npx playwright install --with-deps chromium
+    npx playwright test
+)
+
+stop_port_forward "$ui_pf_pid"
+
 echo ""
 echo "All smoke tests passed"
 echo "- All pods running"
 echo "- HyperDX UI responding"
 echo "- OTEL Collector metrics accessible"
 echo "- OTEL Collector Deployment available"
 echo "- ClickHouseCluster reconciled (Ready)"
-echo "- MongoDBCommunity reconciled (Running)"
+echo "- MongoDBCommunity reconciled (Running)"
+echo "- OTEL traces and logs persisted to ClickHouse"
+echo "- App registers user and displays logs via default connection"
