Fix dashboard errors, enable scheduled searches, and add test automation job

Author: devinslick

default/savedsearches.conf

@@ -11,7 +11,7 @@ cron_schedule = */5 * * * *
 description = Collects dashboard view metrics from splunkd_ui_access logs
 dispatch.earliest_time = -5m
 dispatch.latest_time = now
-enableSched = 0
+enableSched = 1
 search = index=_internal sourcetype=splunkd_ui_access uri_path="/*/app/*" | rex field=uri_path "/[^/]+/app/(?<app>[^/]+)/(?<dashboard_name>[^/?]+)" | where isnotnull(dashboard_name) AND dashboard_name!="search" AND dashboard_name!="home" AND dashboard_name!="launcher" AND dashboard_name!="analytics_workspace" | eval dashboard_uri="/app/".app."/".dashboard_name | lookup dashboard_registry dashboard_uri OUTPUT pretty_name | where isnotnull(pretty_name) | stats count by dashboard_uri, pretty_name, app, user | eval activity_type="view", metric_name="dashboard.views" | mcollect index=caca_metrics split=t pretty_name app user activity_type
 schedule_priority = default
 schedule_window = 5
@@ -24,7 +24,7 @@ cron_schedule = */10 * * * *
 description = Collects dashboard edit/creation metrics from audit logs
 dispatch.earliest_time = -10m
 dispatch.latest_time = now
-enableSched = 0
+enableSched = 1
 search = index=_audit action=edit_ui_view | rex field=object "(?<app>[^:]+):(?<dashboard_name>.+)" | where isnotnull(dashboard_name) | eval dashboard_uri="/app/".app."/".dashboard_name | lookup dashboard_registry dashboard_uri OUTPUT pretty_name app as reg_app owner | where isnotnull(pretty_name) | stats count by dashboard_uri, pretty_name, app, user | eval activity_type="edit", metric_name="dashboard.edits" | mcollect index=caca_metrics split=t pretty_name app user activity_type
 schedule_priority = default
 schedule_window = 5
@@ -37,7 +37,7 @@ cron_schedule = */15 * * * *
 description = Collects dashboard health metrics including errors and performance
 dispatch.earliest_time = -15m
 dispatch.latest_time = now
-enableSched = 0
+enableSched = 1
 search = index=_internal (sourcetype=splunkd log_level=ERROR OR log_level=WARN) (component=ScheduledViewsReporter OR component=DashboardController OR component=SimpleXML) | rex field=_raw "view=(?<dashboard_name>[^\\s,]+)" | rex field=_raw "app=(?<app>[^\\s,]+)" | where isnotnull(dashboard_name) AND isnotnull(app) | eval dashboard_uri="/app/".app."/".dashboard_name | lookup dashboard_registry dashboard_uri OUTPUT pretty_name app as reg_app owner | where isnotnull(pretty_name) | stats count by dashboard_uri, pretty_name, app, log_level | eval severity=lower(log_level), activity_type="health", metric_name="dashboard.errors" | mcollect index=caca_metrics split=t pretty_name app severity activity_type
 schedule_priority = default
 schedule_window = 5
@@ -50,7 +50,7 @@ cron_schedule = */10 * * * *
 description = Collects dashboard load time and performance metrics
 dispatch.earliest_time = -10m
 dispatch.latest_time = now
-enableSched = 0
+enableSched = 1
 search = index=_internal sourcetype=splunkd_ui_access uri_path="/*/app/*" | rex field=uri_path "/[^/]+/app/(?<app>[^/]+)/(?<dashboard_name>[^/?]+)" | where isnotnull(dashboard_name) AND isnotnull(spent) | eval dashboard_uri="/app/".app."/".dashboard_name | lookup dashboard_registry dashboard_uri OUTPUT pretty_name app as reg_app owner | where isnotnull(pretty_name) | eval load_time_ms=tonumber(spent) | where isnotnull(load_time_ms) AND load_time_ms > 0 | stats sum(load_time_ms) as total_load_time, count as request_count by dashboard_uri, pretty_name, app, user | eval _value=round(total_load_time/request_count, 2), activity_type="performance", metric_name="dashboard.load_time" | mcollect index=caca_metrics split=t _value pretty_name app user activity_type
 schedule_priority = default
 schedule_window = 5
@@ -67,7 +67,7 @@ cron_schedule = 0 2 * * *
 description = Automatically updates dashboard registry via REST API, respecting app filter configuration
 dispatch.earliest_time = -5m
 dispatch.latest_time = now
-enableSched = 0
+enableSched = 1
 search = | rest /services/data/ui/views splunk_server=local count=0 | search isDashboard=1 OR isVisible=1 | eval dashboard_uri="/app/".'eai:acl.app'."/".title | eval pretty_name=coalesce(label, title) | eval app='eai:acl.app' | eval owner='eai:acl.owner' | eval sharing='eai:acl.sharing' | rex field="eai:data" "<description>(?<xml_description>.*?)</description>" | eval description=coalesce(xml_description, "") | eval status="active" | lookup app_filter app OUTPUT include | where isnull(include) OR include="true" OR include="1" OR include="yes" | fields - include | table dashboard_uri pretty_name app owner sharing description status | outputlookup dashboard_registry.csv
 schedule_priority = default
 dispatchAs = owner

local/local-environment.yaml

@@ -245,6 +245,8 @@ spec:
   ports:
   - name: http
     port: 8000
+  - name: mgmt
+    port: 8089
   type: ClusterIP
   selector:
     app: splunk
@@ -278,3 +280,70 @@ spec:
   type: ClusterIP
   selector:
     app: splunk
+---
+# Test Runner Job
+# Automates the test environment setup: runs registry update, generates traffic, and runs collectors
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: caca-test-runner
+  namespace: splunk-caca
+spec:
+  ttlSecondsAfterFinished: 600
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: test-runner
+        image: curlimages/curl:latest
+        env:
+          - name: SPLUNK_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: splunk-secret
+                key: password
+          - name: SPLUNK_HOST
+            value: "splunk-http"
+        command:
+        - /bin/sh
+        - -c
+        - |
+          echo "Waiting for Splunk Management API (8089)..."
+          until curl -s -k -o /dev/null https://${SPLUNK_HOST}:8089/services/server/info; do
+            echo "Splunk not ready yet..."
+            sleep 10
+          done
+
+          # Wait a bit more for login to be fully ready
+          echo "Splunk API is responding. Waiting 30s for full startup..."
+          sleep 30
+
+          echo "=== 1. Running Registry Update ==="
+          curl -k -u admin:${SPLUNK_PASSWORD} https://${SPLUNK_HOST}:8089/services/saved/searches/Dashboard%20Registry%20-%20Auto%20Update/dispatch -d trigger_actions=1 -X POST
+          echo "Registry Update Dispatched."
+          sleep 5
+
+          echo "=== 2. Generating Traffic (Views) ==="
+          DASHBOARDS="dashboard_leaderboard poop_deck caca_admin dashboard_details"
+          for dash in $DASHBOARDS; do
+            echo "Viewing $dash..."
+            for i in 1 2 3; do
+              curl -s -u admin:${SPLUNK_PASSWORD} -o /dev/null "http://${SPLUNK_HOST}:8000/en-US/app/splunk-content-monitoring-console/$dash"
+            done
+          done
+
+          echo "=== 3. Simulating Dashboard Edit ==="
+          # Update description to trigger an edit event
+          EDIT_URL="https://${SPLUNK_HOST}:8089/servicesNS/admin/splunk-content-monitoring-console/data/ui/views/dashboard_leaderboard"
+          DESC="Content Activity Checking Application - Monitor all dashboard usage, health, and activity (Automated Test Edit)"
+          curl -k -u admin:${SPLUNK_PASSWORD} $EDIT_URL -d "description=$DESC"
+          echo "Dashboard Edited."
+
+          echo "=== 4. Running Metric Collectors ==="
+          COLLECTORS="Dashboard%20Views%20-%20Metrics%20Collector Dashboard%20Edits%20-%20Metrics%20Collector Dashboard%20Health%20-%20Metrics%20Collector Dashboard%20Performance%20-%20Metrics%20Collector"
+          for collector in $COLLECTORS; do
+            echo "Running $collector..."
+            curl -k -u admin:${SPLUNK_PASSWORD} https://${SPLUNK_HOST}:8089/services/saved/searches/$collector/dispatch -d trigger_actions=1 -X POST
+          done
+
+          echo "=== Test Automation Complete ==="