Fix mcollect syntax and performance metric collection

Co-authored-by: devinslick <[email protected]>

Author: Copilot

default/data/ui/views/dashboard_details.xml

@@ -143,16 +143,16 @@
       <title>Load Time Trend Over Time</title>
       <chart>
         <search>
-          <query>| mstats avg(avg_load_time) as avg_load, max(max_load_time) as max_load WHERE index=caca_metrics AND pretty_name="$dashboard_name$" AND metric_name="dashboard.load_time" span=1h 
-| timechart avg(avg_load) as "Avg Load Time (ms)" max(max_load) as "Max Load Time (ms)" span=1h</query>
+          <query>| mstats avg(_value) as avg_load WHERE index=caca_metrics AND pretty_name="$dashboard_name$" AND metric_name="dashboard.load_time" span=1h 
+| timechart avg(avg_load) as "Avg Load Time (ms)" span=1h</query>
           <earliest>$time_range.earliest$</earliest>
           <latest>$time_range.latest$</latest>
         </search>
         <option name="charting.chart">line</option>
         <option name="charting.axisTitleX.text">Date</option>
         <option name="charting.axisTitleY.text">Load Time (ms)</option>
         <option name="charting.legend.placement">bottom</option>
-        <option name="charting.seriesColors">[0xF8BE34,0xDC4E41]</option>
+        <option name="charting.seriesColors">[0xF8BE34]</option>
         <option name="charting.drilldown">none</option>
       </chart>
     </panel>

default/data/ui/views/dashboard_leaderboard.xml

@@ -225,7 +225,7 @@
         <search>
           <query>`get_slow_dashboards`
 | head 10
-| rename pretty_name as "Dashboard", app as "App", avg_load_time_7d as "Avg Load (ms)", max_load_time_7d as "Max Load (ms)", performance_status as "Status"</query>
+| rename pretty_name as "Dashboard", app as "App", avg_load_time_7d as "Avg Load (ms)", performance_status as "Status"</query>
           <earliest>-7d@h</earliest>
           <latest>now</latest>
         </search>
@@ -237,9 +237,6 @@
         <format type="number" field="Avg Load (ms)">
           <option name="precision">0</option>
         </format>
-        <format type="number" field="Max Load (ms)">
-          <option name="precision">0</option>
-        </format>
         <drilldown>
           <link target="_blank">/app/splunk-content-monitoring-console/dashboard_details?form.dashboard_name=$row.Dashboard$</link>
         </drilldown>

default/data/ui/views/poop_deck.xml

@@ -285,12 +285,11 @@
       <title>🐌 The Slow - Dashboards with Performance Issues (Last 7 Days)</title>
       <table>
         <search>
-          <query>| mstats avg(avg_load_time) as avg_load, max(max_load_time) as max_load WHERE index=caca_metrics AND metric_name="dashboard.load_time" BY pretty_name, app span=1d 
+          <query>| mstats avg(_value) as avg_load WHERE index=caca_metrics AND metric_name="dashboard.load_time" BY pretty_name, app span=1d 
 | where _time >= relative_time(now(), "-7d")
-| stats avg(avg_load) as avg_load_time, max(max_load) as max_load_time by pretty_name, app
+| stats avg(avg_load) as avg_load_time by pretty_name, app
 | where avg_load_time &gt; 3000
 | eval avg_load_time=round(avg_load_time, 0)
-| eval max_load_time=round(max_load_time, 0)
 | lookup dashboard_registry pretty_name OUTPUT owner description
 | join type=left pretty_name 
     [| mstats sum(_value) as views WHERE index=caca_metrics AND metric_name="dashboard.views" BY pretty_name span=1d
@@ -312,8 +311,8 @@
     avg_load_time &gt; 7000, "High priority optimization",
     avg_load_time &gt; 5000, "Should optimize",
     1=1, "Consider optimization")
-| table pretty_name app avg_load_time max_load_time views_7d performance_level user_impact recommendation owner description
-| rename pretty_name as "Dashboard", app as "App", avg_load_time as "Avg Load (ms)", max_load_time as "Max Load (ms)", views_7d as "Views (7d)", performance_level as "Performance", user_impact as "Impact", recommendation as "Recommendation", owner as "Owner", description as "Description"
+| table pretty_name app avg_load_time views_7d performance_level user_impact recommendation owner description
+| rename pretty_name as "Dashboard", app as "App", avg_load_time as "Avg Load (ms)", views_7d as "Views (7d)", performance_level as "Performance", user_impact as "Impact", recommendation as "Recommendation", owner as "Owner", description as "Description"
 | sort -"Avg Load (ms)"</query>
           <earliest>-7d@d</earliest>
           <latest>now</latest>
@@ -326,12 +325,6 @@
         <format type="number" field="Avg Load (ms)">
           <option name="precision">0</option>
         </format>
-        <format type="number" field="Max Load (ms)">
-          <option name="precision">0</option>
-        </format>
-        <format type="number" field="Views (7d)">
-          <option name="precision">0</option>
-        </format>
         <format type="number" field="Views (7d)">
           <option name="precision">0</option>
         </format>

default/macros.conf

@@ -70,27 +70,25 @@ iseval = 0
 
 [get_dashboard_performance(1)]
 args = dashboard_name
-definition = | mstats avg(avg_load_time) as avg_load_time, max(max_load_time) as max_load_time WHERE index=caca_metrics AND pretty_name="$dashboard_name$" AND metric_name="dashboard.load_time" BY pretty_name span=1h \
+definition = | mstats avg(_value) as avg_load_time WHERE index=caca_metrics AND pretty_name="$dashboard_name$" AND metric_name="dashboard.load_time" BY pretty_name span=1h \
 | eval avg_load_time=round(avg_load_time, 2) \
-| eval max_load_time=round(max_load_time, 2) \
 | eval performance_rating=case(\
-    avg_load_time < 1000, "Excellent",\
-    avg_load_time < 3000, "Good",\
-    avg_load_time < 5000, "Fair",\
+    avg_load_time &lt; 1000, "Excellent",\
+    avg_load_time &lt; 3000, "Good",\
+    avg_load_time &lt; 5000, "Fair",\
     1=1, "Poor")
 iseval = 0
 
 [get_slow_dashboards]
-definition = | mstats avg(avg_load_time) as avg_load_time, max(max_load_time) as max_load_time WHERE index=caca_metrics AND metric_name="dashboard.load_time" BY pretty_name, app span=1d \
+definition = | mstats avg(_value) as avg_load_time WHERE index=caca_metrics AND metric_name="dashboard.load_time" BY pretty_name, app span=1d \
 | where _time >= relative_time(now(), "-7d") \
-| stats avg(avg_load_time) as avg_load_time_7d, max(max_load_time) as max_load_time_7d by pretty_name, app \
-| where avg_load_time_7d > 3000 \
+| stats avg(avg_load_time) as avg_load_time_7d by pretty_name, app \
+| where avg_load_time_7d &gt; 3000 \
 | eval avg_load_time_7d=round(avg_load_time_7d, 0) \
-| eval max_load_time_7d=round(max_load_time_7d, 0) \
 | eval performance_status=case(\
-    avg_load_time_7d > 10000, "Critical",\
-    avg_load_time_7d > 5000, "Poor",\
-    avg_load_time_7d > 3000, "Fair",\
+    avg_load_time_7d &gt; 10000, "Critical",\
+    avg_load_time_7d &gt; 5000, "Poor",\
+    avg_load_time_7d &gt; 3000, "Fair",\
     1=1, "Good") \
 | sort -avg_load_time_7d
 iseval = 0

default/savedsearches.conf

@@ -85,11 +85,11 @@ search = index=_internal sourcetype=splunkd_ui_access uri_path="/*/app/*" \
 | where isnotnull(pretty_name) \
 | eval load_time_ms=tonumber(spent) \
 | where isnotnull(load_time_ms) AND load_time_ms > 0 \
-| stats avg(load_time_ms) as avg_load_time, max(load_time_ms) as max_load_time, count as request_count by dashboard_uri, pretty_name, app, user \
+| stats sum(load_time_ms) as total_load_time, count as request_count by dashboard_uri, pretty_name, app, user \
+| eval avg_load_time=round(total_load_time/request_count, 2) \
 | eval metric_name="dashboard.load_time" \
 | eval activity_type="performance" \
-| foreach avg_load_time max_load_time [ eval <<FIELD>>=round(<<FIELD>>, 2) ] \
-| mcollect index=caca_metrics metric_name=metric_name pretty_name app user activity_type avg_load_time max_load_time split=t
+| mcollect index=caca_metrics metric_name=metric_name _value=avg_load_time pretty_name app user activity_type split=t
 schedule_priority = low
 schedule_window = 5