Update Dashboard Performance collector to use REST API for better accuracy

Author: devinslick

default/savedsearches.conf

@@ -47,11 +47,11 @@ dispatchAs = owner
 action.email.useNSSubject = 1
 alert.track = 0
 cron_schedule = */10 * * * *
-description = Collects dashboard load time and performance metrics
+description = Collects dashboard load time and performance metrics via REST API
 dispatch.earliest_time = -10m
 dispatch.latest_time = now
 enableSched = 1
-search = index=_internal sourcetype=splunkd_ui_access uri_path="/*/app/*" | rex field=uri_path "/[^/]+/app/(?<app>[^/]+)/(?<dashboard_name>[^/?]+)" | where isnotnull(dashboard_name) AND isnotnull(spent) | eval dashboard_uri="/app/".app."/".dashboard_name | lookup dashboard_registry dashboard_uri OUTPUT pretty_name app as reg_app owner | where isnotnull(pretty_name) | eval load_time_ms=tonumber(spent) | where isnotnull(load_time_ms) AND load_time_ms > 0 | stats sum(load_time_ms) as total_load_time, count as request_count by dashboard_uri, pretty_name, app, user | eval _value=round(total_load_time/request_count, 2), activity_type="performance", metric_name="dashboard.load_time" | mcollect index=caca_metrics split=t _value pretty_name app user activity_type
+search = | rest /services/search/jobs | rename eai:acl.app as app, author as user, runDuration as runtime, provenance as provenance | rex field=provenance "UI:Dashboard:(?<dashboard_name>.+)" | search dashboard_name=* | eval dashboard_uri="/app/".app."/".dashboard_name | lookup dashboard_registry dashboard_uri OUTPUT pretty_name | where isnotnull(pretty_name) | stats sum(runtime) as total_runtime, count as search_count by dashboard_uri, pretty_name, app, user | eval _value=round(total_runtime, 2), activity_type="performance", metric_name="dashboard.load_time" | mcollect index=caca_metrics split=t _value pretty_name app user activity_type
 schedule_priority = default
 schedule_window = 5
 dispatchAs = owner