Updated alert:

AllSplunkEnterpriseLevel - Email Sending Failures - to exclude a 9.3.3 warning

Updated macro:
search_type_from_sid - for subsearches

Updated reports:
SearchHeadLevel - Lookup file owners - description/comment update
SearchHeadLevel - Detect lookups that have not being accessed for a period of time - description/comment update

Author: gjanders

README.md

@@ -359,6 +359,24 @@ These are appear to be from premium apps but it does imply that there is a mecha
 Feel free to open an issue on github or use the contact author on the SplunkBase link and I will try to get back to you when possible, thanks!
 
 ## Release Notes
+### 4.0.4
+New reports:
+- `SearchHeadLevel - access logs kvstore usage`
+- `SearchHeadLevel - Lookup Watcher Recent Modification Summary`
+
+Updated alert:
+- `AllSplunkEnterpriseLevel - Email Sending Failures` - to exclude a warning noticed in 9.3.3
+
+Updated macro:
+- `search_type_from_sid` - for subsearches
+
+Updated reports:
+- `SearchHeadLevel - Lookup file owners` - description/comment update
+- `SearchHeadLevel - Detect lookups that have not being accessed for a period of time` - description/comment update
+
+
+Updated cron schedules of various reports to move them to different times
+
 ### 4.0.3
 New reports:
 - `SearchHeadLevel - Datamodel access summary`

default/app.conf

@@ -14,7 +14,7 @@ supported_themes = light,dark
 [launcher]
 author = Gareth Anderson
 description = Alerts and dashboards as described in the Splunk 2017 conf presentation How did you get so big?
-version = 4.0.3
+version = 4.0.4
 
 [package]
 id = SplunkAdmins

default/data/ui/nav/default.xml

@@ -277,6 +277,8 @@
 		    <saved name="SearchHeadLevel - platform_stats.remote_searches metrics populating search 24 hour" />
 		    <saved name="SearchHeadLevel - audit.log - lookup usage" />
 		    <saved name="SearchHeadLevel - Lookup Editor lookup updates" />
+		    <saved name="SearchHeadLevel - Lookup Watcher Recent Modification Summary" />
+	            <saved name="SearchHeadLevel - access logs kvstore usage" />
                     <saved name="IndexerLevel - platform_stats.counters hosts" />
                     <saved name="IndexerLevel - platform_stats.counters hosts 24hour" />
                     <saved name="IndexerLevel - platform_stats.indexers totalgb measurement" />
@@ -310,6 +312,7 @@
                         <saved name="SearchHeadLevel - Search Queries summary exact match by index" />
 			<saved name="SearchHeadLevel - Search Queries summary loadjob and savedsearch usage in audit logs" />
 			<saved name="SearchHeadLevel - Sourcetypes usage from search telemetry data" />
+			<saved name="SearchHeadLevel - access logs kvstore usage" />
 			<saved name="SearchHeadLevel - Searches by search type" />			
                         <saved name="SearchHeadLevel - IndexesPerUser Report" />
 			<saved name="SearchHeadLevel - license usage per sourcetype per index" />
@@ -322,6 +325,7 @@
 			<saved name="SearchHeadLevel - Jobs endpoint example" />
 			<saved name="SearchHeadLevel - configtracker index example" />
 			<saved name="SearchHeadLevel - configtracker index example2" />
+			<saved name="SearchHeadLevel - Lookup Watcher Recent Modification Summary" />
                         <saved name="IndexerLevel - RemoteSearches Indexes Stats" />
                         <saved name="IndexerLevel - RemoteSearches Indexes Stats Wilcard" />			
 			<saved name="IndexerLevel - RemoteSearches - lookup usage" />
@@ -468,6 +472,7 @@
 			<saved name="SearchHeadLevel - Lookup Editor lookup updates" />
 			<saved name="SearchHeadLevel - REST API usage via audit.log" />
 			<saved name="SearchHeadLevel - User created kvstore collections" />
+			<saved name="SearchHeadLevel - access logs kvstore usage" />
                         <saved name="IndexerLevel - RemoteSearches find all time searches" />
 			<saved name="IndexerLevel - RemoteSearches find datamodel acceleration with wildcards" />
 			<saved name="IndexerLevel - RemoteSearches - lookup usage" />			
@@ -483,6 +488,7 @@
 			<saved name="SearchHeadLevel - Lookup file owners" />
 			<saved name="SearchHeadLevel - Lookups within a dashboard" />
 			<saved name="SearchHeadLevel - Lookups within savedsearches" />
+			<saved name="SearchHeadLevel - Lookup Watcher Recent Modification Summary" />
 			<saved name="SearchHeadLevel - Knowledge bundle status on indexers" />
 			<saved name="SearchHeadLevel - Knowledge bundle replication times metrics.log" />		
 			<saved name="SearchHeadLevel - Knowledge Bundle contents" />
@@ -495,6 +501,8 @@
 		<collection label="Summary_Reports">
 			<saved name="SearchHeadLevel - audit.log - lookup usage" />
 			<saved name="SearchHeadLevel - Lookup Editor lookup updates" />
+			<saved name="SearchHeadLevel - Lookup Watcher Recent Modification Summary" />
+			<saved name="SearchHeadLevel - access logs kvstore usage" />
 			<saved name="SearchHeadLevel - license usage per sourcetype per index" />
 			<saved name="SearchHeadLevel - indexes per savedsearch" />
 			<saved name="SearchHeadLevel - macros in use" />
@@ -551,8 +559,10 @@
         <collection label="Summary_Reports">
 		<saved name="SearchHeadLevel - audit.log - lookup usage" />
 		<saved name="SearchHeadLevel - Lookup Editor lookup updates" />
+		<saved name="SearchHeadLevel - Lookup Watcher Recent Modification Summary" />
 		<saved name="SearchHeadLevel - license usage per sourcetype per index" />
 		<saved name="SearchHeadLevel - indexes per savedsearch" />
+		<saved name="SearchHeadLevel - access logs kvstore usage" />
 		<saved name="SearchHeadLevel - macros in use" />
 		<saved name="SearchHeadLevel - platform_stats.audit metrics searches" />
                 <saved name="SearchHeadLevel - platform_stats.audit metrics users" />

default/macros.conf

@@ -708,6 +708,7 @@ iseval = 0
 args = search_id
 definition = eval from=null(), username=null(), searchname2=null(), searchname=null()\
 | rex field=$search_id$ "'?(_rt)?(_?subsearch)*_?(?P<from>[^_]+)((_(?P<base64username>[^_]+))|(__(?P<username>[^_]+)))((__(?P<app>[^_]+)__(?P<searchname2>[^_]+))|(_(?P<base64appname>[^_]+)__(?P<searchname>[^_]+)))"\
+| rex field=$search_id$ "subsearch_(?P<username>[^_]+)__[^_]+(__(?P<app>[^_]+)__(?P<searchname2>[^_]+))" \
 | rex field=$search_id$ "^_?(?P<from>SummaryDirector)"\
   ```Pattern appears to vary but remote_<hostname>_ is consistent along with the optional _subsearch, the _from can be <username>__ownername__appname__RMD for dashboards as one pattern, it can also be unixepoch (ad-hoc), or scheduler__username__appname (scheduled search), or  username__owner__(something)__dashboardview, among others. RMD values can be translated via audit.log, scheduler.log or remote_searches.log (if savedsearch_name is there)!```\
 | fillnull from value="adhoc"\