DVPL-3903: splunklib.searchcommands | Search results info should be exposed as a property of a search command

Verified with automated unit tests and ad-hoc integration tests.

Signed-off-by: David Noble <[email protected]>

Author: David Noble

splunklib/searchcommands/search_command.py

@@ -25,6 +25,7 @@
 from logging import getLevelName
 from os import path
 from sys import argv, stdin, stdout
+from xml.etree import ElementTree
 
 # Relative imports
 
@@ -53,8 +54,9 @@ def __init__(self):
 
         self._default_logging_level = self.logger.level
         self._configuration = None
-        self._option_view = None
         self._fieldnames = None
+        self._option_view = None
+        self._search_results_info = None
 
         self.parser = SearchCommandParser()
 
@@ -117,10 +119,16 @@ def logging_level(self, value):
 
     @property
     def configuration(self):
+        """ Returns the configuration settings for this command
+
+        """
         return self._configuration
 
     @property
     def fieldnames(self):
+        """ Returns the fieldnames specified as argument to this command
+
+        """
         return self._fieldnames
 
     @fieldnames.setter
@@ -129,10 +137,73 @@ def fieldnames(self, value):
 
     @property
     def options(self):
+        """ Returns the options specified as argument to this command
+
+        """
         if self._option_view is None:
             self._option_view = Option.View(self)
         return self._option_view
 
+    @property
+    def search_results_info(self):
+        """ Returns the search results info for this command invocation or None
+
+        Splunk does not pass search results information by default. You must
+        request it by specifying these configuration settings in commands.conf:
+
+        .. code-block:: python
+            enableheader=true
+            requires_srinfo=true
+
+        Splunk will then pass the location of a search information file for
+        each command command invocation in :code:`SearchCommand.input_headers[
+        'infoPath']`. This property represents the contents of that file as a
+        :code:`SearchResultsInfo` object.
+
+        """
+        if self._search_results_info is None:
+
+            try:
+                info_path = self.input_header['infoPath']
+            except KeyError:
+                return None
+
+            self.logger.debug('infoPath = %s' % info_path)
+
+            def convert_field(field):
+                return (field[1:] if field[0] == '_' else field).replace('.', '_')
+
+            def convert_value(field, value):
+
+                if field == 'countMap':
+                    split = value.split(';')
+                    value = {k: int(v) for k, v in zip(split[0::2], split[1::2])}
+                elif field == 'vix_families':
+                    value = ElementTree.fromstring(value)
+                elif value == '':
+                    value = None
+                else:
+                    try:
+                        value = float(value)
+                        if value.is_integer():
+                            value = int(value)
+                    except ValueError:
+                        pass
+
+                return value
+
+            with open(info_path, 'rb') as f:
+                from collections import namedtuple
+                import csv
+                reader = csv.reader(f, dialect='splunklib.searchcommands')
+                fields = [convert_field(x) for x in reader.next()]
+                values = [convert_value(f, v) for f, v in zip(fields,reader.next())]
+
+            search_results_info_type = namedtuple("SearchResultsInfo", fields)
+            self._search_results_info = search_results_info_type._make(values)
+
+        return self._search_results_info
+
     #endregion
 
     #region Methods

splunklib/searchcommands/search_command_internals.py

@@ -138,7 +138,7 @@ def read(self, input_file):
 
     def _update(self, k, v):
         if k is not None:
-            self._settings[k] = v if k != 'infoPath' else open(v, 'r')
+            self._settings[k] = v
         return
 
 

tests/searchcommands_data/input/externSearchResultsInfo.csv

@@ -0,0 +1,13 @@
+"_sid","_timestamp",now,"_setStart","_index_et","_index_lt","_startTime","_rt_earliest","_rt_latest","_rtspan","_scan_count","_drop_count","_maxevents","_countMap","_columnOrder","_keySet","_remoteServers","is_remote_sorted","rt_backfill","read_raw","enable_event_stream","_rtoptions","field_rendering","_query_finished","_request_finalization","_auth_token","_splunkd_port","_splunkd_protocol","_splunkd_uri","internal_only","summary_mode","summary_maxtimespan","summary_stopped","is_batch_mode","_root_sid","_shp_id","_search","_remote_search","_reduce_search","_datamodel_map","_tstats_reduce","_normalized_search","summary_id","normalized_summary_id","generation_id",label,"is_saved_search",realtime,"indexed_realtime","indexed_realtime_offset","_ppc.app","_ppc.user","_ppc.bs","_bundle_version","vix_families","_tz",msgType,msg
+"1391366014.3","1391366014.757223000","1391366014.000000000","1391366007.000000000","1391366011.000000000","1391366011.000000000","1386621222.000000000","","","",0,0,0,"duration.command.fields;1;duration.command.head;1;duration.command.prehead;1;duration.command.search;13;duration.command.search.calcfields;1;duration.command.search.fieldalias;1;duration.command.search.index;1;duration.command.search.index.usec_1_8;0;duration.command.search.kv;7;duration.command.search.lookups;1;duration.command.search.rawdata;2;duration.command.search.summary;1;duration.command.search.tags;1;duration.command.search.typer;4;duration.dispatch.check_disk_usage;1;duration.dispatch.createProviderQueue;27;duration.dispatch.evaluate;130;duration.dispatch.evaluate.countmatches;88;duration.dispatch.evaluate.head;1;duration.dispatch.evaluate.search;41;duration.dispatch.fetch;14;duration.dispatch.results_combiner;1;duration.dispatch.stream.local;14;duration.dispatch.writeStatus;3;duration.startup.handoff;39;in_ct.command.fields;76;in_ct.command.head;10;in_ct.command.prehead;76;in_ct.command.search;0;in_ct.command.search.calcfields;76;in_ct.command.search.fieldalias;76;in_ct.command.search.lookups;76;in_ct.command.search.tags;76;in_ct.command.search.typer;76;invocations.command.fields;1;invocations.command.head;1;invocations.command.prehead;1;invocations.command.search;1;invocations.command.search.calcfields;1;invocations.command.search.fieldalias;1;invocations.command.search.index;1;invocations.command.search.index.usec_1_8;7;invocations.command.search.kv;1;invocations.command.search.lookups;1;invocations.command.search.rawdata;1;invocations.command.search.summary;1;invocations.command.search.tags;1;invocations.command.search.typer;1;invocations.dispatch.check_disk_usage;1;invocations.dispatch.createProviderQueue;1;invocations.dispatch.evaluate;1;invocations.dispatch.evaluate.countmatches;1;invocations.dispatch.evaluate.head;1;invocations.dispatch.evaluate.search;1;invocations.dispatch.fetch;1;invocations.dispatch.results_combiner;1;invocations.dispatch.stream.local;1;invocations.dispatch.writeStatus;3;invocations.startup.handoff;1;out_ct.command.fields;76;out_ct.command.head;10;out_ct.command.prehead;10;out_ct.command.search;76;out_ct.command.search.calcfields;76;out_ct.command.search.fieldalias;76;out_ct.command.search.lookups;76;out_ct.command.search.tags;76;out_ct.command.search.typer;76;prereport_events;0;","","index::_internal","",1,0,1,1,"","",1,0,9ce2897f792c52a6ecdcd2e03aa4677c,8089,https,"https://127.0.0.1:8089",0,none,"",0,0,"","A8797F6F-B6BF-43E9-9AFE-857D2FBC8534","search index=_internal | head 10 | countmatches fieldname=word_count pattern=\\w+ uri","litsearch index=_internal | fields  keepcolorder=t ""*"" ""_bkt"" ""_cd"" ""_si"" ""host"" ""index"" ""linecount"" ""source"" ""sourcetype"" ""splunk_server"" ""uri,word_count"" | prehead  limit=10 null=false keeplast=false","","","","litsearch index=_internal | fields keepcolorder=t ""*"" ""_bkt"" ""_cd"" ""_si"" ""host"" ""index"" ""linecount"" ""source"" ""sourcetype"" ""splunk_server"" ""uri,word_count"" | prehead limit=10 null=false keeplast=false","A8797F6F-B6BF-43E9-9AFE-857D2FBC8534_searchcommands_app_admin_013dda14f276a384","A8797F6F-B6BF-43E9-9AFE-857D2FBC8534_searchcommands_app_admin_NS91c147524d89ae5a",0,"",0,0,0,0,"searchcommands_app",admin,"$SPLUNK_HOME/etc",0,"<root>
+  <add_ons>
+    <add_on name=""hadoop"" type=""external_results_provider"">
+      <parameter key=""erp_type"" value=""report""/>
+      <parameter key=""max_nodes"" value=""50""/>
+    </add_on>
+  </add_ons>
+</root>
+","### SERIALIZED TIMEZONE FORMAT 1.0;Y-25200 YW 50 44 54;Y-28800 NW 50 53 54;Y-25200 YW 50 57 54;Y-25200 YG 50 50 54;@-1633269600 0;@-1615129200 1;@-1601820000 0;@-1583679600 1;@-880207200 2;@-769395600 3;@-765385200 1;@-687967200 0;@-662655600 1;@-620834400 0;@-608137200 1;@-589384800 0;@-576082800 1;@-557935200 0;@-544633200 1;@-526485600 0;@-513183600 1;@-495036000 0;@-481734000 1;@-463586400 0;@-450284400 1;@-431532000 0;@-418230000 1;@-400082400 0;@-386780400 1;@-368632800 0;@-355330800 1;@-337183200 0;@-323881200 1;@-305733600 0;@-292431600 1;@-273679200 0;@-260982000 1;@-242229600 0;@-226508400 1;@-210780000 0;@-195058800 1;@-179330400 0;@-163609200 1;@-147880800 0;@-131554800 1;@-116431200 0;@-100105200 1;@-84376800 0;@-68655600 1;@-52927200 0;@-37206000 1;@-21477600 0;@-5756400 1;@9972000 0;@25693200 1;@41421600 0;@57747600 1;@73476000 0;@89197200 1;@104925600 0;@120646800 1;@126698400 0;@152096400 1;@162381600 0;@183546000 1;@199274400 0;@215600400 1;@230724000 0;@247050000 1;@262778400 0;@278499600 1;@294228000 0;@309949200 1;@325677600 0;@341398800 1;@357127200 0;@372848400 1;@388576800 0;@404902800 1;@420026400 0;@436352400 1;@452080800 0;@467802000 1;@483530400 0;@499251600 1;@514980000 0;@530701200 1;@544615200 0;@562150800 1;@576064800 0;@594205200 1;@607514400 0;@625654800 1;@638964000 0;@657104400 1;@671018400 0;@688554000 1;@702468000 0;@720003600 1;@733917600 0;@752058000 1;@765367200 0;@783507600 1;@796816800 0;@814957200 1;@828871200 0;@846406800 1;@860320800 0;@877856400 1;@891770400 0;@909306000 1;@923220000 0;@941360400 1;@954669600 0;@972810000 1;@986119200 0;@1004259600 1;@1018173600 0;@1035709200 1;@1049623200 0;@1067158800 1;@1081072800 0;@1099213200 1;@1112522400 0;@1130662800 1;@1143972000 0;@1162112400 1;@1173607200 0;@1194166800 1;@1205056800 0;@1225616400 1;@1236506400 0;@1257066000 1;@1268560800 0;@1289120400 1;@1300010400 0;@1320570000 1;@1331460000 0;@1352019600 1;@1362909600 0;@1383469200 1;@1394359200 0;@1414918800 1;@1425808800 0;@1446368400 1;@1457863200 0;@1478422800 1;@1489312800 0;@1509872400 1;@1520762400 0;@1541322000 1;@1552212000 0;@1572771600 1;@1583661600 0;@1604221200 1;@1615716000 0;@1636275600 1;@1647165600 0;@1667725200 1;@1678615200 0;@1699174800 1;@1710064800 0;@1730624400 1;@1741514400 0;@1762074000 1;@1772964000 0;@1793523600 1;@1805018400 0;@1825578000 1;@1836468000 0;@1857027600 1;@1867917600 0;@1888477200 1;@1899367200 0;@1919926800 1;@1930816800 0;@1951376400 1;@1962871200 0;@1983430800 1;@1994320800 0;@2014880400 1;@2025770400 0;@2046330000 1;@2057220000 0;@2077779600 1;@2088669600 0;@2109229200 1;@2120119200 0;@2140678800 1;$",,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,DEBUG,"The 'countmatches' command is implemented as an external script and may cause the search to be significantly slower."
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,DEBUG,"base lispy: [ AND index::_internal ]"
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,DEBUG,"search context: user=""admin"", app=""searchcommands_app"", bs-pathname=""/Applications/Splunk/etc"""