Skip to content

Commit 2c4fd04

Browse files
author
David Noble
committed
DVPL-3313: splunklib | Improve the SDK to make it easier to talk back to Splunk's API from within a modular input or custom search command
Part 1: splunklib.searchcommands Added SearchCommand.service property that returns a service object or None. Verified with automated unit tests and ad-hoc integration tests. The service object is created from the Splunkd URI and the authentication token passed to the command invocation in the search results info file. This data is not passed to a command invocation by default. You must request it by specifying this pair of configuration settings in commands.conf: enableheader=true requires_srinfo=true The enableheader setting is true by default. Hence, you need not set it. The requires_srinfo setting is false by default. Hence, you must set it. If either enable header or requires_srinfo are false, SearchCommand.service will return None. The service object is instantiated with these arguments. Service( scheme=splunkd_protocol, host=splunkd_host, port=splunkd_port, token=auth_token, app=ppc_app) The arguments are obtained from SearchCommand.search_results_info. Signed-off-by: David Noble <[email protected]>
1 parent 72ea25d commit 2c4fd04

File tree

2 files changed

+130
-51
lines changed

2 files changed

+130
-51
lines changed

‎splunklib/searchcommands/search_command.py‎

Lines changed: 104 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
# Absolute imports
1818

19+
from splunklib.client import Service
20+
1921
try:
2022
from collections import OrderedDict # python 2.7
2123
except ImportError:
@@ -25,6 +27,7 @@
2527
from logging import getLevelName
2628
from os import path
2729
from sys import argv, stdin, stdout
30+
from urlparse import urlsplit
2831
from xml.etree import ElementTree
2932

3033
# Relative imports
@@ -57,6 +60,7 @@ def __init__(self):
5760
self._fieldnames = None
5861
self._option_view = None
5962
self._search_results_info = None
63+
self._service = None
6064

6165
self.parser = SearchCommandParser()
6266

@@ -146,64 +150,113 @@ def options(self):
146150

147151
@property
148152
def search_results_info(self):
149-
""" Returns the search results info for this command invocation or None
153+
""" Returns the search results info for this command invocation or None.
150154
151-
Splunk does not pass search results information by default. You must
152-
request it by specifying these configuration settings in commands.conf:
155+
The search results info object is created from the search results info
156+
file associated with the command invocation. Splunk does not pass the
157+
location of this file by default. You must request it by specifying
158+
these configuration settings in commands.conf:
153159
154160
.. code-block:: python
155161
enableheader=true
156162
requires_srinfo=true
157163
158-
Splunk will then pass the location of a search information file for
159-
each command command invocation in :code:`SearchCommand.input_headers[
160-
'infoPath']`. This property represents the contents of that file as a
161-
:code:`SearchResultsInfo` object.
164+
The :code:`enableheader` setting is :code:`true` by default. Hence, you
165+
need not set it. The :code:`requires_srinfo` setting is false by
166+
default. Hence, you must set it.
162167
163-
"""
164-
if self._search_results_info is None:
168+
:return: :class:SearchResultsInfo, if :code:`enableheader` and
169+
:code:`requires_srinfo` are both :code:`true`. Otherwise, if either
170+
:code:`enableheader` or :code:`requires_srinfo` are :code:`false`,
171+
a value of :code:`None` is returned.
165172
166-
try:
167-
info_path = self.input_header['infoPath']
168-
except KeyError:
169-
return None
170-
171-
self.logger.debug('infoPath = %s' % info_path)
172-
173-
def convert_field(field):
174-
return (field[1:] if field[0] == '_' else field).replace('.', '_')
175-
176-
def convert_value(field, value):
177-
178-
if field == 'countMap':
179-
split = value.split(';')
180-
value = {k: int(v) for k, v in zip(split[0::2], split[1::2])}
181-
elif field == 'vix_families':
182-
value = ElementTree.fromstring(value)
183-
elif value == '':
184-
value = None
185-
else:
186-
try:
187-
value = float(value)
188-
if value.is_integer():
189-
value = int(value)
190-
except ValueError:
191-
pass
192-
193-
return value
194-
195-
with open(info_path, 'rb') as f:
196-
from collections import namedtuple
197-
import csv
198-
reader = csv.reader(f, dialect='splunklib.searchcommands')
199-
fields = [convert_field(x) for x in reader.next()]
200-
values = [convert_value(f, v) for f, v in zip(fields,reader.next())]
201-
202-
search_results_info_type = namedtuple("SearchResultsInfo", fields)
203-
self._search_results_info = search_results_info_type._make(values)
173+
"""
174+
if self._search_results_info is not None:
175+
return self._search_results_info
176+
177+
try:
178+
info_path = self.input_header['infoPath']
179+
except KeyError:
180+
return None
181+
182+
def convert_field(field):
183+
return (field[1:] if field[0] == '_' else field).replace('.', '_')
184+
185+
def convert_value(field, value):
186+
187+
if field == 'countMap':
188+
split = value.split(';')
189+
value = {k: int(v) for k, v in zip(split[0::2], split[1::2])}
190+
elif field == 'vix_families':
191+
value = ElementTree.fromstring(value)
192+
elif value == '':
193+
value = None
194+
else:
195+
try:
196+
value = float(value)
197+
if value.is_integer():
198+
value = int(value)
199+
except ValueError:
200+
pass
201+
202+
return value
203+
204+
with open(info_path, 'rb') as f:
205+
from collections import namedtuple
206+
import csv
207+
reader = csv.reader(f, dialect='splunklib.searchcommands')
208+
fields = [convert_field(x) for x in reader.next()]
209+
values = [convert_value(f, v) for f, v in zip(fields, reader.next())]
210+
211+
search_results_info_type = namedtuple("SearchResultsInfo", fields)
212+
self._search_results_info = search_results_info_type._make(values)
204213

205214
return self._search_results_info
206215

216+
@property
217+
def service(self):
218+
""" Returns a Splunk service object for this command invocation or None.
219+
220+
The service object is created from the Splunkd URI and authentication
221+
token passed to the command invocation in the search results info file.
222+
This data is not passed to a command invocation by default. You must
223+
request it by specifying this pair of configuration settings in
224+
commands.conf:
225+
226+
.. code-block:: python
227+
enableheader=true
228+
requires_srinfo=true
229+
230+
The :code:`enableheader` setting is :code:`true` by default. Hence, you
231+
need not set it. The :code:`requires_srinfo` setting is false by
232+
default. Hence, you must set it.
233+
234+
:return: :class:splunklib.client.Service, if :code:`enableheader` and
235+
:code:`requires_srinfo` are both :code:`true`. Otherwise, if either
236+
:code:`enableheader` or :code:`requires_srinfo` are :code:`false`,
237+
a value of :code:`None` is returned.
238+
239+
"""
240+
if self._service is not None:
241+
return self._service
242+
243+
info = self.search_results_info
244+
245+
if info is None:
246+
return None
247+
248+
_, netloc, _, _, _ = urlsplit(
249+
info.splunkd_uri, info.splunkd_protocol, allow_fragments=False)
250+
251+
splunkd_host, _ = netloc.split(':')
252+
253+
self._service = Service(
254+
scheme=info.splunkd_protocol, host=splunkd_host,
255+
port=info.splunkd_port, token=info.auth_token,
256+
app=info.ppc_app)
257+
258+
return self._service
259+
207260
#endregion
208261

209262
#region Methods
@@ -255,14 +308,15 @@ def process(self, args=argv, input_file=stdin, output_file=stdout):
255308
self._configuration = ConfigurationSettings(self)
256309

257310
if self.show_configuration:
258-
self.messages.append('info_message',
259-
'%s command configuration settings: %s' %
260-
(self.name, self._configuration))
311+
self.messages.append(
312+
'info_message', '%s command configuration settings: %s'
313+
% (self.name, self._configuration))
261314

262315
writer = csv.DictWriter(output_file, self)
263316
self._execute(operation, reader, writer)
264317

265318
else:
319+
266320
file_name = path.basename(args[0])
267321
message = (
268322
'Command {0} appears to be statically configured and static '

‎tests/test_searchcommands_command.py‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import unittest
2121

2222
from splunklib.searchcommands import Configuration, StreamingCommand
23+
from splunklib.client import Service
2324
from cStringIO import StringIO
2425
import re
2526

@@ -31,7 +32,7 @@ def stream(self, records):
3132
for record in records:
3233
action = record['Action']
3334
if action == 'access_search_results_info':
34-
results = self.search_results_info
35+
search_results_info = self.search_results_info
3536
if action == 'raise_error':
3637
raise RuntimeError('Testing')
3738
yield {'Data': value}
@@ -133,11 +134,35 @@ def test_process(self):
133134
self.fail("Expected SystemExit, but no exception was raised")
134135

135136
# Command.process should provide access to search results info
137+
136138
expected = \
137139
'''SearchResultsInfo(sid=1391366014.3, timestamp=1391366014.757223, now=1391366014, setStart=1391366007, index_et=1391366011, index_lt=1391366011, startTime=1386621222, rt_earliest=None, rt_latest=None, rtspan=None, scan_count=0, drop_count=0, maxevents=0, countMap={'in_ct.command.fields': 76, 'duration.command.search': 13, 'invocations.startup.handoff': 1, 'duration.dispatch.results_combiner': 1, 'invocations.command.search.kv': 1, 'duration.dispatch.stream.local': 14, 'out_ct.command.head': 10, 'invocations.command.search.rawdata': 1, 'invocations.dispatch.results_combiner': 1, 'invocations.command.fields': 1, 'out_ct.command.search.typer': 76, 'invocations.dispatch.writeStatus': 3, 'invocations.command.search.index': 1, 'duration.command.head': 1, 'duration.dispatch.evaluate': 130, 'duration.dispatch.evaluate.search': 41, 'in_ct.command.search.calcfields': 76, 'duration.dispatch.writeStatus': 3, 'invocations.command.search.tags': 1, 'in_ct.command.search.lookups': 76, 'duration.command.search.tags': 1, 'duration.command.search.lookups': 1, 'invocations.dispatch.evaluate': 1, 'duration.command.fields': 1, 'invocations.command.search.summary': 1, 'duration.dispatch.evaluate.countmatches': 88, 'out_ct.command.prehead': 10, 'in_ct.command.search.typer': 76, 'out_ct.command.search.calcfields': 76, 'invocations.command.head': 1, 'duration.startup.handoff': 39, 'duration.command.search.rawdata': 2, 'duration.command.search.index.usec_1_8': 0, 'invocations.command.search.fieldalias': 1, 'duration.dispatch.createProviderQueue': 27, 'duration.dispatch.fetch': 14, 'out_ct.command.search.tags': 76, 'duration.command.search.typer': 4, 'in_ct.command.head': 10, 'invocations.dispatch.createProviderQueue': 1, 'out_ct.command.fields': 76, 'invocations.dispatch.evaluate.head': 1, 'out_ct.command.search.fieldalias': 76, 'invocations.command.search.typer': 1, 'duration.dispatch.check_disk_usage': 1, 'out_ct.command.search': 76, 'in_ct.command.search': 0, 'invocations.dispatch.fetch': 1, 'duration.command.search.index': 1, 'duration.command.search.summary': 1, 'in_ct.command.search.tags': 76, 'duration.dispatch.evaluate.head': 1, 'duration.command.search.kv': 7, 'invocations.command.search.calcfields': 1, 'duration.command.search.fieldalias': 1, 'invocations.dispatch.evaluate.countmatches': 1, 'in_ct.command.search.fieldalias': 76, 'invocations.command.search': 1, 'invocations.dispatch.stream.local': 1, 'duration.command.search.calcfields': 1, 'invocations.dispatch.check_disk_usage': 1, 'prereport_events': 0, 'invocations.dispatch.evaluate.search': 1, 'invocations.command.search.index.usec_1_8': 7, 'duration.command.prehead': 1, 'invocations.command.prehead': 1, 'invocations.command.search.lookups': 1, 'out_ct.command.search.lookups': 76, 'in_ct.command.prehead': 76}, columnOrder=None, keySet='index::_internal', remoteServers=None, is_remote_sorted=1, rt_backfill=0, read_raw=1, enable_event_stream=1, rtoptions=None, field_rendering=None, query_finished=1, request_finalization=0, auth_token='9ce2897f792c52a6ecdcd2e03aa4677c', splunkd_port=8089, splunkd_protocol='https', splunkd_uri='https://127.0.0.1:8089', internal_only=0, summary_mode='none', summary_maxtimespan=None, summary_stopped=0, is_batch_mode=0, root_sid=None, shp_id='A8797F6F-B6BF-43E9-9AFE-857D2FBC8534', search='search index=_internal | head 10 | countmatches fieldname=word_count pattern=\\\\\\\w+ uri', remote_search='litsearch index=_internal | fields keepcolorder=t "*" "_bkt" "_cd" "_si" "host" "index" "linecount" "source" "sourcetype" "splunk_server" "uri,word_count" | prehead limit=10 null=false keeplast=false', reduce_search=None, datamodel_map=None, tstats_reduce=None, normalized_search='litsearch index=_internal | fields keepcolorder=t "*" "_bkt" "_cd" "_si" "host" "index" "linecount" "source" "sourcetype" "splunk_server" "uri,word_count" | prehead limit=10 null=false keeplast=false', summary_id='A8797F6F-B6BF-43E9-9AFE-857D2FBC8534_searchcommands_app_admin_013dda14f276a384', normalized_summary_id='A8797F6F-B6BF-43E9-9AFE-857D2FBC8534_searchcommands_app_admin_NS91c147524d89ae5a', generation_id=0, label=None, is_saved_search=0, realtime=0, indexed_realtime=0, indexed_realtime_offset=0, ppc_app='searchcommands_app', ppc_user='admin', ppc_bs='$SPLUNK_HOME/etc', bundle_version=0, vix_families=<Element 'root' at 0x103a7e410>, tz='### SERIALIZED TIMEZONE FORMAT 1.0;Y-25200 YW 50 44 54;Y-28800 NW 50 53 54;Y-25200 YW 50 57 54;Y-25200 YG 50 50 54;@-1633269600 0;@-1615129200 1;@-1601820000 0;@-1583679600 1;@-880207200 2;@-769395600 3;@-765385200 1;@-687967200 0;@-662655600 1;@-620834400 0;@-608137200 1;@-589384800 0;@-576082800 1;@-557935200 0;@-544633200 1;@-526485600 0;@-513183600 1;@-495036000 0;@-481734000 1;@-463586400 0;@-450284400 1;@-431532000 0;@-418230000 1;@-400082400 0;@-386780400 1;@-368632800 0;@-355330800 1;@-337183200 0;@-323881200 1;@-305733600 0;@-292431600 1;@-273679200 0;@-260982000 1;@-242229600 0;@-226508400 1;@-210780000 0;@-195058800 1;@-179330400 0;@-163609200 1;@-147880800 0;@-131554800 1;@-116431200 0;@-100105200 1;@-84376800 0;@-68655600 1;@-52927200 0;@-37206000 1;@-21477600 0;@-5756400 1;@9972000 0;@25693200 1;@41421600 0;@57747600 1;@73476000 0;@89197200 1;@104925600 0;@120646800 1;@126698400 0;@152096400 1;@162381600 0;@183546000 1;@199274400 0;@215600400 1;@230724000 0;@247050000 1;@262778400 0;@278499600 1;@294228000 0;@309949200 1;@325677600 0;@341398800 1;@357127200 0;@372848400 1;@388576800 0;@404902800 1;@420026400 0;@436352400 1;@452080800 0;@467802000 1;@483530400 0;@499251600 1;@514980000 0;@530701200 1;@544615200 0;@562150800 1;@576064800 0;@594205200 1;@607514400 0;@625654800 1;@638964000 0;@657104400 1;@671018400 0;@688554000 1;@702468000 0;@720003600 1;@733917600 0;@752058000 1;@765367200 0;@783507600 1;@796816800 0;@814957200 1;@828871200 0;@846406800 1;@860320800 0;@877856400 1;@891770400 0;@909306000 1;@923220000 0;@941360400 1;@954669600 0;@972810000 1;@986119200 0;@1004259600 1;@1018173600 0;@1035709200 1;@1049623200 0;@1067158800 1;@1081072800 0;@1099213200 1;@1112522400 0;@1130662800 1;@1143972000 0;@1162112400 1;@1173607200 0;@1194166800 1;@1205056800 0;@1225616400 1;@1236506400 0;@1257066000 1;@1268560800 0;@1289120400 1;@1300010400 0;@1320570000 1;@1331460000 0;@1352019600 1;@1362909600 0;@1383469200 1;@1394359200 0;@1414918800 1;@1425808800 0;@1446368400 1;@1457863200 0;@1478422800 1;@1489312800 0;@1509872400 1;@1520762400 0;@1541322000 1;@1552212000 0;@1572771600 1;@1583661600 0;@1604221200 1;@1615716000 0;@1636275600 1;@1647165600 0;@1667725200 1;@1678615200 0;@1699174800 1;@1710064800 0;@1730624400 1;@1741514400 0;@1762074000 1;@1772964000 0;@1793523600 1;@1805018400 0;@1825578000 1;@1836468000 0;@1857027600 1;@1867917600 0;@1888477200 1;@1899367200 0;@1919926800 1;@1930816800 0;@1951376400 1;@1962871200 0;@1983430800 1;@1994320800 0;@2014880400 1;@2025770400 0;@2046330000 1;@2057220000 0;@2077779600 1;@2088669600 0;@2109229200 1;@2120119200 0;@2140678800 1;$', msgType=None, msg=None)'''
138140

141+
command = SearchCommand()
142+
139143
command.process(args=['foo.py', '__EXECUTE__'], input_file=StringIO('infoPath:searchcommands_data/input/externSearchResultsInfo.csv\n\nAction\r\naccess_search_results_info'), output_file=result)
140144
observed = re.sub('''vix_families=<Element 'root' at 0x[0-9a-f]+>''', '''vix_families=<Element 'root' at 0x103a7e410>''', repr(command.search_results_info))
141145
self.assertEqual(expected, observed)
142146

147+
# Command.process should provide access to a service object when search
148+
# results info is available
149+
150+
self.assertIsInstance(command.service, Service)
151+
self.assertEqual(command.service.authority, command.search_results_info.splunkd_uri)
152+
self.assertEqual(command.service.scheme, command.search_results_info.splunkd_protocol)
153+
self.assertEqual(command.service.port, command.search_results_info.splunkd_port)
154+
self.assertEqual(command.service.token, command.search_results_info.auth_token)
155+
self.assertEqual(command.service.namespace.app, command.search_results_info.ppc_app)
156+
self.assertEqual(command.service.namespace.owner, None)
157+
self.assertEqual(command.service.namespace.sharing, None)
158+
159+
# Command.process should not provide access to search results info or
160+
# a service object when the 'infoPath' input header is unavailable
161+
162+
command = SearchCommand()
163+
164+
command.process(args=['foo.py', '__EXECUTE__'], input_file=StringIO('\nAction\r\naccess_search_results_info'), output_file=result)
165+
self.assertEqual(command.search_results_info, None)
166+
self.assertEqual(command.service, None)
167+
143168
return

0 commit comments

Comments
 (0)