Updated Splunk python SDK to 1.6.18

Corrected an issue where a DELETE combined with other operations could cause a stacktrace if using the `run_ko_diff` option

Author: gjanders

README.md

@@ -289,6 +289,11 @@ To do this you will need to install Version Control For SplunkCloud on your Splu
 [SplunkVersionControlCloud github](https://github.com/gjanders/SplunkVersionControlCloud)
 
 ## Release Notes
+### 1.2.7
+Updated Splunk python SDK to 1.6.18
+
+Corrected an issue where a DELETE combined with other operations could cause a stacktrace if using the `run_ko_diff` option 
+
 ### 1.2.6
 Updates to:
 `splunkversioncontrol_backup_class.py`

bin/splunkversioncontrol_backup_class.py

@@ -667,7 +667,7 @@ def process_ko_query_contents(self, results, tag):
                     if self.file_per_ko:
                         if not 'ko_name' in a_result:
                             logger.info("i=\"%s\" ko_name is null, cannot identify an exact changed object \"%s\" tag=%s" % (self.stanzaName, a_result, tag))
-                            overall_matches_list = []
+                            overall_matches_list = []                        
                             if 'user' in a_result:
                                 #TODO may not work on Windows
                                 find_str = "/" + a_result['user'] + "/"
@@ -676,7 +676,7 @@ def process_ko_query_contents(self, results, tag):
                                 find_str = a_result['ko_type']
                                 overall_matches_list = [ entry[0:entry.find(find_str)+len(find_str)] for entry in overall_matches if entry.find(find_str) != -1 ]
                             if len(overall_matches_list) > 0:
-                                overall_matches = overall_matches + list(set(overall_matches_list))
+                                overall_matches = overall_matches + list(set(overall_matches_list))                            
                         else:
                             file_name = self.create_file_name(a_result['ko_name'])
                             logger.debug("i=\"%s\" looking for file_name=%s tag=%s" % (self.stanzaName, file_name, tag))
@@ -690,7 +690,7 @@ def process_ko_query_contents(self, results, tag):
                             if len(match) > 0:
                                 overall_matches = match
                             else:
-                                if 'action' in a_result and a_result['action'].find("DELETE") == -1:
+                                if 'action' in a_result and "DELETE" in a_result['action']:
                                     logger.warn("i=\"%s\" looking for file_name=%s in file_str=%s tag=%s but did not find a match" % (self.stanzaName, file_name, file_str, tag))
                                 else:
                                     deleted_file_str = a_result['app_name'] + "/" + a_scope + "/" + a_result['ko_type'] + "/" + a_result['user'] + "/" + file_name

default/app.conf

@@ -12,7 +12,7 @@ label = SplunkVersionControl
 [launcher]
 author = Gareth Anderson
 description = Version Control software for Splunk instances (backup/restore from git)
-version = 1.2.6
+version = 1.2.7
 
 [package]
 id = SplunkVersionControl

lib/splunklib/__init__.py

@@ -16,5 +16,5 @@
 
 from __future__ import absolute_import
 from splunklib.six.moves import map
-__version_info__ = (1, 6, 16)
+__version_info__ = (1, 6, 18)
 __version__ = ".".join(map(str, __version_info__))

lib/splunklib/binding.py

@@ -30,7 +30,6 @@
 import logging
 import socket
 import ssl
-import sys
 from base64 import b64encode
 from contextlib import contextmanager
 from datetime import datetime
@@ -39,7 +38,6 @@
 from xml.etree.ElementTree import XML
 
 from splunklib import six
-from splunklib.six import StringIO
 from splunklib.six.moves import urllib
 
 from .data import record
@@ -471,7 +469,7 @@ class Context(object):
     """
     def __init__(self, handler=None, **kwargs):
         self.http = HttpLib(handler, kwargs.get("verify", False), key_file=kwargs.get("key_file"),
-                            cert_file=kwargs.get("cert_file"))  # Default to False for backward compat
+                            cert_file=kwargs.get("cert_file"), context=kwargs.get("context"))  # Default to False for backward compat
         self.token = kwargs.get("token", _NoAuthenticationToken)
         if self.token is None: # In case someone explicitly passes token=None
             self.token = _NoAuthenticationToken
@@ -1070,7 +1068,7 @@ def __init__(self, message, cause):
 #
 
 # Encode the given kwargs as a query string. This wrapper will also _encode
-# a list value as a sequence of assignemnts to the corresponding arg name,
+# a list value as a sequence of assignments to the corresponding arg name,
 # for example an argument such as 'foo=[1,2,3]' will be encoded as
 # 'foo=1&foo=2&foo=3'.
 def _encode(**kwargs):
@@ -1137,9 +1135,9 @@ class HttpLib(object):
 
     If using the default handler, SSL verification can be disabled by passing verify=False.
     """
-    def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None):
+    def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None, context=None):
         if custom_handler is None:
-            self.handler = handler(verify=verify, key_file=key_file, cert_file=cert_file)
+            self.handler = handler(verify=verify, key_file=key_file, cert_file=cert_file, context=context)
         else:
             self.handler = custom_handler
         self._cookies = {}
@@ -1351,7 +1349,7 @@ def readinto(self, byte_array):
         return bytes_read
 
 
-def handler(key_file=None, cert_file=None, timeout=None, verify=False):
+def handler(key_file=None, cert_file=None, timeout=None, verify=False, context=None):
     """This class returns an instance of the default HTTP request handler using
     the values you provide.
 
@@ -1363,6 +1361,8 @@ def handler(key_file=None, cert_file=None, timeout=None, verify=False):
     :type timeout: ``integer`` or "None"
     :param `verify`: Set to False to disable SSL verification on https connections.
     :type verify: ``Boolean``
+    :param `context`: The SSLContext that can is used with the HTTPSConnection when verify=True is enabled and context is specified
+    :type context: ``SSLContext`
     """
 
     def connect(scheme, host, port):
@@ -1376,6 +1376,10 @@ def connect(scheme, host, port):
 
             if not verify:
                 kwargs['context'] = ssl._create_unverified_context()
+            elif context:
+                # verify is True in elif branch and context is not None
+                kwargs['context'] = context
+
             return six.moves.http_client.HTTPSConnection(host, port, **kwargs)
         raise ValueError("unsupported scheme: %s" % scheme)
 
@@ -1385,7 +1389,7 @@ def request(url, message, **kwargs):
         head = {
             "Content-Length": str(len(body)),
             "Host": host,
-            "User-Agent": "splunk-sdk-python/1.6.16",
+            "User-Agent": "splunk-sdk-python/1.6.18",
             "Accept": "*/*",
             "Connection": "Close",
         } # defaults

lib/splunklib/client.py

@@ -295,7 +295,7 @@ def connect(**kwargs):
     :type port: ``integer``
     :param scheme: The scheme for accessing the service (the default is "https").
     :type scheme: "https" or "http"
-    :param verify: Enable (True) or disable (False) SSL verrification for
+    :param verify: Enable (True) or disable (False) SSL verification for
                    https connections. (optional, the default is True)
     :type verify: ``Boolean``
     :param `owner`: The owner context of the namespace (optional).
@@ -318,6 +318,8 @@ def connect(**kwargs):
     :type username: ``string``
     :param `password`: The password for the Splunk account.
     :type password: ``string``
+    :param `context`: The SSLContext that can be used when setting verify=True (optional)
+    :type context: ``SSLContext``
     :return: An initialized :class:`Service` connection.
 
     **Example**::
@@ -365,7 +367,7 @@ class Service(_BaseService):
     :type port: ``integer``
     :param scheme: The scheme for accessing the service (the default is "https").
     :type scheme: "https" or "http"
-    :param verify: Enable (True) or disable (False) SSL verrification for
+    :param verify: Enable (True) or disable (False) SSL verification for
                    https connections. (optional, the default is True)
     :type verify: ``Boolean``
     :param `owner`: The owner context of the namespace (optional; use "-" for wildcard).
@@ -401,6 +403,7 @@ class Service(_BaseService):
     def __init__(self, **kwargs):
         super(Service, self).__init__(**kwargs)
         self._splunk_version = None
+        self._kvstore_owner = None
 
     @property
     def apps(self):
@@ -673,12 +676,34 @@ def splunk_version(self):
             self._splunk_version = tuple([int(p) for p in self.info['version'].split('.')])
         return self._splunk_version
 
+    @property
+    def kvstore_owner(self):
+        """Returns the KVStore owner for this instance of Splunk.
+
+        By default is the kvstore owner is not set, it will return "nobody"
+        :return: A string with the KVStore owner.
+        """
+        if self._kvstore_owner is None:
+            self._kvstore_owner = "nobody"
+        return self._kvstore_owner
+
+    @kvstore_owner.setter
+    def kvstore_owner(self, value):
+        """
+        kvstore is refreshed, when the owner value is changed
+        """
+        self._kvstore_owner = value
+        self.kvstore
+
     @property
     def kvstore(self):
         """Returns the collection of KV Store collections.
 
+        sets the owner for the namespace, before retrieving the KVStore Collection
+
         :return: A :class:`KVStoreCollections` collection of :class:`KVStoreCollection` entities.
         """
+        self.namespace['owner'] = self.kvstore_owner
         return KVStoreCollections(self)
 
     @property
@@ -856,7 +881,7 @@ class Entity(Endpoint):
         ent.whitelist
 
     However, because some of the field names are not valid Python identifiers,
-    the dictionary-like syntax is preferrable.
+    the dictionary-like syntax is preferable.
 
     The state of an :class:`Entity` object is cached, so accessing a field
     does not contact the server. If you think the values on the
@@ -3619,7 +3644,7 @@ def __init__(self, collection):
         self.service = collection.service
         self.collection = collection
         self.owner, self.app, self.sharing = collection._proper_namespace()
-        self.path = 'storage/collections/data/' + UrlEncoded(self.collection.name) + '/'
+        self.path = 'storage/collections/data/' + UrlEncoded(self.collection.name, encode_slash=True) + '/'
 
     def _get(self, url, **kwargs):
         return self.service.get(self.path + url, owner=self.owner, app=self.app, sharing=self.sharing, **kwargs)
@@ -3640,6 +3665,11 @@ def query(self, **query):
         :return: Array of documents retrieved by query.
         :rtype: ``array``
         """
+
+        for key, value in query.items():
+            if isinstance(query[key], dict):
+                query[key] = json.dumps(value)
+
         return json.loads(self._get('', **query).body.read().decode('utf-8'))
 
     def query_by_id(self, id):
@@ -3652,7 +3682,7 @@ def query_by_id(self, id):
         :return: Document with id
         :rtype: ``dict``
         """
-        return json.loads(self._get(UrlEncoded(str(id))).body.read().decode('utf-8'))
+        return json.loads(self._get(UrlEncoded(str(id), encode_slash=True)).body.read().decode('utf-8'))
 
     def insert(self, data):
         """
@@ -3664,6 +3694,8 @@ def insert(self, data):
         :return: _id of inserted object
         :rtype: ``dict``
         """
+        if isinstance(data, dict):
+            data = json.dumps(data)
         return json.loads(self._post('', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8'))
 
     def delete(self, query=None):
@@ -3686,7 +3718,7 @@ def delete_by_id(self, id):
 
         :return: Result of DELETE request
         """
-        return self._delete(UrlEncoded(str(id)))
+        return self._delete(UrlEncoded(str(id), encode_slash=True))
 
     def update(self, id, data):
         """
@@ -3700,7 +3732,9 @@ def update(self, id, data):
         :return: id of replaced document
         :rtype: ``dict``
         """
-        return json.loads(self._post(UrlEncoded(str(id)), headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8'))
+        if isinstance(data, dict):
+            data = json.dumps(data)
+        return json.loads(self._post(UrlEncoded(str(id), encode_slash=True), headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8'))
 
     def batch_find(self, *dbqueries):
         """

lib/splunklib/modularinput/event_writer.py

@@ -15,7 +15,6 @@
 from __future__ import absolute_import
 import sys
 
-from io import TextIOWrapper, TextIOBase
 from splunklib.six import ensure_str
 from .event import ET
 

lib/splunklib/searchcommands/generating_command.py

@@ -15,6 +15,7 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function, unicode_literals
+import sys
 
 from .decorators import ConfigurationSetting
 from .search_command import SearchCommand
@@ -212,13 +213,49 @@ def _execute(self, ifile, process):
 
     def _execute_chunk_v2(self, process, chunk):
         count = 0
+        records = []
         for row in process:
-            self._record_writer.write_record(row)
+            records.append(row)
             count += 1
             if count == self._record_writer._maxresultrows:
-                self._finished = False
-                return
-        self._finished = True
+                break
+
+        for row in records:
+            self._record_writer.write_record(row)
+
+        if count == self._record_writer._maxresultrows:
+            self._finished = False
+        else:
+            self._finished = True
+
+    def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_input=True):
+        """ Process data.
+
+        :param argv: Command line arguments.
+        :type argv: list or tuple
+
+        :param ifile: Input data file.
+        :type ifile: file
+
+        :param ofile: Output data file.
+        :type ofile: file
+
+        :param allow_empty_input: For generating commands, it must be true. Doing otherwise will cause an error.
+        :type allow_empty_input: bool
+
+        :return: :const:`None`
+        :rtype: NoneType
+
+        """
+
+        # Generating commands are expected to run on an empty set of inputs as the first command being run in a search,
+        # also this class implements its own separate _execute_chunk_v2 method which does not respect allow_empty_input
+        # so ensure that allow_empty_input is always True
+
+        if not allow_empty_input:
+            raise ValueError("allow_empty_input cannot be False for Generating Commands")
+        else:
+            return super(GeneratingCommand, self).process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=True)
 
     # endregion
 

lib/splunklib/searchcommands/internals.py

@@ -508,6 +508,7 @@ def __init__(self, ofile, maxresultrows=None):
         self._chunk_count = 0
         self._pending_record_count = 0
         self._committed_record_count = 0
+        self.custom_fields = set()
 
     @property
     def is_flushed(self):
@@ -572,6 +573,7 @@ def write_record(self, record):
 
     def write_records(self, records):
         self._ensure_validity()
+        records = list(records)
         write_record = self._write_record
         for record in records:
             write_record(record)
@@ -593,6 +595,7 @@ def _write_record(self, record):
 
         if fieldnames is None:
             self._fieldnames = fieldnames = list(record.keys())
+            self._fieldnames.extend([i for i in self.custom_fields if i not in self._fieldnames])
             value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)), fieldnames)
             self._writerow(list(chain.from_iterable(value_list)))