Version 1.1.8 - Merged pull request from calesanz #16 to allow a new sslVerify option to pass in the CA certificate file, or to leave SSL validation disabled

In addition this pull request adds a requestingAddress which optionally controls the call-back ip when using the postversioncontrolrestore command
Finally this pull requests adds scripts and a testing suite using docker

Updated python SDK to 1.6.15

Minor fixes to the pull request

Author: gjanders

README.md

@@ -265,6 +265,11 @@ To do this you will need to install Version Control For SplunkCloud on your Splu
 ## Release Notes 
 ### 1.1.8
 README.md update - git repositories must be dedicated per-backup and not shared with other items as the root level / top level directory is used
+Merged pull request from calesanz https://github.com/gjanders/SplunkVersionControl/pull/16 to allow a new sslVerify option to pass in the CA certificate file, or to leave SSL validation disabled 
+In addition this pull request adds a requestingAddress which optionally controls the call-back ip when using the postversioncontrolrestore command
+Finally this pull requests adds scripts and a testing suite using docker into the github version, for SplunkBase the test directory is removed (you can access it on https://github.com/gjanders/SplunkVersionControl)
+
+Updated Splunk python SDK to 1.6.15
 
 ### 1.1.7
 Increase timeout for commands to a default of 60 seconds

SplunkVersionControl.tgz

@@ -163,7 +163,7 @@ def handle_POST(self):
         else:
             time_wait = 600
 
-        if 'requestingAddress' in payload:
+        if 'requestingAddress' in payload and payload['requestingAddress'][0].lower() != 'false':
             requestingAddress = payload['requestingAddress'][0]
         else:
             requestingAddress = None

bin/splunkversioncontrol_rest_restore.py

@@ -12,7 +12,7 @@ label = SplunkVersionControl
 [launcher]
 author = Gareth Anderson
 description = Version Control software for Splunk instances (backup/restore from git)
-version = 1.1.7
+version = 1.1.8
 
 [package]
 id = SplunkVersionControl

default/app.conf

@@ -20,10 +20,10 @@ definition = 30
 iseval = 0
 
 [sslVerify]
-definition = false
+definition = False
 iseval = 0
 
 [requestingAddress]
-definition = false
+definition = False
 iseval = 0
 

default/macros.conf

@@ -16,5 +16,5 @@
 
 from __future__ import absolute_import
 from splunklib.six.moves import map
-__version_info__ = (1, 6, 14)
+__version_info__ = (1, 6, 15)
 __version__ = ".".join(map(str, __version_info__))

lib/splunklib/__init__.py

@@ -724,7 +724,12 @@ def post(self, path_segment, owner=None, app=None, sharing=None, headers=None, *
         :type headers: ``list`` of 2-tuples.
         :param query: All other keyword arguments, which are used as query
             parameters.
-        :type query: ``string``
+        :param body: Parameters to be used in the post body. If specified,
+            any parameters in the query will be applied to the URL instead of
+            the body. If a dict is supplied, the key-value pairs will be form
+            encoded. If a string is supplied, the body will be passed through
+            in the request unchanged.
+        :type body: ``dict`` or ``str``
         :return: The response from the server.
         :rtype: ``dict`` with keys ``body``, ``headers``, ``reason``,
                 and ``status``
@@ -1223,6 +1228,8 @@ def post(self, url, headers=None, **kwargs):
                 headers.append(("Content-Type", "application/x-www-form-urlencoded"))
 
             body = kwargs.pop('body')
+            if isinstance(body, dict):
+                body = _encode(**body).encode('utf-8')
             if len(kwargs) > 0:
                 url = url + UrlEncoded('?' + _encode(**kwargs), skip_encode=True)
         else:
@@ -1378,7 +1385,7 @@ def request(url, message, **kwargs):
         head = {
             "Content-Length": str(len(body)),
             "Host": host,
-            "User-Agent": "splunk-sdk-python/1.6.14",
+            "User-Agent": "splunk-sdk-python/1.6.15",
             "Accept": "*/*",
             "Connection": "Close",
         } # defaults

lib/splunklib/binding.py

@@ -1872,7 +1872,7 @@ def delete(self, username, realm=None):
             name = UrlEncoded(realm, encode_slash=True) + ":" + UrlEncoded(username, encode_slash=True)
 
         # Append the : expected at the end of the name
-        if name[-1] is not ":":
+        if name[-1] != ":":
             name = name + ":"
         return Collection.delete(self, name)
 

lib/splunklib/client.py

@@ -204,19 +204,21 @@ def _execute(self, ifile, process):
 
         """
         if self._protocol_version == 2:
-            result = self._read_chunk(self._as_binary_stream(ifile))
+            self._execute_v2(ifile, self.generate())
+        else:
+            assert self._protocol_version == 1
+            self._record_writer.write_records(self.generate())
+        self.finish()
 
-            if not result:
+    def _execute_chunk_v2(self, process, chunk):
+        count = 0
+        for row in process:
+            self._record_writer.write_record(row)
+            count += 1
+            if count == self._record_writer._maxresultrows:
+                self._finished = False
                 return
-
-            metadata, body = result
-            action = getattr(metadata, 'action', None)
-
-            if action != 'execute':
-                raise RuntimeError('Expected execute action, not {}'.format(action))
-
-        self._record_writer.write_records(self.generate())
-        self.finish()
+        self._finished = True
 
     # endregion
 

lib/splunklib/searchcommands/generating_command.py

@@ -35,6 +35,7 @@
 import os
 import re
 import sys
+import warnings
 
 from . import environment
 
@@ -74,7 +75,7 @@ def set_binary_mode(fh):
 
 
 class CommandLineParser(object):
-    """ Parses the arguments to a search command.
+    r""" Parses the arguments to a search command.
 
     A search command line is described by the following syntax.
 
@@ -505,8 +506,8 @@ def __init__(self, ofile, maxresultrows=None):
 
         self._inspector = OrderedDict()
         self._chunk_count = 0
-        self._record_count = 0
-        self._total_record_count = 0
+        self._pending_record_count = 0
+        self._committed_record_count = 0
 
     @property
     def is_flushed(self):
@@ -524,6 +525,30 @@ def ofile(self):
     def ofile(self, value):
         self._ofile = set_binary_mode(value)
 
+    @property
+    def pending_record_count(self):
+        return self._pending_record_count
+
+    @property
+    def _record_count(self):
+        warnings.warn(
+            "_record_count will be deprecated soon. Use pending_record_count instead.",
+             PendingDeprecationWarning
+        )
+        return self.pending_record_count
+
+    @property
+    def committed_record_count(self):
+        return self._committed_record_count
+
+    @property
+    def _total_record_count(self):
+        warnings.warn(
+            "_total_record_count will be deprecated soon. Use committed_record_count instead.",
+             PendingDeprecationWarning
+        )
+        return self.committed_record_count
+
     def write(self, data):
         bytes_type = bytes if sys.version_info >= (3, 0) else str
         if not isinstance(data, bytes_type):
@@ -555,8 +580,7 @@ def _clear(self):
         self._buffer.seek(0)
         self._buffer.truncate()
         self._inspector.clear()
-        self._record_count = 0
-        self._flushed = False
+        self._pending_record_count = 0
 
     def _ensure_validity(self):
         if self._finished is True:
@@ -651,9 +675,9 @@ def _write_record(self, record):
             values += (repr(value), None)
 
         self._writerow(values)
-        self._record_count += 1
+        self._pending_record_count += 1
 
-        if self._record_count >= self._maxresultrows:
+        if self.pending_record_count >= self._maxresultrows:
             self.flush(partial=True)
 
     try:
@@ -690,7 +714,7 @@ def flush(self, finished=None, partial=None):
 
         RecordWriter.flush(self, finished, partial)  # validates arguments and the state of this instance
 
-        if self._record_count > 0 or (self._chunk_count == 0 and 'messages' in self._inspector):
+        if self.pending_record_count > 0 or (self._chunk_count == 0 and 'messages' in self._inspector):
 
             messages = self._inspector.get('messages')
 
@@ -728,9 +752,9 @@ def flush(self, finished=None, partial=None):
                     print(level, text, file=stderr)
 
             self.write(self._buffer.getvalue())
-            self._clear()
             self._chunk_count += 1
-            self._total_record_count += self._record_count
+            self._committed_record_count += self.pending_record_count
+            self._clear()
 
         self._finished = finished is True
 
@@ -748,37 +772,36 @@ class RecordWriterV2(RecordWriter):
     def flush(self, finished=None, partial=None):
 
         RecordWriter.flush(self, finished, partial)  # validates arguments and the state of this instance
-        inspector = self._inspector
-
-        if self._flushed is False:
-
-            self._total_record_count += self._record_count
-            self._chunk_count += 1
-
-            # TODO: DVPL-6448: splunklib.searchcommands | Add support for partial: true when it is implemented in
-            # ChunkedExternProcessor (See SPL-103525)
-            #
-            # We will need to replace the following block of code with this block:
-            #
-            # metadata = [
-            #     ('inspector', self._inspector if len(self._inspector) else None),
-            #     ('finished', finished),
-            #     ('partial', partial)]
-
-            if len(inspector) == 0:
-                inspector = None
-
-            if partial is True:
-                finished = False
 
-            metadata = [item for item in (('inspector', inspector), ('finished', finished))]
-            self._write_chunk(metadata, self._buffer.getvalue())
-            self._clear()
+        if partial or not finished:
+            # Don't flush partial chunks, since the SCP v2 protocol does not
+            # provide a way to send partial chunks yet.
+            return
 
-        elif finished is True:
-            self._write_chunk((('finished', True),), '')
+        if not self.is_flushed:
+            self.write_chunk(finished=True)
 
-        self._finished = finished is True
+    def write_chunk(self, finished=None):
+        inspector = self._inspector
+        self._committed_record_count += self.pending_record_count
+        self._chunk_count += 1
+
+        # TODO: DVPL-6448: splunklib.searchcommands | Add support for partial: true when it is implemented in
+        # ChunkedExternProcessor (See SPL-103525)
+        #
+        # We will need to replace the following block of code with this block:
+        #
+        # metadata = [item for item in (('inspector', inspector), ('finished', finished), ('partial', partial))]
+        #
+        # if partial is True:
+        #     finished = False
+
+        if len(inspector) == 0:
+            inspector = None
+
+        metadata = [item for item in (('inspector', inspector), ('finished', finished))]
+        self._write_chunk(metadata, self._buffer.getvalue())
+        self._clear()
 
     def write_metadata(self, configuration):
         self._ensure_validity()
@@ -793,7 +816,7 @@ def write_metric(self, name, value):
         self._inspector['metric.' + name] = value
 
     def _clear(self):
-        RecordWriter._clear(self)
+        super(RecordWriterV2, self)._clear()
         self._fieldnames = None
 
     def _write_chunk(self, metadata, body):
@@ -818,4 +841,4 @@ def _write_chunk(self, metadata, body):
         self.write(metadata)
         self.write(body)
         self._ofile.flush()
-        self._flushed = False
+        self._flushed = True