Merge branch 'develop' of github.com:splunk/splunk-sdk-python into develop

Author: glennblock

CHANGELOG.md

@@ -15,15 +15,27 @@
 
   2. Logs a traceback to SearchCommand.logger. This is old behavior.
 
-### Bug fixes
+* Made ResponseReader more streamlike, so that it can be wrapped in an 
+  io.BufferedReader to realize a significant performance gain.
+
+  *Example usage*
 
-1. Addressed a problem in the results reader running under Python 2.6.
+  ```
+  import io
+  ...
+  response = job.results(count=maxRecords, offset=self._offset)
+  resultsList = results.ResultsReader(io.BufferedReader(response)) 
+  ```
 
-   The results reader now catches SyntaxError exceptions instead of
+### Bug fixes
+
+1. The results reader now catches SyntaxError exceptions instead of
    `xml.etree.ElementTree.ParseError` exceptions. `ParseError` wasn't
-   introduced until Python 2.7. This masked the root cause of certain issues
-   with result elements.
+   introduced until Python 2.7. This masked the root cause of errors
+   data errors in result elements.
 
+2. When writing a ReportingCommand you no longer need to include a map method.
+ 
 ## Version 1.2.2
 
 ### Bug fixes

Commands.conf.spec.xlsx

@@ -2718,10 +2718,11 @@ def export(self, query, **params):
                     # Normal events are returned as dicts
                     print result
             assert rr.is_preview == False
-        
-        Running an export search is more efficient than running a similar
-        preview search because no post-processing is done on the retrieved
-        events. The raw events are simply returned.
+
+        Running an export search is more efficient as it streams the results
+        directly to you, rather than having to write them out to disk and make
+        them available later. As soon as results are ready, you will receive
+        them.
 
         The ``export`` method makes a single roundtrip to the server (as opposed
         to two for :meth:`create` followed by :meth:`preview`), plus at most two

splunklib/client.py

@@ -13,6 +13,8 @@
 # under the License.
 
 from __future__ import absolute_import
+
+from numbers import Number
 import csv
 
 
@@ -59,13 +61,24 @@ def _encode_list(self, value):
         if len(value) == 1:
             return value[0], None
         multi_value = ';'.join(
-            ['$' + repr(item).replace('$', '$$') + '$' for item in value])
+            ['$' + DictWriter._to_string(item).replace('$', '$$') + '$' for item
+             in value])
         value = self._mv_delimiter.join([repr(item) for item in value])
         return value, multi_value
 
     def _header_written(self):
         return self._fieldnames is not None
 
+    @staticmethod
+    def _to_string(item):
+        if isinstance(item, bool):
+            return 't' if item else 'f'
+        if isinstance(item, str):
+            return item
+        if isinstance(item, Number):
+            return str(item)
+        return repr(item)
+
     def _writeheader(self, record):
         if self.fieldnames is None:
             self.fieldnames = sorted(record.keys())

splunklib/searchcommands/csv/dict_writer.py

@@ -162,10 +162,16 @@ def streaming_preop(self):
             Computed.
 
             """
+            command = type(self.command)
+
+            if command.map == ReportingCommand.map:
+                return ""
+
             command_line = str(self.command)
             command_name = type(self.command).name
             text = ' '.join([
                 command_name, '__map__', command_line[len(command_name) + 1:]])
+
             return text
 
         #endregion
@@ -196,15 +202,15 @@ def fix_up(cls, command):
             if command.reduce == ReportingCommand.reduce:
                 raise AttributeError('No ReportingCommand.reduce override')
 
+            if command.map == ReportingCommand.map:
+                cls._requires_preop = False
+                return
+
             f = vars(command)['map']   # Function backing the map method
                 # There is no way to add custom attributes to methods. See
                 # [Why does setattr fail on a method](http://goo.gl/aiOsqh)
                 # for an explanation.
 
-            if f == vars(ReportingCommand)['map']:
-                cls._requires_preop = False
-                return
-
             try:
                 settings = f._settings
             except AttributeError:

splunklib/searchcommands/reporting_command.py

@@ -183,4 +183,7 @@ def test_builtin_options(self):
 
         return
 
-    _package_directory = os.path.dirname(__file__)
+    _package_directory = os.path.dirname(__file__)
+
+if __name__ == "__main__":
+    unittest.main()

tests/searchcommands/test_decorators.py

@@ -54,12 +54,18 @@ def test_process(self):
         # We support dynamic configuration, not static
 
         expected = \
-            'error_message=Command search appears to be statically configured and static configuration is unsupported by splunklib.searchcommands. Please ensure that default/commands.conf contains this stanza: [search] | filename = foo.py | supports_getinfo = true | supports_rawargs = true | outputheader = true\r\n' \
+            '\r\n' \
+            'ERROR,__mv_ERROR' \
+            '\r\n' \
+            'Command search appears to be statically configured and static configuration is unsupported by splunklib.searchcommands. Please ensure that default/commands.conf contains this stanza: [search] | filename = foo.py | supports_getinfo = true | supports_rawargs = true | outputheader = true,' \
             '\r\n'
 
         command = SearchCommand()
         result = StringIO()
-        command.process(['foo.py'], output_file=result)
+
+        self.assertRaises(
+            SystemExit, command.process, ['foo.py'], output_file=result)
+
         result.reset()
         observed = result.read()
         self.assertEqual(expected, observed)
@@ -88,7 +94,8 @@ def test_process(self):
 
         command = SearchCommand()
         result = StringIO()
-        command.process(['foo.py', '__GETINFO__', 'undefined_option=value'], output_file=result)
+
+        self.assertRaises(SystemExit, command.process, ['foo.py', '__GETINFO__', 'undefined_option=value'], output_file=result)
         result.reset()
         observed = result.read()
         self.assertEqual(expected, observed)
@@ -97,8 +104,9 @@ def test_process(self):
         # errors, if invoked to execute
 
         expected = \
-            'error_message=Unrecognized option: undefined_option = value\r\n' \
-            '\r\n'
+            '\r\n' \
+            'ERROR,__mv_ERROR\r\n' \
+            'Unrecognized option: undefined_option = value,\r\n'
 
         command = SearchCommand()
         result = StringIO()
@@ -118,7 +126,9 @@ def test_process(self):
         # Command.process should exit on processing exceptions
 
         expected = \
-            ''
+            '\r\n' \
+            'ERROR,__mv_ERROR\r\n' \
+            '\'NoneType\' object is not iterable,\r\n'
 
         command = SearchCommand()
         result = StringIO()
@@ -175,4 +185,7 @@ def test_process(self):
 
         return
 
-    _package_directory = os.path.dirname(__file__)
+    _package_directory = os.path.dirname(__file__)
+
+if __name__ == "__main__":
+    unittest.main()

tests/searchcommands/test_examples.py

@@ -332,4 +332,7 @@ def test_messages_header(self):
 
         return
 
-    _package_path = os.path.dirname(__file__)
+    _package_path = os.path.dirname(__file__)
+
+if __name__ == "__main__":
+    unittest.main()

tests/searchcommands/test_search_command.py

@@ -96,3 +96,6 @@ def test_file(self):
             self.assertRaises(ValueError, validator, path)
 
         return
+
+if __name__ == "__main__":
+    unittest.main()