Fixed '\r\n' problem correctly by setting the mode of sys.stdout.fileno to binary on 'win32'

Reenabled generating command test on/off server, but disabled content comparisons until we make time to create a more sensible generation command for testing.

Author: David Noble

splunklib/searchcommands/__init__.py

@@ -150,6 +150,14 @@ class sets this value unconditionally. You cannot override it.
 from .reporting_command import ReportingCommand
 from .streaming_command import StreamingCommand
 
+if sys.platform == 'win32':
+    # Work around the fact that on Windows '\n' is mapped to '\r\n'
+    # The typical solution is to simply open files in binary mode, but stdout
+    # is already open, thus this hack
+    import msvcrt
+    import os
+    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+
 
 def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=
              sys.stdout, module_name=None):

splunklib/searchcommands/search_command_internals.py

@@ -13,7 +13,6 @@
 # under the License.
 
 import collections
-import os
 import re
 import urllib2 as urllib
 
@@ -170,9 +169,7 @@ def write(self, output_file):
         # for level, message in self:
         #     output_file.write('%s=%s\r\n' % (level, message))
         self
-        output_file.write(MessagesHeader.line_ending)
-
-    line_ending = '\n' if os.name == 'nt' else '\r\n'
+        output_file.write('\r\n')
 
 
 class SearchCommandParser(object):

tests/test_searchcommands_app.py

@@ -285,6 +285,10 @@ def disable_test_option_show_configuration(self):
                 os.path.join('log', 'test_option_show_configuration.log')))
         return
 
+    # TODO, use a generating command that doesn't do random sampling because
+    # a seed is no guarantee that the same sample is produced on every platform
+    # and all versions of python
+
     def test_generating_command_configuration(self):
         self._assertCorrectConfiguration(
             StubbedGeneratingCommand(), 'test_generating_command_configuration')
@@ -306,14 +310,14 @@ def test_generating_command_in_isolation(self):
                 os.path.join('output', 'test_generating_command_in_isolation.execute.csv'),
                 os.path.join('log', 'test_generating_command_in_isolation.log')))
         self._assertCorrectOutputFile('test_generating_command_in_isolation.getinfo.csv')
-        self._assertCorrectOutputFile('test_generating_command_in_isolation.execute.csv')
+        # self._assertCorrectOutputFile('test_generating_command_in_isolation.execute.csv')
         return
 
-    def disabled_test_generating_command_on_server(self):
-        # TODO, use a generating command that doesn't do random sampling
-        self._assertCorrectOneshotResults(
+    def test_generating_command_on_server(self):
+        expected, actual = self._getOneshotResults(
             '| simulate csv=population.csv rate=200 interval=00:00:01 duration=00:00:02 seed=%s' % TestSearchCommandsApp._seed,
             'test_generating_command_on_server')
+        # self.assertMultilLineEqual(expected, actual)
         return
 
     def test_reporting_command_configuration(self):
@@ -351,9 +355,10 @@ def test_reporting_command_in_isolation(self):
         return
 
     def test_reporting_command_on_server(self):
-        self._assertCorrectOneshotResults(
+        expected, actual = self._getOneshotResults(
             '| inputcsv tweets_with_word_counts.csv | sum total=total word_count',
             'test_reporting_command_on_server')
+        self.assertMultiLineEqual(expected, actual)
         return
 
     def test_streaming_command_configuration(self):
@@ -379,9 +384,10 @@ def test_streaming_command_in_isolation(self):
         return
 
     def test_streaming_command_on_server(self):
-        self._assertCorrectOneshotResults(
+        expected, actual = self._getOneshotResults(
             '| inputcsv tweets.csv | countmatches fieldname=word_count pattern="\\\\w+" text',
             'test_streaming_command_on_server')
+        self.assertMultiLineEqual(expected, actual)
         return
 
     def _assertCorrectConfiguration(self, command, test_name):
@@ -398,7 +404,16 @@ def _assertCorrectConfiguration(self, command, test_name):
             expected = ''.join(input_file.readlines())
         self.assertMultiLineEqual(expected, actual)
 
-    def _assertCorrectOneshotResults(self, query, test_name):
+    def _assertCorrectOutputFile(self, name):
+        expected = os.path.join('_expected_results', name)
+        actual = os.path.join('output', name)
+        with TestSearchCommandsApp._open_data_file(expected, 'r') as expected:
+            with TestSearchCommandsApp._open_data_file(actual, 'r') as actual:
+                for actual_line, expected_line in zip(actual, expected):
+                    self.assertTrue(actual_line == expected_line)
+        return
+
+    def _getOneshotResults(self, query, test_name):
         response = self.service.jobs.oneshot(query, app="searchcommands_app")
         reader = ResultsReader(response)
         actual = []
@@ -409,18 +424,9 @@ def _assertCorrectOneshotResults(self, query, test_name):
                 actual += ['Message: %s' % result]
         actual = actual + ['is_preview = %s' % reader.is_preview]
         actual = '\n'.join(actual)
-        with TestSearchCommandsApp._open_data_file('_expected_results/%s.txt' % test_name, 'r') as expected:
-            self.assertMultiLineEqual(''.join(expected.readlines()), ''.join(actual))
-        return
-
-    def _assertCorrectOutputFile(self, name):
-        expected = os.path.join('_expected_results', name)
-        actual = os.path.join('output', name)
-        with TestSearchCommandsApp._open_data_file(expected, 'r') as expected:
-            with TestSearchCommandsApp._open_data_file(actual, 'r') as actual:
-                for actual_line, expected_line in zip(actual, expected):
-                    self.assertTrue(actual_line == expected_line)
-        return
+        with TestSearchCommandsApp._open_data_file('_expected_results/%s.txt' % test_name, 'r') as expected_file:
+            expected = ''.join(expected_file.readlines())
+        return actual, expected
 
     def _run(self, command, args, **kwargs):
         for operation in ['__GETINFO__', '__EXECUTE__']: