Merge branch 'develop' into feature/typeProperty

Shakeel Mohamed · Shakeel Mohamed · commit 968c31e3b1a2 · 2015-05-27T17:32:38.000-07:00
diff --git a/.gitignore b/.gitignore
@@ -5,6 +5,7 @@
 *coverage_html_report*
 .coverage
 .coverage.*
+.python-version
 __stdout__
 docs/_build
 build/
@@ -18,4 +19,5 @@ examples/*/metadata
 tests/searchcommands_data/log/
 tests/searchcommands_data/output/
 examples/searchcommands_app/searchcommand_app.log
-Test Results*.html
+Test Results*.html
+tests/searchcommands/data/app/app.log
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Splunk SDK for Python Changelog
 
+## Version 1.3.1
+
+### Bug fixes
+
+* Hot fix to `binding.py` to work with Python 2.7.9, which introduced SSL certificate validation by default as outlined in [PEP 476](https://www.python.org/dev/peps/pep-0476).
+* Update `async`, `handler_proxy`, and `handler_urllib2` examples to work with Python 2.7.9 by disabling SSL certificate validation by default.
 
 ## Version 1.3.0
 
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # The Splunk Software Development Kit for Python
 
-#### Version 1.3.0
+#### Version 1.3.1
 
 The Splunk Software Development Kit (SDK) for Python contains library code and 
 examples designed to enable developers to build applications using Splunk.
diff --git a/examples/async/async.py b/examples/async/async.py
@@ -22,6 +22,7 @@
 
 import sys, os, datetime
 import urllib
+import ssl
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
 
 import splunklib.binding as binding
@@ -171,7 +172,12 @@ def request(url, message, **kwargs):
     method = message.get("method", "GET")
     
     # Note that we do not support proxies in this example
-    opener = urllib2.build_opener()
+    # If running Python 2.7.9+, disable SSL certificate validation
+    if sys.version_info >= (2, 7, 9):
+        unverified_ssl_handler = urllib2.HTTPSHandler(context=ssl._create_unverified_context())
+        opener = urllib2.build_opener(unverified_ssl_handler)
+    else:
+        opener = urllib2.build_opener()
 
     # Unfortunately, we need to use the hack of 
     # "overriding" `request.get_method` to specify
diff --git a/examples/github_forks/github_forks.py b/examples/github_forks/github_forks.py
@@ -50,6 +50,7 @@ def get_scheme(self):
         scheme.use_single_instance = True
 
         owner_argument = Argument("owner")
+        owner_argument.title = "Owner"
         owner_argument.data_type = Argument.data_type_string
         owner_argument.description = "Github user or organization that created the repository."
         owner_argument.required_on_create = True
@@ -59,6 +60,7 @@ def get_scheme(self):
         scheme.add_argument(owner_argument)
 
         repo_name_argument = Argument("repo_name")
+        repo_name_argument.title = "Repo Name"
         repo_name_argument.data_type = Argument.data_type_string
         repo_name_argument.description = "Name of the Github repository."
         repo_name_argument.required_on_create = True
diff --git a/examples/handlers/handler_proxy.py b/examples/handlers/handler_proxy.py
@@ -29,6 +29,7 @@
 from pprint import pprint
 from StringIO import StringIO
 import sys, os
+import ssl
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
 import urllib2
 
@@ -52,9 +53,15 @@ def request(url, message, **kwargs):
     method = message['method'].lower()
     data = message.get('body', "") if method == 'post' else None
     headers = dict(message.get('headers', []))
-    context = urllib2.Request(url, data, headers)
+    req = urllib2.Request(url, data, headers)
     try:
-        response = urllib2.urlopen(context)
+        response = urllib2.urlopen(req)
+    except urllib2.URLError, response:
+        # If running Python 2.7.9+, disable SSL certificate validation and try again
+        if sys.version_info >= (2, 7, 9):
+            response = urllib2.urlopen(req, context=ssl._create_unverified_context())
+        else:
+            raise
     except urllib2.HTTPError, response:
         pass # Propagate HTTP errors via the returned response message
     return {
diff --git a/examples/handlers/handler_urllib2.py b/examples/handlers/handler_urllib2.py
@@ -21,6 +21,7 @@
 import sys, os
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
 import urllib2
+import ssl
 
 import splunklib.client as client
 
@@ -34,9 +35,13 @@ def request(url, message, **kwargs):
     method = message['method'].lower()
     data = message.get('body', "") if method == 'post' else None
     headers = dict(message.get('headers', []))
-    context = urllib2.Request(url, data, headers)
+    # If running Python 2.7.9+, disable SSL certificate validation
+    req = urllib2.Request(url, data, headers)
     try:
-        response = urllib2.urlopen(context)
+        if sys.version_info >= (2, 7, 9):
+            response = urllib2.urlopen(req, context=ssl._create_unverified_context())
+        else:
+            response = urllib2.urlopen(req)
     except urllib2.HTTPError, response:
         pass # Propagate HTTP errors via the returned response message
     return {
diff --git a/examples/random_numbers/random_numbers.py b/examples/random_numbers/random_numbers.py
@@ -50,6 +50,7 @@ def get_scheme(self):
         scheme.use_single_instance = True
 
         min_argument = Argument("min")
+        min_argument.title = "Minimum"
         min_argument.data_type = Argument.data_type_number
         min_argument.description = "Minimum random number to be produced by this input."
         min_argument.required_on_create = True
@@ -59,6 +60,7 @@ def get_scheme(self):
         scheme.add_argument(min_argument)
 
         max_argument = Argument("max")
+        max_argument.title = "Maximum"
         max_argument.data_type = Argument.data_type_number
         max_argument.description = "Maximum random number to be produced by this input."
         max_argument.required_on_create = True
diff --git a/setup.py b/setup.py
@@ -34,6 +34,17 @@ def run_test_suite():
     unittest.TextTestRunner().run(suite)
     os.chdir(original_cwd)
 
+def run_test_suite_with_junit_output():
+    try:
+        import unittest2 as unittest
+    except ImportError:
+        import unittest
+    import xmlrunner
+    original_cwd = os.path.abspath(os.getcwd())
+    os.chdir('tests')
+    suite = unittest.defaultTestLoader.discover('.')
+    xmlrunner.XMLTestRunner(output='../test-reports').run(suite)
+    os.chdir(original_cwd)
 
 class CoverageCommand(Command):
     """setup.py command to run code coverage of the test suite."""
@@ -73,6 +84,20 @@ def finalize_options(self):
     def run(self):
         run_test_suite()
 
+class JunitXmlTestCommand(Command):
+    """setup.py command to run the whole test suite."""
+    description = "Run test full test suite with JUnit-formatted output."
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        run_test_suite_with_junit_output()
+
 
 class DistCommand(Command):
     """setup.py command to create .spl files for modular input and search
@@ -175,6 +200,7 @@ def exclude(path):
 
     cmdclass={'coverage': CoverageCommand,
               'test': TestCommand,
+              'testjunit': JunitXmlTestCommand,
               'dist': DistCommand},
 
     description="The Splunk Software Development Kit for Python.",
@@ -202,4 +228,4 @@ def exclude(path):
         "Topic :: Software Development :: Libraries :: Python Modules",
         "Topic :: Software Development :: Libraries :: Application Frameworks",
     ],
-)
+)
diff --git a/splunklib/__init__.py b/splunklib/__init__.py
@@ -14,6 +14,6 @@
 
 """Python library for Splunk."""
 
-__version_info__ = (1, 3, 0)
+__version_info__ = (1, 3, 1)
 __version__ = ".".join(map(str, __version_info__))
 
diff --git a/splunklib/binding.py b/splunklib/binding.py
@@ -30,6 +30,7 @@
 import ssl
 import urllib
 import io
+import sys
 
 from datetime import datetime
 from functools import wraps
@@ -1204,6 +1205,10 @@ def connect(scheme, host, port):
         if scheme == "https":
             if key_file is not None: kwargs['key_file'] = key_file
             if cert_file is not None: kwargs['cert_file'] = cert_file
+
+            # If running Python 2.7.9+, disable SSL certificate validation
+            if sys.version_info >= (2,7,9) and key_file is None and cert_file is None:
+                kwargs['context'] = ssl._create_unverified_context()
             return httplib.HTTPSConnection(host, port, **kwargs)
         raise ValueError("unsupported scheme: %s" % scheme)
 
diff --git a/splunklib/modularinput/argument.py b/splunklib/modularinput/argument.py
@@ -80,6 +80,9 @@ def add_to_document(self, parent):
         arg = ET.SubElement(parent, "arg")
         arg.set("name", self.name)
 
+        if self.title is not None:
+            ET.SubElement(arg, "title").text = self.title
+
         if self.description is not None:
             ET.SubElement(arg, "description").text = self.description
 
diff --git a/splunklib/results.py b/splunklib/results.py
@@ -246,7 +246,25 @@ def _parse_results(self, stream):
                         elem.clear()
 
                 elif elem.tag in ('text', 'v') and event == 'end':
-                    text = "".join(elem.itertext())
+                    try:
+                        text = "".join(elem.itertext())
+                    except AttributeError:
+                        # Assume we're running in Python < 2.7, before itertext() was added
+                        # So we'll define it here
+
+                        def __itertext(self):
+                          tag = self.tag
+                          if not isinstance(tag, basestring) and tag is not None:
+                              return
+                          if self.text:
+                              yield self.text
+                          for e in self:
+                              for s in __itertext(e):
+                                  yield s
+                              if e.tail:
+                                  yield e.tail
+
+                        text = "".join(__itertext(elem))
                     values.append(text.encode('utf8'))
                     elem.clear()
 
diff --git a/splunklib/searchcommands/splunk_csv/dict_reader.py b/splunklib/searchcommands/splunk_csv/dict_reader.py
@@ -31,6 +31,7 @@ def fieldnames(self):
             try:
                 self._fieldnames = self.reader.next()
             except StopIteration:
+                self._fieldnames = []
                 pass
             self.line_num = self.reader.line_num
             self.__mv_fieldnames = []
diff --git a/tests/modularinput/data/scheme_without_defaults_and_argument_title.xml b/tests/modularinput/data/scheme_without_defaults_and_argument_title.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<scheme>
+    <title>abcd</title>
+    <description>쎼 and 쎶 and &lt;&amp;&gt; für</description>
+    <use_external_validation>false</use_external_validation>
+    <use_single_instance>true</use_single_instance>
+    <streaming_mode>simple</streaming_mode>
+    <endpoint>
+        <args>
+            <arg name="arg1">
+                <data_type>string</data_type>
+                <required_on_edit>false</required_on_edit>
+                <required_on_create>false</required_on_create>
+            </arg>
+            <arg name="arg2">
+                <title>Argument for ``test_scheme``</title>
+                <description>쎼 and 쎶 and &lt;&amp;&gt; für</description>
+                <validation>is_pos_int('some_name')</validation>
+                <data_type>number</data_type>
+                <required_on_edit>true</required_on_edit>
+                <required_on_create>true</required_on_create>
+            </arg>
+        </args>
+    </endpoint>
+</scheme>
diff --git a/tests/modularinput/test_scheme.py b/tests/modularinput/test_scheme.py
@@ -47,6 +47,35 @@ def test_generate_xml_from_scheme(self):
         arg1 = Argument(name="arg1")
         scheme.add_argument(arg1)
 
+        arg2 = Argument(
+            name="arg2",
+            description=u"쎼 and 쎶 and <&> für",
+            validation="is_pos_int('some_name')",
+            data_type=Argument.data_type_number,
+            required_on_edit=True,
+            required_on_create=True
+        )
+        scheme.add_argument(arg2)
+
+        constructed = scheme.to_xml()
+
+        expected = ET.parse(data_open("data/scheme_without_defaults.xml")).getroot()
+
+        self.assertTrue(xml_compare(expected, constructed))
+
+    def test_generate_xml_from_scheme_with_arg_title(self):
+        """Checks that the XML generated by a Scheme object with all its fields set and
+        some arguments added matches what we expect. Also sets the title on an argument."""
+
+        scheme = Scheme("abcd")
+        scheme.description = u"쎼 and 쎶 and <&> für"
+        scheme.streaming_mode = Scheme.streaming_mode_simple
+        scheme.use_external_validation = "false"
+        scheme.use_single_instance = "true"
+
+        arg1 = Argument(name="arg1")
+        scheme.add_argument(arg1)
+
         arg2 = Argument(
             name="arg2",
             description=u"쎼 and 쎶 and <&> für",
@@ -60,7 +89,7 @@ def test_generate_xml_from_scheme(self):
 
         constructed = scheme.to_xml()
 
-        expected = ET.parse(data_open("data/scheme_without_defaults.xml")).getroot()
+        expected = ET.parse(data_open("data/scheme_without_defaults_and_argument_title.xml")).getroot()
         self.assertEqual("Argument for ``test_scheme``", arg2.title)
 
         self.assertTrue(xml_compare(expected, constructed))
diff --git a/tests/test_binding.py b/tests/test_binding.py
@@ -24,6 +24,8 @@
 import testlib
 import unittest
 import socket
+import sys
+import ssl
 
 import splunklib.binding as binding
 from splunklib.binding import HTTPError, AuthenticationError, UrlEncoded
@@ -435,9 +437,13 @@ def urllib2_handler(url, message, **kwargs):
     method = message['method'].lower()
     data = message.get('body', "") if method == 'post' else None
     headers = dict(message.get('headers', []))
-    context = urllib2.Request(url, data, headers)
+    req = urllib2.Request(url, data, headers)
     try:
-        response = urllib2.urlopen(context)
+        # If running Python 2.7.9+, disable SSL certificate validation
+        if sys.version_info >= (2, 7, 9):
+            response = urllib2.urlopen(req, context=ssl._create_unverified_context())
+        else:
+            response = urllib2.urlopen(req)
     except urllib2.HTTPError, response:
         pass # Propagate HTTP errors via the returned response message
     return {
diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -1,13 +1,4 @@
-import os
-from subprocess import PIPE, Popen
-from os.path import expanduser
-import time
-import sys
-
-import testlib 
-
-import splunklib.client as client
-from splunklib.binding import UrlEncoded
+import testlib
 
 try:
     from utils import *
