Merge pull request #16 from calesanz/master

Enable SSL verification for inter server communication

Author: gjanders

README/inputs.conf.spec

@@ -9,6 +9,8 @@ gitTempDir = <value>
 * location where to store the output of the script on the filesystem (note this directory will be deleted/re-created but the parent dir must exist)
 gitRepoURL = <value>
 * git repository URL to store the objects (SSH URL only)
+sslVerify = <boolean>
+* Set to 'true' or 'false' to enable/disable SSL verification for REST requests to `srcUrl`. Set to a path to specify a file with valid CA. (https://2.python-requests.org/en/master/user/advanced/#ssl-cert-verification)
 noPrivate = <boolean>
 * disable the backup of user level / private objects (true/false), default false
 noDisabled = <boolean>
@@ -47,6 +49,9 @@ gitTempDir = <value>
 * location where to store the output of the script on the filesystem (note this directory will be deleted/re-created but the parent dir must exist)
 gitRepoURL = <value>
 * git repository URL to store the objects (SSH URL only)
+sslVerify = <boolean>
+* Set to 'true' or 'false' to enable/disable SSL verification for REST requests to `srcUrl`. Set to a path to specify a file with valid CA. (https://2.python-requests.org/en/master/user/advanced/#ssl-cert-verification)
+
 auditLogsLookupBackTime = <value>
 * This is how far back the audit logs will be checked to ensure that a restore entry is valid, this should be set to your interval time or slightly more, defaults to -1h (use Splunk format)
 debugMode = <boolean>

bin/postversioncontrolrestore.py

@@ -10,8 +10,35 @@
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib"))
 
 from splunklib.searchcommands import dispatch, GeneratingCommand, Configuration, Option
+from splunklib.searchcommands.validators import Validator, Boolean
 from splunklib.binding import HTTPError
 
+class OrValidator(Validator):
+    def __init__(self, a, b):
+        self.a = a
+        self.b = b
+    def __call__(self, value):
+        """Returns b if a raises an exception otherwise a."""
+        try:
+            return self.a.__call__(value)
+        except ValueError:
+            return self.b.__call__(value)
+
+    def format(self, value):
+        """Returns b if a raises an exception otherwise a."""
+        try:
+            return self.a.format(value)
+        except:
+            return self.b.format(value)
+
+class Filename(Validator):
+    # TODO Validate file path
+    def __call__(self, value):
+        return value
+
+    def format(self, value):
+        return value
+
 splunkLogsDir = os.environ['SPLUNK_HOME'] + "/var/log/splunk"
 #Setup the logging
 logging_config = dict(
@@ -60,6 +87,8 @@ class SVCPostRestore(GeneratingCommand):
     restoreAsUser = Option(require=True)
     scope = Option(require=True)
     timeout = Option(require=True)
+    sslVerify = Option(require=False, default=False, validate=OrValidator(Boolean(), Filename()))
+    requestingAddress = Option(require=False, default=False)
 
     def generate(self):
         """
@@ -87,14 +116,16 @@ def generate(self):
         body['restoreAsUser'] = self.restoreAsUser
         body['scope'] = self.scope
         body['timeout'] = self.timeout
+        if self.requestingAddress:
+            body['requestingAddress'] = self.requestingAddress
 
         logger.info("Attempting POST request to url=%s with body=\"%s\"" % (url, body))
 
         body['Authorization'] = 'Splunk ' + self._metadata.searchinfo.session_key
 
         logger.debug("Using token %s" % (body['Authorization']))
 
-        attempt = requests.post(url, verify=False, data=body)
+        attempt = requests.post(url, verify=self.sslVerify, data=body)
         if attempt.status_code != 200:
             logger.error("POST request failed with status_code=%s, reason=%s, text=%s on url=%s" % (attempt.status_code, attempt.reason, attempt.text, url))
             yield {'result': 'Unknown failure, received a non-200 response code of %s on the url %s, reason %s, text result is %s' % (attempt.status_code, url, attempt.reason, attempt.text)}

bin/splunkversioncontrol_backup.py

@@ -47,6 +47,12 @@
                 <title>gitRepoURL</title>
                 <description>git repository URL to store the objects (SSH URL only)</description>
             </arg>
+            <arg name="sslVerify">
+                <title>sslVerify</title>
+                <description>Set to 'true' or 'false' to enable/disable SSL verification for REST requests to `srcUrl`. Set to a path to specify a file with valid CA. (https://2.python-requests.org/en/master/user/advanced/#ssl-cert-verification)</description>
+                <validation>is_bool('sslVerify')</validation>
+                <required_on_create>false</required_on_create>
+            </arg>
             <arg name="noPrivate">
                 <title>noPrivate</title>
                 <description>disable the backup of user level / private objects (true/false), default false</description>
@@ -203,10 +209,13 @@ def validate_arguments():
     else:
         ssh_command = "ssh"
 
+    sslVerify = False
+    if 'sslVerify' in val_data:
+        sslVerify = val_data['sslVerify']
+
     #Run a sanity check and make sure we can connect into the remote Splunk instance
     if not useLocalAuth:
         url = val_data['srcURL'] + "/servicesNS/nobody/%s/search/jobs/export?search=makeresults" % (appName)
-        #Verify=false is hardcoded to workaround local SSL issues
         srcUsername = val_data['srcUsername']
         srcPassword = val_data['srcPassword']
         if srcPassword.find("password:") == 0:
@@ -224,7 +233,7 @@ def validate_arguments():
 
         try:
             logger.debug("Running query against URL %s with username %s proxies_length=%s" % (url, srcUsername, len(proxies)))
-            res = requests.get(url, auth=(srcUsername, srcPassword), verify=False, proxies=proxies)
+            res = requests.get(url, auth=(srcUsername, srcPassword), verify=self.sslVerify, proxies=proxies)
             logger.debug("End query against URL %s with username %s" % (url, srcUsername))
 
             if (res.status_code != requests.codes.ok):

bin/splunkversioncontrol_backup_class.py

@@ -77,6 +77,7 @@ class SplunkVersionControlBackup:
     gitRepoURL = None
     stanzaName = None
     lastRunEpoch = None
+    sslVerify = False
 
     # read XML configuration passed from splunkd
     def get_config(self):
@@ -136,8 +137,7 @@ def getAllAppsList(self):
         else:
             auth = HTTPBasicAuth(self.srcUsername, self.srcPassword)
 
-        #Verify=false is hardcoded to workaround local SSL issues
-        res = requests.get(url, auth=auth, headers=headers, verify=False, proxies=self.proxies)
+        res = requests.get(url, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies)
         if (res.status_code != requests.codes.ok):
             logger.fatal("i=\"%s\" Could not obtain a list of all apps, URL=%s statuscode=%s reason=%s, response=\"%s\"" % (self.stanzaName, url, res.status_code, res.reason, res.text))
             sys.exit(-1)
@@ -190,8 +190,7 @@ def runQueries(self, app, endpoint, type, fieldIgnoreList, aliasAttributes={}, v
         else:
             auth = HTTPBasicAuth(self.srcUsername, self.srcPassword)
 
-        #Verify=false is hardcoded to workaround local SSL issues
-        res = requests.get(url, auth=auth, headers=headers, verify=False, proxies=self.proxies)
+        res = requests.get(url, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies)
         if (res.status_code != requests.codes.ok):
             logger.error("i=\"%s\" URL=%s in app=%s statuscode=%s reason=%s response=\"%s\"" % (self.stanzaName, url, app, res.status_code, res.reason, res.text))
 
@@ -433,8 +432,7 @@ def macros(self, app):
         else:
             auth = HTTPBasicAuth(self.srcUsername, self.srcPassword)
 
-        #Verify=false is hardcoded to workaround local SSL issues
-        res = requests.get(url, auth=auth, headers=headers, verify=False, proxies=self.proxies)
+        res = requests.get(url, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies)
         if (res.status_code != requests.codes.ok):
             logger.error("i=\"%s\" Type macro in app=%s, URL=%s statuscode=%s reason=%s, response=\"%s\"" % (self.stanzaName, app, url, res.status_code, res.reason, res.text))
 
@@ -749,7 +747,7 @@ def runSearchJob(self, query):
             headers = {'Authorization': 'Splunk %s' % self.session_key }
         else:
             auth = HTTPBasicAuth(self.srcUsername, self.srcPassword)
-        res = requests.post(url, auth=auth, headers=headers, verify=False, data=data, proxies=self.proxies)
+        res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=data, proxies=self.proxies)
         if (res.status_code != requests.codes.ok):
             logger.error("i=\"%s\" URL=%s statuscode=%s reason=%s response=\"%s\"" % (self.stanzaName, url, res.status_code, res.reason, res.text))
         res = json.loads(res.text)
@@ -1022,6 +1020,9 @@ def run_script(self):
 
         self.proxies = proxies
 
+        if 'sslVerify' in config:
+            self.sslVerify = config['sslVerify']
+
         #From server
         self.splunk_rest = config['srcURL']
         excludedList = [ "srcPassword", "session_key" ]
@@ -1035,7 +1036,6 @@ def run_script(self):
 
         headers={'Authorization': 'Splunk %s' % config['session_key']}
 
-        #Verify=false is hardcoded to workaround local SSL issues
         url = 'https://localhost:8089/services/shcluster/captain/info?output_mode=json'
         res = requests.get(url, headers=headers, verify=False)
         if (res.status_code == 503):