New option disable_git_ssl_verify

gjanders · gjanders · commit 6aabb8bbf969 · 2021-07-27T12:57:37.000+10:00
Support for password: syntax for the gitRepoURL parameter when using http/https

Bugfix for proxy code to work with git &amp; HTTP proxies
diff --git a/README.md b/README.md
@@ -288,7 +288,14 @@ To do this you will need to install Version Control For SplunkCloud on your Splu
 
 [SplunkVersionControlCloud github](https://github.com/gjanders/SplunkVersionControlCloud)
 
-## Release Notes 
+## Release Notes
+### 1.2.3
+New option `disable_git_ssl_verify`
+
+Support for password: syntax for the gitRepoURL parameter when using http/https
+
+Bugfix for proxy code to work with git & HTTP proxies
+ 
 ### 1.2.2
 This version includes a few changes, these include two new parameters on the version control backup:
 `run_ko_query` - if enabled this runs a Splunk savedsearch and adds the additional information of tag=`git_tag_name` into the output of the modular input which is then indexed
diff --git a/README/inputs.conf.spec b/README/inputs.conf.spec
@@ -8,7 +8,7 @@ srcPassword = <value>
 gitTempDir = <value>
 * location where to store the output of the script on the filesystem (note this directory will be deleted/re-created but the parent dir must exist)
 gitRepoURL = <value>
-* git repository URL to store the objects
+* git repository URL to store the objects. password:passwordinpasswordsconf can be used for token/password substitution if required for http/https URL's
 sslVerify = <boolean>
 * Set to 'true' or 'false' to enable/disable SSL verification for REST requests to `srcUrl`. Set to a path to specify a file with valid CA. (https://2.python-requests.org/en/master/user/advanced/#ssl-cert-verification)
 noPrivate = <boolean>
@@ -51,6 +51,8 @@ run_ko_query = <boolean>
 * Do you want to run a Splunk query to determine which knowledge objects changed? macro 'splunk_vc_ko_query' (defaults to false)
 run_ko_diff = <boolean>
 * Should output of the modular input include diff information (requires run_ko_query to be true, defaults to false)
+disable_git_ssl_verify = <boolean>
+* Use GIT_SSL_NO_VERIFY=true on all git commands
 
 [splunkversioncontrol_restore://<name>]
 destURL = <value>
@@ -62,7 +64,7 @@ destPassword = <value>
 gitTempDir = <value>
 * location where to store the output of the script on the filesystem (note this directory will be deleted/re-created but the parent dir must exist)
 gitRepoURL = <value>
-* git repository URL to restore the objects from
+* git repository URL to restore the objects from. password:passwordinpasswordsconf can be used for token/password substitution if required for http/https URL's
 sslVerify = <boolean>
 * Set to 'true' or 'false' to enable/disable SSL verification for REST requests to `srcUrl`. Set to a path to specify a file with valid CA. (https://2.python-requests.org/en/master/user/advanced/#ssl-cert-verification)
 auditLogsLookupBackTime = <value>
@@ -87,3 +89,6 @@ git_proxy  = <value>
 * If supplied provides a proxy setting to use to access the git repository (https proxy). Use https://user:password:passwordinpasswordsconf@10.10.1.0:3128 and the application will obtain the password for the entry 'passwordinpasswordsconf'. If password: is not used the password is used as per a normal proxy setting, for example https://user:password@10.10.1.0:3128</description>
 file_per_ko = <boolean>
 * Do you want one file per knowledge object? Or a combined file? Defaults to false (i.e. 1 large file for global dashboards in an app). Note that if you change this you will need to re-create or wipe the repository as the files are stored differently...Note this setting should match in both backup and restore modular inputs for a particular repo
+disable_git_ssl_verify = <boolean>
+* Use GIT_SSL_NO_VERIFY=true on all git commands
+
diff --git a/bin/splunkversioncontrol_backup.py b/bin/splunkversioncontrol_backup.py
@@ -46,7 +46,7 @@
             </arg>
             <arg name="gitRepoURL">
                 <title>gitRepoURL</title>
-                <description>git repository URL to store the objects</description>
+                <description>git repository URL to store the objects. password:passwordinpasswordsconf can be used for token/password substitution if required for http/https URL's</description>
             </arg>
             <arg name="sslVerify">
                 <title>sslVerify</title>
@@ -167,6 +167,13 @@
                 <data_type>boolean</data_type>
                 <validation>is_bool('run_ko_diff')</validation>
             </arg>
+            <arg name="disable_git_ssl_verify">
+                <title>disable_git_ssl_verify</title>
+                <description>Use GIT_SSL_NO_VERIFY=true on all git commands</description>
+                <required_on_create>false</required_on_create>
+                <data_type>boolean</data_type>
+                <validation>is_bool('disable_git_ssl_verify')</validation>
+            </arg>
         </args>
     </endpoint>
 </scheme>
@@ -256,6 +263,17 @@ def validate_arguments():
     else:
         ssh_command = "ssh"
 
+    disable_git_ssl_verify = False
+    if 'disable_git_ssl_verify' in val_data:
+        if val_data['disable_git_ssl_verify'].lower() == 'true' or val_data['disable_git_ssl_verify'] == "1":
+            git_command = "GIT_SSL_NO_VERIFY=true " + git_command
+            logger.debug('git_command now has GIT_SSL_NO_VERIFY=true because disable_git_ssl_verify: ' + val_data['disable_git_ssl_verify'])
+            disable_git_ssl_verify = True
+        elif val_data['disable_git_ssl_verify'].lower() == 'false' or val_data['disable_git_ssl_verify'] == "0":
+            logger.debug('disable_git_ssl_verify set to boolean False from: ' + val_data['disable_git_ssl_verify'])
+        else:
+            logger.warn('disable_git_ssl_verify not set to a valid value, ignoring the setting, please update the setting from: ' + val_data['disable_git_ssl_verify'])
+
     sslVerify = False
     if 'sslVerify' in val_data:
         if val_data['sslVerify'].lower() == 'true' or val_data['sslVerify'] == "1":
@@ -302,6 +320,12 @@ def validate_arguments():
     proxy_command = ""
     if gitRepoURL.find("http") == 0:
         gitRepoHTTP = True
+        if gitRepoURL.find("password:") != -1:
+            start = gitRepoURL.find("password:") + 9
+            end = gitRepoURL.find("@")
+            logger.debug("Attempting to replace gitRepoURL=%s by subsituting=%s with a password" % (gitRepoURL, gitRepoURL[start:end]))
+            temp_password = get_password(gitRepoURL[start:end], session_key, logger)
+            gitRepoURL = gitRepoURL[0:start-9] + temp_password + gitRepoURL[end:]
     else:
         gitRepoHTTP = False
 
@@ -335,7 +359,6 @@ def validate_arguments():
         logger.error("Failed to validate the git repo URL, stdout of '%s', stderr of '%s'" % (stdout, stderr))
         sys.exit(6)
 
-
 #Print the scheme
 def do_scheme():
     print(SCHEME)    
diff --git a/bin/splunkversioncontrol_backup_class.py b/bin/splunkversioncontrol_backup_class.py
@@ -1322,11 +1322,23 @@ def run_script(self):
 
         self.gitRepoURL = config['gitRepoURL']
 
+        self.session_key = config['session_key']
+
         # a flag for a http/https vs SSH based git repo
         if self.gitRepoURL.find("http") == 0:
             self.gitRepoHTTP = True
+            if self.gitRepoURL.find("password:") != -1:
+                self.gitRepoURL_logsafe = self.gitRepoURL
+                start = self.gitRepoURL.find("password:") + 9
+                end = self.gitRepoURL.find("@")
+                logger.debug("Attempting to replace self.gitRepoURL=%s by subsituting=%s with a password" % (self.gitRepoURL, self.gitRepoURL[start:end]))
+                temp_password = get_password(self.gitRepoURL[start:end], self.session_key, logger)
+                self.gitRepoURL = self.gitRepoURL[0:start-9] + temp_password + self.gitRepoURL[end:]
+            else:
+                self.gitRepoURL_logsafe = self.gitRepoURL
         else:
             self.gitRepoHTTP = False
+            self.gitRepoURL_logsafe = self.gitRepoURL
 
         if 'git_command' in config:
             self.git_command = config['git_command'].strip()
@@ -1335,6 +1347,15 @@ def run_script(self):
         else:
             self.git_command = "git"
 
+        if 'disable_git_ssl_verify' in config:
+            if config['disable_git_ssl_verify'].lower() == 'true' or config['disable_git_ssl_verify'].lower() == 't' or config['disable_git_ssl_verify'] == "1":
+                self.git_command = "GIT_SSL_NO_VERIFY=true " + self.git_command
+                logger.debug('git_command now has GIT_SSL_NO_VERIFY=true because disable_git_ssl_verify: ' + config['disable_git_ssl_verify'])
+            elif config['disable_git_ssl_verify'].lower() == 'false' or config['disable_git_ssl_verify'] == "0":
+                logger.debug('disable_git_ssl_verify set to boolean False from: ' + config['disable_git_ssl_verify'])
+            else:
+                logger.warn('disable_git_ssl_verify not set to a valid value, ignoring the setting, please update the setting from: ' + config['disable_git_ssl_verify'])
+
         if 'ssh_command' in config:
             self.ssh_command = config['ssh_command'].strip()
             self.ssh_command = self.ssh_command.replace("\\","/")
@@ -1360,7 +1381,7 @@ def run_script(self):
                 start = proxies['https'].find("password:") + 9
                 end = proxies['https'].find("@")
                 logger.debug("Attempting to replace proxy=%s by subsituting=%s with a password" % (proxies['https'], proxies['https'][start:end]))
-                temp_password = get_password(proxies['https'][start:end], session_key, logger)
+                temp_password = get_password(proxies['https'][start:end], self.session_key, logger)
                 proxies['https'] = proxies['https'][0:start-9] + temp_password + proxies['https'][end:]
 
         self.proxies = proxies
@@ -1372,7 +1393,7 @@ def run_script(self):
                 start = git_proxies['https'].find("password:") + 9
                 end = git_proxies['https'].find("@")
                 logger.debug("Attempting to replace git_proxy=%s by subsituting=%s with a password" % (git_proxies['https'], git_proxies['https'][start:end]))
-                temp_password = get_password(git_proxies['https'][start:end], session_key, logger)
+                temp_password = get_password(git_proxies['https'][start:end], self.session_key, logger)
                 git_proxies['https'] = git_proxies['https'][0:start-9] + temp_password + git_proxies['https'][end:]
 
         self.git_proxies = git_proxies
@@ -1407,7 +1428,6 @@ def run_script(self):
         #Use current epoch to output a checkpoint file at the end
         #If we have not run before just backup everything
         currentEpochTime = calendar.timegm(time.gmtime())
-        self.session_key = config['session_key']
 
         headers={'Authorization': 'Splunk %s' % config['session_key']}
 
@@ -1461,10 +1481,10 @@ def run_script(self):
 
             (output, stderrout, res) = self.clone_git_dir(config)
             if res == False:
-                logger.fatal("i=\"%s\" git clone failed for some reason...on url %s stdout of '%s' with stderrout of '%s'" % (self.stanzaName, self.gitRepoURL, output, stderrout))
+                logger.fatal("i=\"%s\" git clone failed for some reason...on url %s stdout of '%s' with stderrout of '%s'" % (self.stanzaName, self.gitRepoURL_logsafe, output, stderrout))
                 sys.exit(1)
             else:
-                logger.info("i=\"%s\" Successfully cloned the git URL from %s into directory %s" % (self.stanzaName, self.gitRepoURL, self.gitTempDir))
+                logger.info("i=\"%s\" Successfully cloned the git URL from %s into directory %s" % (self.stanzaName, self.gitRepoURL_logsafe, self.gitTempDir))
                 if not ".git" in os.listdir(self.gitTempDir):
                     #include the subdirectory which is the git repo
                     self.gitTempDir = self.gitTempDir + "/" + os.listdir(self.gitTempDir)[0]
@@ -1542,10 +1562,10 @@ def run_script(self):
 
             (output, stderrout, res) = self.clone_git_dir(config)
             if res == False:
-                logger.fatal("i=\"%s\" git clone failed for some reason...on url %s stdout of '%s' with stderrout of '%s'" % (self.stanzaName, self.gitRepoURL, output, stderrout))
+                logger.fatal("i=\"%s\" git clone failed for some reason...on url %s stdout of '%s' with stderrout of '%s'" % (self.stanzaName, self.gitRepoURL_logsafe, output, stderrout))
                 sys.exit(1)
             else:
-                logger.info("i=\"%s\" Successfully cloned the git URL from %s into directory %s" % (self.stanzaName, self.gitRepoURL, self.gitTempDir))
+                logger.info("i=\"%s\" Successfully cloned the git URL from %s into directory %s" % (self.stanzaName, self.gitRepoURL_logsafe, self.gitTempDir))
 
             (output2, stderrout2, res) = self.set_git_details(config)
             if res == False:
@@ -1681,10 +1701,10 @@ def run_script(self):
 
             (output, stderrout, res) = self.clone_git_dir(config)
             if res == False:
-                logger.fatal("i=\"%s\" git clone failed for some reason...on url %s stdout of '%s' with stderrout of '%s'" % (self.stanzaName, self.gitRepoURL, output, stderrout))
+                logger.fatal("i=\"%s\" git clone failed for some reason...on url %s stdout of '%s' with stderrout of '%s'" % (self.stanzaName, self.gitRepoURL_logsafe, output, stderrout))
                 sys.exit(1)
             else:
-                logger.info("i=\"%s\" Successfully cloned the git URL from %s into directory %s" % (self.stanzaName, self.gitRepoURL, self.gitTempDir))
+                logger.info("i=\"%s\" Successfully cloned the git URL from %s into directory %s" % (self.stanzaName, self.gitRepoURL_logsafe, self.gitTempDir))
 
             (output2, stderrout2, res) = self.set_git_details(config)
             if res == False:
diff --git a/bin/splunkversioncontrol_rest_restore.py b/bin/splunkversioncontrol_rest_restore.py
@@ -120,6 +120,8 @@ def handle_POST(self):
             useLocalAuth = json_dict['useLocalAuth']
             if isinstance(useLocalAuth, bool) and useLocalAuth:
                 useLocalAuth = True
+            elif isinstance(useLocalAuth, bool) and not useLocalAuth:
+                pass
             elif useLocalAuth.lower() == 't' or useLocalAuth.lower() == "true":
                 useLocalAuth = True
         if not useLocalAuth:
@@ -141,7 +143,16 @@ def handle_POST(self):
 
         sslVerify = False
         if 'sslVerify' in json_dict:
-            sslVerify = json_dict['sslVerify']
+            sslVerifyValue = json_dict['sslVerify']
+            if sslVerifyValue.lower() == 'true' or sslVerifyValue == "1":
+                sslVerify = True
+                logger.debug('sslverify set to boolean True from: ' + sslVerifyValue)
+            elif sslVerifyValue.lower() == 'false' or sslVerifyValue == "0":
+                sslVerify = False
+                logger.debug('sslverify set to boolean False from: ' + sslVerifyValue)
+            else:
+                sslVerify = sslVerifyValue
+                logger.debug('sslverify set to: %s' % (sslVerifyValue))
 
         headers = {}
         auth = None
diff --git a/bin/splunkversioncontrol_restore.py b/bin/splunkversioncontrol_restore.py
@@ -48,7 +48,7 @@
             </arg>
             <arg name="gitRepoURL">
                 <title>gitRepoURL</title>
-                <description>git repository URL to store the objects</description>
+                <description>git repository URL to store the objects. password:passwordinpasswordsconf can be used for token/password substitution if required for http/https URL's</description>
             </arg>
             <arg name="sslVerify">
                 <title>sslVerify</title>
@@ -118,6 +118,13 @@
                 <data_type>boolean</data_type>
                 <validation>is_bool('file_per_ko')</validation>
             </arg>
+            <arg name="disable_git_ssl_verify">
+                <title>disable_git_ssl_verify</title>
+                <description>Use GIT_SSL_NO_VERIFY=true on all git commands</description>
+                <required_on_create>false</required_on_create>
+                <data_type>boolean</data_type>
+                <validation>is_bool('disable_git_ssl_verify')</validation>
+            </arg>
         </args>
     </endpoint>
 </scheme>
@@ -262,6 +269,17 @@ def validate_arguments():
     else:
         ssh_command = "ssh"
 
+    disable_git_ssl_verify = False
+    if 'disable_git_ssl_verify' in val_data:
+        if val_data['disable_git_ssl_verify'].lower() == 'true' or val_data['disable_git_ssl_verify'] == "1":
+            git_command = "GIT_SSL_NO_VERIFY=true " + git_command
+            logger.debug('git_command now has GIT_SSL_NO_VERIFY=true because disable_git_ssl_verify: ' + val_data['disable_git_ssl_verify'])
+            disable_git_ssl_verify = True
+        elif val_data['disable_git_ssl_verify'].lower() == 'false' or val_data['disable_git_ssl_verify'] == "0":
+            logger.debug('disable_git_ssl_verify set to boolean False from: ' + val_data['disable_git_ssl_verify'])
+        else:
+            logger.warn('disable_git_ssl_verify not set to a valid value, ignoring the setting, please update the setting from: ' + val_data['disable_git_ssl_verify'])
+
     git_proxies = {}
     if 'git_proxy' in val_data:
         git_proxies["https"] = val_data['git_proxy']
@@ -274,6 +292,12 @@ def validate_arguments():
 
     if gitRepoURL.find("http") == 0:
         gitRepoHTTP = True
+        if gitRepoURL.find("password:") != -1:
+            start = gitRepoURL.find("password:") + 9
+            end = gitRepoURL.find("@")
+            logger.debug("Attempting to replace gitRepoURL=%s by subsituting=%s with a password" % (gitRepoURL, gitRepoURL[start:end]))
+            temp_password = get_password(gitRepoURL[start:end], session_key, logger)
+            gitRepoURL = gitRepoURL[0:start-9] + temp_password + gitRepoURL[end:]
     else:
         gitRepoHTTP = False
 
diff --git a/bin/splunkversioncontrol_restore_class.py b/bin/splunkversioncontrol_restore_class.py
diff --git a/default/app.conf b/default/app.conf
