Updated splunk_vc_kom_audit_summary report

Added i=StanzaName to the indexed data when running the audit query

Now attempting to hide (most) passwords from the logs by default (for example when an OS error occurs don't print the stdout including the password in use)

git diff now uses --no-pager to prevent trucation of the diff command with -U0 (no context)

New options:
`disable_file_deletion` - do not delete files in remote git repo that are not found during backup, useful for testing

`use_wdiff` - sends the output of the diff command to Unix command wdiff to provide a nicer diff output

Updated report:
`SplunkVersionControl ChangeDetector Non-Directory` now excludes the CIM Risk and Incident_Management datamodels as they update very frequently with close to zero changes (calculationId changes only)

Author: gjanders

README.md

@@ -289,6 +289,23 @@ To do this you will need to install Version Control For SplunkCloud on your Splu
 [SplunkVersionControlCloud github](https://github.com/gjanders/SplunkVersionControlCloud)
 
 ## Release Notes
+### 1.2.4
+Updated `splunk_vc_kom_audit_summary` report
+
+Added i=StanzaName to the indexed data when running the audit query
+
+Now attempting to hide (most) passwords from the logs by default (for example when an OS error occurs don't print the stdout including the password in use)
+
+git diff now uses --no-pager to prevent trucation of the diff command with -U0 (no context)
+
+New options:
+`disable_file_deletion` - do not delete files in remote git repo that are not found during backup, useful for testing
+
+`use_wdiff` - sends the output of the diff command to Unix command wdiff to provide a nicer diff output
+
+Updated report:
+`SplunkVersionControl ChangeDetector Non-Directory` now excludes the CIM Risk and Incident_Management datamodels as they update very frequently with close to zero changes (calculationId changes only)
+
 ### 1.2.3
 New option `disable_git_ssl_verify`
 

README/inputs.conf.spec

@@ -25,6 +25,8 @@ excludeOwner = <value>
 * comma separated list of owners objects that should be transferred
 debugMode = <boolean>
 * turn on DEBUG level logging (defaults to INFO) (true/false), default false
+show_passwords = <boolean>
+* Show passwords in the DEBUG/ERROR logs (hidden by default)
 useLocalAuth = <boolean>
 * do not use the srcUsername/srcPassword, use the session_key of the user running the modular input instead (works on localhost only) (true/false), default false
 remoteAppName = <value>
@@ -33,7 +35,7 @@ appsList = <value>
 * Comma separated list of apps, this changes Splunk Version Control to not list all applications and instead only runs a backup on the specified apps
 git_command = <value>
 * defaults to 'git', can be overriden (for example on a Windows server) to use a full path to the git command
-ssh_command = <value> 
+ssh_command = <value>
 * defaults to 'ssh', can be overriden (for example on a Windows server) to use a full path to the ssh command
 proxy = <value>
 * If supplied provides a proxy setting to use to access the srcURL (https proxy). Use https://user:password:[email protected]:3128 and the application will obtain the password for the entry 'passwordinpasswordsconf'. If password: is not used the password is used as per a normal proxy setting, for example https://user:[email protected]:3128
@@ -53,6 +55,10 @@ run_ko_diff = <boolean>
 * Should output of the modular input include diff information (requires run_ko_query to be true, defaults to false)
 disable_git_ssl_verify = <boolean>
 * Use GIT_SSL_NO_VERIFY=true on all git commands
+use_wdiff = <boolean>
+* Enables the diff HEAD~1 to be passed to wdiff for improved formatting if run_ko_diff is enabled
+disable_file_deletion = <boolean>
+* By default if the app or file no longer exists than it is deleted from the git repo, this stops the deletion from occurring
 
 [splunkversioncontrol_restore://<name>]
 destURL = <value>
@@ -71,6 +77,8 @@ auditLogsLookupBackTime = <value>
 * This is how far back the audit logs will be checked to ensure that a restore entry is valid, this should be set to your interval time or slightly more, defaults to -1h (use Splunk format)
 debugMode = <boolean>
 * turn on DEBUG level logging (defaults to INFO) (true/false), default false
+show_passwords = <boolean>
+* Show passwords in the DEBUG/ERROR logs (hidden by default)
 useLocalAuth = <boolean>
 * do not use the srcUsername/srcPassword, use the session_key of the user running the modular input instead (works on localhost only) (true/false), default false
 remoteAppName = <value>
@@ -91,4 +99,3 @@ file_per_ko = <boolean>
 * Do you want one file per knowledge object? Or a combined file? Defaults to false (i.e. 1 large file for global dashboards in an app). Note that if you change this you will need to re-create or wipe the repository as the files are stored differently...Note this setting should match in both backup and restore modular inputs for a particular repo
 disable_git_ssl_verify = <boolean>
 * Use GIT_SSL_NO_VERIFY=true on all git commands
-

bin/splunkversioncontrol_backup.py

@@ -15,7 +15,7 @@
  Store Knowledge Objects
    Attempt to run against the Splunk REST API to obtain various knowledge objects, then persist the knowledge object information required
    to restore the knowledge object if it was deleted/changed to the filesystem
- 
+
 """
 
 #Define the XML scheme for the inputs page
@@ -94,6 +94,13 @@
                 <required_on_create>false</required_on_create>
                 <data_type>boolean</data_type>
             </arg>
+            <arg name="show_passwords">
+                <title>show_passwords</title>
+                <description>Show passwords in the DEBUG/ERROR logs (hidden by default)</description>
+                <validation>is_bool('show_passwords')</validation>
+                <required_on_create>false</required_on_create>
+                <data_type>boolean</data_type>
+            </arg>
             <arg name="useLocalAuth">
                 <title>useLocalAuth</title>
                 <description>Instead of using the srcUsername/srcPassword, use the session_key of the user running the modular input instead (works on localhost only) (true/false), default false</description>
@@ -174,6 +181,20 @@
                 <data_type>boolean</data_type>
                 <validation>is_bool('disable_git_ssl_verify')</validation>
             </arg>
+            <arg name="use_wdiff">
+                <title>use_wdiff</title>
+                <description>Enables the diff HEAD~1 to be passed to wdiff for improved formatting if run_ko_diff is enabled</description>
+                <required_on_create>false</required_on_create>
+                <data_type>boolean</data_type>
+                <validation>is_bool('use_wdiff')</validation>
+            </arg>
+            <arg name="disable_file_deletion">
+                <title>disable_file_deletion</title>
+                <description>By default if the app or file no longer exists than it is deleted from the git repo, this stops the deletion from occurring</description>
+                <required_on_create>false</required_on_create>
+                <data_type>boolean</data_type>
+                <validation>is_bool('disable_file_deletion')</validation>
+            </arg>
         </args>
     </endpoint>
 </scheme>
@@ -218,7 +239,7 @@ def print_error(s):
 #Validate the arguments to the app to ensure this will work...
 def validate_arguments():
     val_data = get_validation_data()
-    
+
     if 'debugMode' in val_data:
         debugMode = val_data['debugMode'].lower()
         if debugMode == "true" or debugMode == "t" or debugMode == "1":
@@ -240,7 +261,7 @@ def validate_arguments():
         else:
             print_error("useLocalAuth argument should be true or false, invalid config")
             sys.exit(2)
-    
+
     #If we're not using the useLocalAuth we must have a username/password to work with
     if not useLocalAuth and ('srcUsername' not in val_data or 'srcPassword' not in val_data):
         print_error("useLocalAuth is not set to true and srcUsername/srcPassword not set, invalid config")
@@ -318,14 +339,15 @@ def validate_arguments():
 
     gitRepoURL = val_data['gitRepoURL']
     proxy_command = ""
+    git_password = False
     if gitRepoURL.find("http") == 0:
         gitRepoHTTP = True
         if gitRepoURL.find("password:") != -1:
             start = gitRepoURL.find("password:") + 9
             end = gitRepoURL.find("@")
             logger.debug("Attempting to replace gitRepoURL=%s by subsituting=%s with a password" % (gitRepoURL, gitRepoURL[start:end]))
-            temp_password = get_password(gitRepoURL[start:end], session_key, logger)
-            gitRepoURL = gitRepoURL[0:start-9] + temp_password + gitRepoURL[end:]
+            git_password = get_password(gitRepoURL[start:end], session_key, logger)
+            gitRepoURL = gitRepoURL[0:start-9] + git_password + gitRepoURL[end:]
     else:
         gitRepoHTTP = False
 
@@ -347,21 +369,33 @@ def validate_arguments():
         else:
             proxy_command = "export " + proxy_command + " ; "
 
+    show_passwords = False
+    if 'show_passwords' in val_data:
+        if val_data['show_passwords'].lower() == 'true' or val_data['show_passwords'] == "1":
+            show_passwords = True
+            logger.debug('show_passwords is now true due to show_passwords: ' + val_data['show_passwords'])
+
     (stdout, stderr, res) = runOSProcess("%s %s ls-remote %s" % (proxy_command, git_command, gitRepoURL), logger, shell=True)
     #If we didn't manage to ls-remote perhaps we just need to trust the fingerprint / this is the first run?
     if res == False and not gitRepoHTTP:
+        if not show_passwords and git_password:
+            stdout = stdout.replace(git_password, "password_removed")
+            stderr = stderr.replace(git_password, "password_removed")
         logger.error("Possible first run trying again" % (stdout, stderr))
         (stdout, stderrout, res) = runOSProcess(ssh_command + " -n -o \"BatchMode yes\" -o StrictHostKeyChecking=no " + gitRepoURL[:gitRepoURL.find(":")], logger)
         (stdout, stderr, res) = runOSProcess("%s ls-remote %s" % (git_command, gitRepoURL), logger)
 
     if res == False:
+        if not show_passwords and git_password:
+            stdout = stdout.replace(git_password, "password_removed")
+            stderr = stderr.replace(git_password, "password_removed")
         print_error("Failed to validate the git repo URL, stdout of '%s', stderr of '%s'" % (stdout, stderr))
         logger.error("Failed to validate the git repo URL, stdout of '%s', stderr of '%s'" % (stdout, stderr))
         sys.exit(6)
 
 #Print the scheme
 def do_scheme():
-    print(SCHEME)    
+    print(SCHEME)
 
 splunkLogsDir = os.environ['SPLUNK_HOME'] + "/var/log/splunk"
 #Setup the logging
@@ -381,7 +415,7 @@ def do_scheme():
               'maxBytes' :  2097152,
               'level': logging.DEBUG,
               'backupCount' : 5 }
-        },        
+        },
     root = {
         'handlers': ['h','file'],
         'level': logging.DEBUG,