Increase timeout for commands to a default of 60 seconds

Ensure a valid message is sent back to the user if a dynamic restore fails
If git checkout times out, cancel the restore attempt

Author: gjanders

README.md

@@ -261,6 +261,11 @@ To do this you will need to install Version Control For SplunkCloud on your Splu
 [SplunkVersionControlCloud github](https://github.com/gjanders/SplunkVersionControlCloud)
 
 ## Release Notes 
+### 1.1.7
+Increase timeout for commands to a default of 60 seconds
+Ensure a valid message is sent back to the user if a dynamic restore fails
+If git checkout times out, cancel the restore attempt
+
 ### 1.1.6
 Allow the backup process to run on search head clusters for those that wish to do this...
 

SplunkVersionControl.tgz

@@ -37,7 +37,7 @@
               'maxBytes' :  2097152,
               'level': logging.DEBUG,
               'backupCount' : 5 }
-        },        
+        },
     root = {
         'handlers': ['h','file'],
         'level': logging.DEBUG,
@@ -64,7 +64,7 @@ def handle_POST(self):
         payload = six.moves.urllib.parse.parse_qs(self.request['payload'])
         #self.response.write(str(payload) + "\n")
         #currently we only receive the Splunk authorization key, so obtain that
-        
+
         req_attributes = ['Authorization', 'splunk_vc_name', 'app', 'type', 'obj_name', 'tag', 'scope', 'timeout' ]
         for attr in req_attributes:
             if attr not in payload:
@@ -74,49 +74,49 @@ def handle_POST(self):
                 logger.error("Received remote call but attr=%s was missing from arguments, received=\"%s\"" % (attr, payload))
                 self.response.write("Received remote call but attr=%s was missing from arguments, received=\"%s\"" % (attr, payload))
                 return
-        
+
         headers = { "Authorization" : payload['Authorization'][0] }
 
         #Run a query back against the source system to check the username/role
         remoteAddr = self.request['remoteAddr']
         url = "https://" + remoteAddr + ":8089/services/authentication/current-context?output_mode=json"
         logger.info("Received remote request checking username and role related to the token on url=%s" % (url))
         logger.debug("token=%s" % (payload['Authorization'][0]))
-        
+
         res = self.runHttpRequest(url, headers, None, "get", "checking username with token (based on incoming ip address)")
         if not res:
             return
-        
-        json_dict = json.loads(res.text) 
+
+        json_dict = json.loads(res.text)
         #self.response.write(str(json_dict) + "\n\n\n")
         username = json_dict['entry'][0]['content']['username']
         roles = json_dict['entry'][0]['content']['roles']
 
         #self.response.write(username + "\n")
         #self.response.write(str(roles) + "\n")
         logger.info("username=%s roles=%s" % (username, roles))
-        
+
         splunk_vc_name = payload['splunk_vc_name'][0]
         #self.response.write(splunk_vc_name + "\n\n\n")
-        
+
         #Now run queries locally to check if the mentioned config matches an existing backup name
         headers = { "Authorization" : "Splunk " + self.request['systemAuth'] }
         url = "https://localhost:8089/servicesNS/-/-/data/inputs/splunkversioncontrol_restore/" + six.moves.urllib.parse.quote(splunk_vc_name) + "?output_mode=json"
         logger.debug("Now running query against url=%s to obtain config information" % (url))
-        
+
         res = self.runHttpRequest(url, headers, None, "get", "querying the inputs for splunkversioncontrol_restore with name %s" % (splunk_vc_name))
         if not res:
             return
-        
+
         #Look under the entry/content section for the relevant information we require, mainly destURL, useLocalAuth and potentially destUsername/destPassword
         json_dict = json.loads(res.text)['entry'][0]['content']
         #self.response.write(str(json_dict) + "\n\n\n")
-        
+
         useLocalAuth = False
         if 'useLocalAuth' in json_dict:
             if json_dict['useLocalAuth'].lower() == 't' or json_dict['useLocalAuth'].lower() == "true":
                 useLocalAuth = True
-        
+
         if not useLocalAuth:
             if not 'destUsername' in json_dict or not 'destPassword' in json_dict or not 'destURL' in json_dict:
                 logger.error("Missing one of destUsername, destPassword or destURL from the splunk version control restore stanza, and useLocalAuth is not true, invalid configuration")
@@ -125,7 +125,7 @@ def handle_POST(self):
             destUsername = json_dict['destUsername']
             destPassword = json_dict['destPassword']
             if destPassword.find("password:") == 0:
-                destPassword = get_password(destPassword[9:], self.request['systemAuth'], logger)
+                destPassword = get_password(destPassword[9:], self.request['systemAuth'], logger)            
         else:
             if not 'destURL' in json_dict:
                 logger.error("Missing one of destURL from the splunk version control restore stanza, invalid configuration")
@@ -136,7 +136,7 @@ def handle_POST(self):
 
         headers = {}
         auth = None
-        
+
         if useLocalAuth:
             headers = { "Authorization" : "Splunk " + self.request['systemAuth'] }
         else:
@@ -154,30 +154,30 @@ def handle_POST(self):
                 logger.warn("Time wait value of %s is invalid, not an integer, defaulting to 600 seconds" % (json_dict['timewait']))
                 time_wait = 600
         else:
-            time_wait = 600  
-        
+            time_wait = 600
+
         app = payload['app'][0]
         type = payload['type'][0]
         obj_name = payload['obj_name'][0]
         tag = payload['tag'][0]
         timeout = payload['timeout'][0]
-        
+
         if not 'restoreAsUser' in payload:
             restoreAsUser = ''
         else:
             restoreAsUser = payload['restoreAsUser'][0]
         scope = payload['scope'][0]
-        
+
         logger.debug("Converting timeout of argument %s to integer" % timeout)
         timeout = int(timeout)
         #We need a little bit of time to index the _audit event that literally just happened
         #a 30 second delay is annoying but it appears to work...hardcoding this for now
         logger.info("Sleeping for %s seconds to wait for audit logs" % (timeout))
         time.sleep(timeout)
         logger.info("Sleep completed")
-        
+
         starttime = starttime-60-timeout
-        
+
         json_res = self.runSearchJob(destURL, remoteAppName, headers, auth, username, starttime)
 
         if 'error' in json_res:
@@ -202,8 +202,8 @@ def handle_POST(self):
             url = "https://localhost:8089/servicesNS/nobody/SplunkVersionControl/storage/collections/data/splunkversioncontrol_rest_restore_status"
             res = self.runHttpRequest(url, headers, None, "get", "checking kvstore collection splunkversioncontrol_rest_restore_status")
             if not res:
-                return 
-            
+                return
+
             res = json.loads(res.text)
             #An empty list is good in this case, we are safe to run, if not we have checks to do
             if not len(res) == 0:
@@ -223,37 +223,37 @@ def handle_POST(self):
                         self.response.write("Attempted to run but found a running restore instance with time %s and current time is %s, will delete and move on after current time minus %s seconds (which would be %s) " % (kvstore_start_time, curtime, time_wait, removal_target))
                         self.response.write("Please try your restore request again in a minute...")
                         return
-            
+
             payload = json.dumps({ 'start_time': curtime })
             headers['Content-Type'] = 'application/json'
             #update kvstore with runtime
             res = self.runHttpRequest(url, headers, payload, 'post', 'updating kvstore collection splunkversioncontrol_rest_restore_status')
             if not res:
                 return res
-            
-            result = svc_restore_obj.run_script(resList, json_dict)
-            if result:
+
+            (result, message) = svc_restore_obj.run_script(resList, json_dict)
+            if result == True:
                 self.response.write("Restore has completed successfully in app %s, object of type %s, with name %s was restored from tag %s, scope %s with restoreAsUser %s and your username of %s" % (app, type, obj_name, tag, scope, restoreAsUser, username))
                 logger.info("Restore has completed successfully in app=%s, object of type=%s, with name=%s was restored from tag=%s, scope=%s with restoreAsUser=%s and requested by username=%s" % (app, type, obj_name, tag, scope, restoreAsUser, username))
             else:
-                self.response.write("Restore has failed to complete successfully in app %s, object of type %s, with name %s was not restored from tag %s, scope %s with restoreAsUser %s and your username of %s" % (app, type, obj_name, tag, scope, restoreAsUser, username))
-                logger.warn("Restore has failed to complete successfully in app=%s, object of type=%s, with name=%s was not restored from tag=%s, scope=%s with restoreAsUser=%s and requested by username=%s" % (app, type, obj_name, tag, scope, restoreAsUser, username))    
-            
+                self.response.write("Restore has failed to complete successfully in app %s, object of type %s, with name %s, from tag %s, scope %s with restoreAsUser %s and your username of %s. Message is %s" % (app, type, obj_name, tag, scope, restoreAsUser, username, message))
+                logger.warn("Restore has failed to complete successfully in app=%s, object of type=%s, with name=%s, from tag=%s, scope=%s with restoreAsUser=%s and requested by username=%s, message=%s" % (app, type, obj_name, tag, scope, restoreAsUser, username, message))
+
             self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status after completed run')
-    
+
     #Run a Splunk query via the search/jobs endpoint
     def runSearchJob(self, url, appname, headers, auth, username, earliest_time):
         url = url + "/servicesNS/-/%s/search/jobs" % (appname)
         query = "savedsearch \"Splunk Version Control Audit Query POST\" username=\"%s\" | stats count | where count>0" % (username)
         logger.debug("Running requests.post() on url=%s query=\"%s\"" % (url, query))
         data = { "search" : query, "output_mode" : "json", "exec_mode" : "oneshot", "earliest_time" : earliest_time }
-         
+
         res = requests.post(url, auth=auth, headers=headers, verify=False, data=data)
         if (res.status_code != requests.codes.ok):
             logger.error("url=%s status_code=%s reason=%s, response=\"%s\"" % (url, res.status_code, res.reason, res.text))
-            return { "error": "url=%s status_code=%s reason=%s, response=\"%s\"" % (url, res.status_code, res.reason, res.text) } 
+            return { "error": "url=%s status_code=%s reason=%s, response=\"%s\"" % (url, res.status_code, res.reason, res.text) }
         res = json.loads(res.text)
-        
+
         #Log return messages from Splunk, often these advise of an issue but not always...
         if len(res["messages"]) > 0:
             firstMessage = res["messages"][0]
@@ -271,10 +271,10 @@ def runHttpRequest(self, url, headers, data, type, text):
             res = requests.post(url, headers=headers, verify=False, data=data)
         elif type == "get":
             res = requests.get(url, headers=headers, verify=False)
-        
+
         if (res.status_code != requests.codes.ok and res.status_code != 201):
             logger.error("Unexpected response code while %s, on url=%s, statuscode=%s reason=%s, response=\"%s\", payload=\"%s\"" % (text, url, res.status_code, res.reason, res.text, data))
             self.response.write("Error unexpected response code while %s, on url %s, statuscode %s reason %s, response \"%s\", payload=\"%s\"" % (text, url, res.status_code, res.reason, res.text, data))
             return
-        
+
         return res