fix: add retry limit for cloning

Author: sumedha-bhatt

config.json.sample

@@ -9,7 +9,8 @@
   "access_modules": {
     "git_urls": [
       "https://github.com/browserstack/enigma-public-access-modules.git"
-    ]
+    ],
+    "RETRY_LIMIT": 5
   },
   "enigmaGroup": {
     "MAIL_APPROVER_GROUPS": [

schema.json

@@ -69,11 +69,16 @@
     "access_modules": {
       "description": "List of access modules attached to this tool",
       "type": "object",
-      "additionalProperties": false,
+      "additionalProperties": true,
       "properties": {
         "git_urls": {
           "description": "List of git URLs of access modules",
           "type": "array"
+        },
+        "RETRY_LIMIT": {
+          "description": "Number of retries before raising cloning failure exception",
+          "type": "integer",
+          "minimum": 1
         }
       }
     },

scripts/clone_access_modules.py

@@ -1,13 +1,15 @@
 import json
 import sys
 import shutil
-from git import Repo
+from git import Repo, GitCommandError
+import time
 import os
 
 print("Starting cloning setup")
 try:
     f = open("./config.json", "r")
     config = json.load(f)
+    RETRY_LIMIT = config["access_modules"]["RETRY_LIMIT"]
     urls = config["access_modules"]["git_urls"]
 
     for each_access_module in os.listdir('Access/access_modules'):
@@ -33,10 +35,26 @@
         folder_name = url.split("/").pop()[:-4]
         folder_path = "./Access/access_modules/" + folder_name
         try:
-            if specified_branch:
-                Repo.clone_from(url, folder_path, branch=specified_branch)
-            else:
-                Repo.clone_from(url, folder_path)
+            retry_exception = None
+            for i in range(1, RETRY_LIMIT + 1):
+                try:
+                    if specified_branch:
+                        Repo.clone_from(url, folder_path, branch=specified_branch)
+                    else:
+                        Repo.clone_from(url, folder_path)
+                except (GitCommandError, Exception) as ex:
+                    print("error occurred: ", ex)
+                    print("retry:{1}/{2}".format(ex, i, RETRY_LIMIT))
+                    retry_exception = ex
+                    time.sleep(10 * i)
+                    print("retrying")
+                else:
+                    retry_exception = None
+                    break
+            if (retry_exception != None):
+                print("max retry count reached")
+                raise retry_exception
+
             # move all folders, not files in the cloned repo to the access_modules
             # folder except the .git, .github and secrets folder
             for file in os.listdir(folder_path):
@@ -52,7 +70,7 @@
                         )
                     except:
                          print("File is already present.")
-                            
+
                 if(file == "requirements.txt"):
                     current_requirements_file = folder_path + "/" + file
                     #Read the requirements
@@ -64,12 +82,12 @@
 
                     # Merge the requirements
                     merged_requirements = list(set(requirements1 + requirements2))
-                       
+
                     #update the requirements.txt
                     with open(requirements_file, 'w') as out_file:
                         for requirement in sorted(merged_requirements):
                              out_file.write(requirement)
-                       
+
             print("Cloning successful!")
         except Exception as e:
            print("error-->",e)
@@ -84,7 +102,7 @@
         except Exception as e:
             print(e)
             print("failed to remove " + folder_path + " folder.")
-        
+
 except Exception as e:
     print("Access module cloning failed!")
     print(str(e))

scripts/validator.py

@@ -25,6 +25,6 @@
     validate(instance=config, schema=schema)
     logger.info("Schema validation passed!")
 except Exception as e:
-    logger.error("Schema validation failed! Error is: " + e)
+    logger.error("Schema validation failed! Error is: %s", e)
     traceback.format_exc()
     sys.exit(1)