reduce complexity of get_all_extensions.py

Author: amvanbaren

reports/get_all_extensions.py

@@ -14,7 +14,7 @@
 
 url = API_ENDPOINT + 'api/-/search?size=100'
 
-def get_all_extensions():
+def retrieve_extensions():
     extensions = []
     done = False
     offset = 0
@@ -31,35 +31,43 @@ def get_all_extensions():
         except Exception as e:
             print("%s: %s" % (datetime.now(), e))
             done = True
-    
+
+    return extensions
+
+def get_extension(extension):
+    extension_url = API_ENDPOINT + 'api/%s/%s' % (extension['namespace'], extension['name'])
+    retry_count = 5
+    while retry_count > 0:
+        try:
+            response = requests.get(extension_url)
+            if response.status_code == 200:
+                return response.json()
+            else:
+                raise Exception('%s: HTTP %s Error retrieving %s' % (datetime.now(), response.status_code,extension['url']))
+        except Exception as e:
+            print("%s: %s" % (datetime.now(), e))
+            retry_count -= 1
+            time.sleep(2)
+
+    return None
+
+def get_all_extensions():
     count = 1
     all_extensions = []
+    extensions = retrieve_extensions()    
     print("\n\nStarting: %s" % datetime.now())
     for extension in extensions:
-        namespace_url = API_ENDPOINT + 'api/%s/%s' % (extension['namespace'], extension['name'])
-        retry_count = 5
-        while retry_count > 0:
-            try:
-                response = requests.get(namespace_url)
-                if response.status_code == 200:
-                    break
-                else:
-                    raise Exception('%s: HTTP %s Error retrieving %s' % (datetime.now(), response.status_code,extension['url']))
-            except Exception as e:
-                print("%s: %s" % (datetime.now(), e))
-                retry_count -= 1
-                time.sleep(2)
-        if retry_count == 0:
+        results = get_extension(extension)
+        if results is None:
             print('Error retrieving %s' % extension['url'])
         else:
-            results = response.json()
             all_extensions.append(results)
         if int(count/100) == count/100:
             print('Processed %s of %s.' % (count, len(extensions)))
         count += 1
     print("\n\nFinished %s API Calls: %s" % (count, datetime.now()))
 
-    return(all_extensions)
+    return all_extensions
 
 def get_all_by_license():
     extensions_by_license = {}
@@ -75,7 +83,7 @@ def get_all_by_license():
             print('Processed %s of %s.' % (count, len(all_extensions)))
         count += 1
 
-    return(dict(sorted(extensions_by_license.items())))
+    return dict(sorted(extensions_by_license.items()))
 
 def write_json_file(extensions):
     f = open(JSON_FILENAME, 'w')