Merge pull request #3840 from amvanbaren/fix-reports-issues

Reports improvements

Author: amvanbaren

reports/get_all_extensions.py

@@ -40,11 +40,9 @@ def get_extension(extension):
     while retry_count > 0:
         try:
             response = requests.get(extension_url)
-            if response.status_code == 200:
-                return response.json()
-            else:
-                raise Exception('%s: HTTP %s Error retrieving %s' % (datetime.now(), response.status_code,extension['url']))
-        except Exception as e:
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.RequestException as e:
             print("%s: %s" % (datetime.now(), e))
             retry_count -= 1
             time.sleep(2)
@@ -74,11 +72,11 @@ def get_all_by_license():
     all_extensions = get_all_extensions()
     count = 1
     for extension in all_extensions:
-        license = extension.get('license', 'None')
-        if license in extensions_by_license:
-            extensions_by_license[license].append(extension)
+        license_name = extension.get('license', 'None')
+        if license_name in extensions_by_license:
+            extensions_by_license[license_name].append(extension)
         else:
-            extensions_by_license[license] = [extension]
+            extensions_by_license[license_name] = [extension]
         if int(count/100) == count/100:
             print('Processed %s of %s.' % (count, len(all_extensions)))
         count += 1
@@ -89,7 +87,6 @@ def write_json_file(extensions):
     f = open(JSON_FILENAME, 'w')
     f.write(json.dumps(extensions, indent=4))
     f.close()
-    return
 
 def write_tsv_file(extensions):
     f = open(TSV_FILENAME, 'w')

reports/get_availability_data.py

@@ -22,18 +22,15 @@ def make_api_call(url):
     while not done:
         try:
             response = requests.get(url, headers=HEADERS)
-            if response.status_code != 200:
-                raise Exception('%s HTTP error %s' % (url, response.status_code))
-            else:
-                done = True
-        except Exception as e:
-            print(" %s, retrying..." % e)
+            response.raise_for_status()
+            done = True
+        except requests.exceptions.RequestException as e:
             if retry_count > 0:
+                print(" %s, retrying..." % e)
                 time.sleep((6 - retry_count) * 5)
                 retry_count = retry_count - 1
             else:
-                done = True
-                raise Exception('Failing call to %s after multiple retries' % url)
+                raise
 
     return response.json()
 
@@ -58,7 +55,7 @@ def get_monitor_url(id, start_date, end_date):
     return '%s/monitors/%s/sla?from=%s&to=%s' % (API_URL, id, start_date.strftime('%Y-%m-%d'), end_date.strftime('%Y-%m-%d'))
 
 def get_monitor_data(monitor, time_span):
-    id = monitor['id']
+    monitor_id = monitor['id']
     name = monitor['attributes']['pronounceable_name']
     date_str = monitor['attributes']['created_at']
     start_date = datetime.strptime(date_str[0:10], '%Y-%m-%d')
@@ -69,11 +66,11 @@ def get_monitor_data(monitor, time_span):
     downtime_data = []
     print('processing %s' % name)
     while end_date <= today:
-        availability_url = get_monitor_url(id, start_date, end_date)
+        availability_url = get_monitor_url(monitor_id, start_date, end_date)
         json_results = make_api_call(availability_url)
         dates.append(np.datetime64(end_date.strftime('%Y-%m-%d')))
         sla_data.append(json_results['data']['attributes']['availability'])
-        downtime_url = get_monitor_url(id, start_date, start_date)
+        downtime_url = get_monitor_url(monitor_id, start_date, start_date)
         json_results = make_api_call(downtime_url)
         downtime_data.append(json_results['data']['attributes']['total_downtime']/60)
         start_date = start_date + timedelta(days=1)
@@ -85,15 +82,15 @@ def get_continuous_data(time_span=30):
     monitors = get_all_monitors()
     results = []
     for monitor in monitors:
-        name, dates, sla_data, downtime_data = get_monitor_data(monitor, time_span=30)
+        name, dates, sla_data, downtime_data = get_monitor_data(monitor, time_span)
         results.append({'name': name,
                         'dates': dates,
                         'sla_data': sla_data,
                         'downtime_data': downtime_data})
     return results
 
 def get_monthly_monitor_data(monitor):
-    id = monitor['id']
+    monitor_id = monitor['id']
     name = monitor['attributes']['pronounceable_name']
     date_str = monitor['attributes']['created_at']
     interval_start_date = datetime.strptime(date_str[0:10], '%Y-%m-%d')
@@ -105,12 +102,12 @@ def get_monthly_monitor_data(monitor):
     while interval_start_date < end_date:
         interval_days_in_month = calendar.monthrange(interval_start_date.year, interval_start_date.month)[1]
         interval_end_date = interval_start_date + timedelta(days=interval_days_in_month - interval_start_date.day)
-        availability_url = get_monitor_url(id, interval_start_date, interval_end_date)
+        availability_url = get_monitor_url(monitor_id, interval_start_date, interval_end_date)
         json_results = make_api_call(availability_url)
         dt = interval_start_date.strftime('%Y-%m')
         dates.append(np.datetime64(dt))
         sla_data.append(json_results['data']['attributes']['availability'])
-        downtime_url = get_monitor_url(id, interval_start_date, interval_end_date)
+        downtime_url = get_monitor_url(monitor_id, interval_start_date, interval_end_date)
         json_results = make_api_call(downtime_url)
         downtime_data.append(json_results['data']['attributes']['total_downtime']/60)
         interval_start_date = interval_end_date + timedelta(days=1)

reports/get_open_vsx_data.py

@@ -65,31 +65,63 @@ def get_publishing_data(starting_year, starting_month):
     df = pd.DataFrame(data,columns=HEADERS)
     return df
 
-def get_most_active_data(starting_year, starting_month):
+def process_most_active_data(most_active):
+    resulting_dfs = {}
+    for key in most_active.keys():
+        if key == 'dates':
+            dates = most_active['dates']
+        else:
+            data = {'dates': dates}
+            for entry in most_active[key]['unique']:
+                data[entry] = [None] * len(most_active['dates'])
+            for i in range(len(dates)):
+                date = dates[i]
+                for entry in most_active[key][date]:
+                    item = list(entry.values())[0]
+                    value = list(entry.values())[1]
+                    data[item][i] = value
+            df = pd.DataFrame(data, columns=most_active[key]['unique'])
+            df['date'] = dates
+            resulting_dfs[key] = df
+    return resulting_dfs
+
+def most_active_data_append_unique(most_active, json_results, top_prop, key):
+    for item in json_results[top_prop]:
+        if item[key] not in most_active[top_prop]['unique']:
+            most_active[top_prop]['unique'].append(item[key])
+
+def extract_most_active_data_from_json(most_active, json_results, year, month):
+    top_publishers = 'topMostActivePublishingUsers'
+    top_extensions = 'topNamespaceExtensions'
+    top_extension_versions = 'topNamespaceExtensionVersions'
+    top_downloads = 'topMostDownloadedExtensions'
+
+    no_data = True
+    top_props = [top_publishers, top_extensions, top_extension_versions, top_downloads]
+    for top_prop in top_props:
+        if len(json_results[top_prop]) > 0:
+            no_data = False
+            break
+
+    if no_data:
+        return
 
-    def process_data(most_active):
-        resulting_dfs = {}
-        for key in most_active.keys():
-            if key == 'dates':
-                dates = most_active['dates']
-            else:
-                data = {'dates': dates}
-                for entry in most_active[key]['unique']:
-                    data[entry] = [None] * len(most_active['dates'])
-                for i in range(len(dates)):
-                    date = dates[i]
-                    for entry in most_active[key][date]:
-                        item = list(entry.values())[0]
-                        value = list(entry.values())[1]
-                        data[item][i] = value
-                df = pd.DataFrame(data, columns=most_active[key]['unique'])
-                df['date'] = dates
-                resulting_dfs[key] = df
-        return resulting_dfs
-
-    current_year = date.today().year
-    current_month = date.today().month
+    year_month = '%s/%s' % (month, str(year)[2:])
+    most_active['dates'].append(year_month)
 
+    most_active[top_publishers][year_month] = json_results[top_publishers]
+    most_active_data_append_unique(most_active, json_results, top_publishers, 'userLoginName')
+
+    most_active[top_extensions][year_month] = json_results[top_extensions]
+    most_active_data_append_unique(most_active, json_results, top_extensions, 'namespace')
+
+    most_active[top_extension_versions][year_month] = json_results[top_extension_versions]
+    most_active_data_append_unique(most_active, json_results, top_extension_versions, 'namespace')
+
+    most_active[top_downloads][year_month] = json_results[top_downloads]
+    most_active_data_append_unique(most_active, json_results, top_downloads, 'extensionIdentifier')
+
+def get_most_active_data(starting_year, starting_month):
     most_active = {
         'dates': [],
         'topMostActivePublishingUsers': {
@@ -105,44 +137,22 @@ def process_data(most_active):
             'unique': []
         }
     }
-    for year in range(starting_year, current_year + 1):
-        for month in range(1, 13):
-            if year == starting_year and month >= starting_month or \
-                starting_year < year < current_year or year == current_year and month <= current_month:
-                url = '%sadmin/report?year=%s&month=%s&token=%s' % (API_ENDPOINT, year, month, ACCESS_TOKEN)
-                response = requests.get(url)
-                if response.status_code == 200:
-                    try:
-                        json_results = response.json()
-                        if len(json_results['topMostActivePublishingUsers']) > 0 or len(json_results['topNamespaceExtensions']) > 0 or \
-                            len(json_results['topNamespaceExtensionVersions']) > 0 or len(json_results['topMostDownloadedExtensions']) > 0:
-                            year_month = '%s/%s' % (month, str(year)[2:])
-                            most_active['dates'].append(year_month)
-                            most_active['topMostActivePublishingUsers'][year_month] = json_results['topMostActivePublishingUsers']
-                            for item in json_results['topMostActivePublishingUsers']:
-                                if item['userLoginName'] not in most_active['topMostActivePublishingUsers']['unique']:
-                                    most_active['topMostActivePublishingUsers']['unique'].append(item['userLoginName'])
-                            most_active['topNamespaceExtensions'][year_month] = json_results['topNamespaceExtensions']
-                            for item in json_results['topNamespaceExtensions']:
-                                if item['namespace'] not in most_active['topNamespaceExtensions']['unique']:
-                                    most_active['topNamespaceExtensions']['unique'].append(item['namespace'])
-                            most_active['topNamespaceExtensionVersions'][year_month] = json_results['topNamespaceExtensionVersions']
-                            for item in json_results['topNamespaceExtensionVersions']:
-                                if item['namespace'] not in most_active['topNamespaceExtensionVersions']['unique']:
-                                    most_active['topNamespaceExtensionVersions']['unique'].append(item['namespace'])
-                            most_active['topMostDownloadedExtensions'][year_month] = json_results['topMostDownloadedExtensions']
-                            for item in json_results['topMostDownloadedExtensions']:
-                                if item['extensionIdentifier'] not in most_active['topMostDownloadedExtensions']['unique']:
-                                    most_active['topMostDownloadedExtensions']['unique'].append(item['extensionIdentifier'])
-                    except JSONDecodeError:
-                        json_results = None
-                        print("Error decoding JSON results for %s" % url)
-                else:
-                    print("%s error processing results for %s" % (response.status_code, url))
-    
-    resulting_dfs = process_data(most_active)
 
-    return resulting_dfs
+    today = date.today()
+    start_date = date(starting_year, starting_month, 1)
+    while start_date.year < today.year or (start_date.year == today.year and start_date.month < today.month):
+        url = '%sadmin/report?year=%s&month=%s&token=%s' % (API_ENDPOINT, start_date.year, start_date.month, ACCESS_TOKEN)
+        response = requests.get(url)
+        if response.status_code == 200:
+            try:
+                extract_most_active_data_from_json(most_active, response.json(), start_date.year, start_date.month)                
+            except JSONDecodeError:
+                print("Error decoding JSON results for %s" % url)
+        else:
+            print("%s error processing results for %s" % (response.status_code, url))
+        start_date = start_date + relativedelta(months=1)
+    
+    return process_most_active_data(most_active)
 
 if __name__ == '__main__':
     reports = get_available_reports()

reports/get_vs_marketplace_data.py

@@ -10,10 +10,9 @@
 from datetime import datetime
 
 def get_ms_info(ext):
-    extensionName = ext['extensionName']
-    publisherName = ext['publisher']['publisherName']
-    displayName = ext['displayName']
-    displayName = displayName.replace(',', ' ')
+    extension_name = ext['extensionName']
+    publisher_name = ext['publisher']['publisherName']
+    display_name = ext['displayName'].replace(',', ' ')
     latest_version = ext['versions'][0]['version']
     last_updated = ext['versions'][0]['lastUpdated']
 
@@ -27,7 +26,7 @@ def get_ms_info(ext):
     except Exception:
         pricing = None
 
-    return extensionName, publisherName, displayName, latest_version, last_updated, repo, pricing
+    return extension_name, publisher_name, display_name, latest_version, last_updated, repo, pricing
 
 def convert_date_str(input_str):
     date_str = input_str[0:input_str.find('T')]
@@ -107,15 +106,15 @@ def convert_date_str(input_str):
     csv_file.write("MS Publisher (Namespace), MS Extension, MS DisplayName, MS Pricing, MS Version, MS Date, VSX Version, VSX Date, VSX Publisher, VSX License, Repo\n")
     for ext in extensions:
         print("%s.%s" % (ext['publisher']['publisherName'], ext['extensionName']))
-        ms_extensionName, ms_publisherName, ms_displayname, ms_latest_version, ms_last_updated, ms_repo, ms_pricing = get_ms_info(ext)
-        vsx_extension_url = '%s/%s/%s' % (VSX_API, ms_publisherName, ms_extensionName)
+        ms_extension_name, ms_publisher_name, ms_display_name, ms_latest_version, ms_last_updated, ms_repo, ms_pricing = get_ms_info(ext)
+        vsx_extension_url = '%s/%s/%s' % (VSX_API, ms_publisher_name, ms_extension_name)
         response = requests.get(vsx_extension_url)
         if response.status_code == 200:
             vsx_results = response.json()
             csv_file.write("%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s\n" % (
-                            ms_publisherName,
-                            ms_extensionName,
-                            ms_displayname,
+                            ms_publisher_name,
+                            ms_extension_name,
+                            ms_display_name,
                             ms_pricing,
                             ms_latest_version,
                             convert_date_str(ms_last_updated), 
@@ -127,9 +126,9 @@ def convert_date_str(input_str):
                             ))
         elif response.status_code == 404:
             csv_file.write("%s, %s, %s, %s, %s, %s, , , , %s, %s\n" % (
-                    ms_publisherName,
-                    ms_extensionName,
-                    ms_displayname,
+                    ms_publisher_name,
+                    ms_extension_name,
+                    ms_display_name,
                     ms_pricing,
                     ms_latest_version,
                     convert_date_str(ms_last_updated),
@@ -140,7 +139,7 @@ def convert_date_str(input_str):
         else:
             print(response.status_code)
             print(response.content)
-            raise Exception('HTTP %s error' % response.status_code)
+            response.raise_for_status()
 
     # Output JSON File
     json_file = open(JSON_FILE_NAME, 'w')

reports/graph_availability_trends.ipynb

@@ -131,7 +131,6 @@
    },
    "outputs": [],
    "source": [
-    "# plt.rcParams['figure.figsize'] = (15,9)\n",
     "fig, ax = plt.subplots()\n",
     "plt.ylim(97.0, 100.1)\n",
     "plt.title('Open-vsx.org Monthly Site Availability', fontsize=14)\n",
@@ -189,7 +188,6 @@
    },
    "outputs": [],
    "source": [
-    "# plt.rcParams['figure.figsize'] = (15,9)\n",
     "fig, ax = plt.subplots()\n",
     "plt.title('Open-vsx.org Monthly Site Downtime', fontsize=14)\n",
     "\n",