Add new libraries to daily data collection (#9)

* Add additional libraries to the download analytics collection

* Fix daily build

* Use fixed httplib2 version

* Improve way of inserting columns

* Update metrics.py

* Update metrics.py

Author: pvk-developer

config.yaml

@@ -14,3 +14,8 @@ projects:
   - datomize
   - gretel-trainer
   - ydata-sdk
+  - mostlyai
+  - synthcity
+  - smartnoise-synth
+  - realtabformer
+  - be-great

download_analytics/metrics.py

@@ -35,17 +35,20 @@ def _historical_groupby(downloads, groupbys=None):
     if groupbys is None:
         groupbys = downloads.set_index('timestamp').columns
 
+    new_columns = []
     for groupby in groupbys:
         grouped = downloads.groupby([year_month, groupby])
         grouped_sizes = grouped.size().unstack(-1)  # noqa: PD010
         if len(groupbys) > 1:
             grouped_sizes.columns = f"{groupby}='" + grouped_sizes.columns + "'"
+        new_columns.append(grouped_sizes.fillna(0))
 
-        base[grouped_sizes.columns] = grouped_sizes.fillna(0)
+    if new_columns:
+        base = pd.concat([base] + new_columns, axis=1)
 
     totals = base.sum()
     totals.name = 'total'
-    base = pd.concat([base, totals], ignore_index=True)
+    base = pd.concat([base, totals.to_frame().T], ignore_index=True)
 
     return base.reset_index().iloc[::-1]
 

download_analytics/pypi.py

@@ -150,7 +150,7 @@ def get_pypi_downloads(
         if max_date is None:
             all_downloads = new_downloads
         else:
-            if max_date < end_date:
+            if pd.Timestamp(max_date) < pd.Timestamp(end_date):
                 before = previous[previous.timestamp < new_downloads.timestamp.min()]
                 after = new_downloads
             else:

pyproject.toml

@@ -21,7 +21,8 @@ dependencies = [
     "PyYAML",
     "PyDrive",
     "google-cloud-bigquery",
-    "db-dtypes"
+    "db-dtypes",
+    "httplib2==0.15.0",  # https://stackoverflow.com/questions/59815620/gcloud-upload-httplib2-redirectmissinglocation-redirected-but-the-response-is-m
 ]
 
 [project.urls]