fix parse version

Author: gsheni

.github/workflows/daily_summarize.yaml …ithub/workflows/daily_summarization.yaml.github/workflows/daily_summarize.yaml renamed to .github/workflows/daily_summarization.yaml .github/workflows/daily_summarize.yaml renamed to .github/workflows/daily_summarization.yaml

@@ -1,4 +1,4 @@
-name: Daily Summarize
+name: Daily Summarization
 
 on:
   workflow_dispatch:

README.md

@@ -78,7 +78,7 @@ pymetrics collect-pypi --max-days 30 --add-metrics --output-folder {OUTPUT_FOLDE
 ### Daily Collection
 On a daily basis, this workflow collects download data from PyPI and Anaconda. The data is then published in CSV format (`pypi.csv`). In addition, it computes metrics for the PyPI downloads (see [#Aggregation Metrics](#aggregation-metrics))
 
-### Daily Summarize
+### Daily Summarization
 
 On a daily basis, this workflow summarizes the PyPI download data from `pypi.csv` and calculates downloads for libraries. The summarized data is published to a GitHub repo:
 - [Downloads_Summary.xlsx](https://github.com/sdv-dev/sdv-dev.github.io/blob/gatsby-home/assets/Downloads_Summary.xlsx)

pymetrics/metrics.py

@@ -1,9 +1,10 @@
 """Functions to compute aggregation metrics over raw downloads."""
 
 import logging
-import re
 
+import numpy as np
 import pandas as pd
+from packaging.version import InvalidVersion, Version
 
 from pymetrics.output import create_spreadsheet
 
@@ -105,34 +106,6 @@ def _get_sheet_name(column):
 ]
 
 
-RE_NUMERIC = re.compile(r'^\d+')
-
-
-def _version_element_order_key(version):
-    components = []
-    last_component = None
-    last_numeric = None
-    for component in version.split('.', 2):
-        if RE_NUMERIC.match(component):
-            try:
-                numeric = RE_NUMERIC.match(component).group(0)
-                components.append(int(numeric))
-                last_component = component
-                last_numeric = numeric
-            except AttributeError:
-                # From time to time this errors out in github actions
-                # while it shouldn't enter the `if`.
-                pass
-
-    components.append(last_component[len(last_numeric) :])
-
-    return components
-
-
-def _version_order_key(version_column):
-    return version_column.apply(_version_element_order_key)
-
-
 def _mangle_columns(downloads):
     downloads = downloads.rename(columns=RENAME_COLUMNS)
     for col in [
@@ -154,6 +127,32 @@ def _mangle_columns(downloads):
     return downloads
 
 
+def _safe_version_parse(version_str):
+    if pd.isna(version_str):
+        return np.nan
+
+    try:
+        version = Version(str(version_str))
+    except InvalidVersion:
+        cleaned = str(version_str).rstrip('+~')
+        try:
+            version = Version(cleaned)
+        except (InvalidVersion, TypeError):
+            LOGGER.info(f'Unable to parse version: {version_str}')
+            version = np.nan
+
+    return version
+
+
+def _version_order_key(version_column):
+    return version_column.apply(_safe_version_parse)
+
+
+def _sort_by_version(data, column, ascending=False):
+    data = data.sort_values(by=column, key=_version_order_key, ascending=ascending)
+    return data
+
+
 def compute_metrics(downloads, output_path=None):
     """Compute aggregation metrics over the given downloads.
 
@@ -172,8 +171,7 @@ def compute_metrics(downloads, output_path=None):
         if column in SORT_BY_DOWNLOADS:
             sheet = sheet.sort_values('downloads', ascending=False)
         elif column in SORT_BY_VERSION:
-            sheet = sheet.sort_values(column, ascending=False, key=_version_order_key)
-
+            sheet = _sort_by_version(sheet, column=column, ascending=False)
         sheets[name] = sheet
 
     for column in HISTORICAL_COLUMNS:
@@ -182,7 +180,7 @@ def compute_metrics(downloads, output_path=None):
         sheets[name] = _historical_groupby(downloads, [column])
 
     if output_path:
-        create_spreadsheet(output_path, sheets)
+        create_spreadsheet(output_path, sheets, na_rep='<NaN>')
         return None
 
     return sheets

pymetrics/output.py

@@ -34,8 +34,8 @@ def get_path(folder, filename):
     return str(pathlib.Path(folder) / filename)
 
 
-def _add_sheet(writer, data, sheet_name):
-    data.to_excel(writer, sheet_name=sheet_name, index=False, engine='xlsxwriter')
+def _add_sheet(writer, data, sheet_name, na_rep=''):
+    data.to_excel(writer, sheet_name=sheet_name, index=False, engine='xlsxwriter', na_rep=na_rep)
 
     for column in data:
         column_length = None
@@ -51,7 +51,7 @@ def _add_sheet(writer, data, sheet_name):
         )
 
 
-def create_spreadsheet(output_path, sheets):
+def create_spreadsheet(output_path, sheets, na_rep=''):
     """Create a spreadsheet with the indicated name and data.
 
     If the ``output_path`` variable starts with ``gdrive://`` it is interpreted
@@ -74,7 +74,7 @@ def create_spreadsheet(output_path, sheets):
 
     with pd.ExcelWriter(output, engine='xlsxwriter') as writer:  # pylint: disable=E0110
         for title, data in sheets.items():
-            _add_sheet(writer, data, title)
+            _add_sheet(writer, data, title, na_rep=na_rep)
 
     if drive.is_drive_path(output_path):
         LOGGER.info('Creating file %s', output_path)

pymetrics/summarize.py

@@ -147,8 +147,7 @@ def get_previous_pypi_downloads(output_folder, dry_run=False):
         read_csv_kwargs['nrows'] = 10_000
     data = load_csv(csv_path, read_csv_kwargs=read_csv_kwargs)
     LOGGER.info('Parsing version column to Version class objects')
-    if 'version' in data.columns:
-        data['version'] = data['version'].apply(parse)
+    data['version'] = data['version'].apply(parse)
     return data
 
 

tests/unit/test_metrics.py

@@ -0,0 +1,54 @@
+import numpy as np
+import pandas as pd
+
+from pymetrics.metrics import _sort_by_version
+
+
+def test__sort_by_version():
+    # Setup
+    data = pd.DataFrame({
+        'version': pd.Series(
+            ['1.9.0', '1.9.0.dev0', '1.24.1', '0.9.1', '0.16.0', '0.0.0'], dtype='object'
+        ),
+        'name': ['v5', 'v4', 'v6', 'v2', 'v3', 'v1'],
+    })
+
+    # Run
+    sorted_df = _sort_by_version(data, 'version', ascending=False)
+
+    # Assert
+    expected_versions = ['1.24.1', '1.9.0', '1.9.0.dev0', '0.16.0', '0.9.1', '0.0.0']
+    assert sorted_df['version'].map(str).tolist() == expected_versions
+    assert sorted_df['name'].tolist() == ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']
+
+
+def test__sort_by_version_with_invalid_versions():
+    # Setup
+    data = pd.DataFrame({
+        'version': pd.Series(['2.7.11+', '2.0.0', 'invalid', '3.0', np.nan], dtype='object'),
+        'name': ['v4', 'v3', 'v2', 'v5', 'v1'],
+    })
+
+    # Run
+    sorted_df = _sort_by_version(data, 'version')
+
+    # Assert
+    expected_versions = ['3.0', '2.7.11+', '2.0.0', 'invalid', np.nan]
+    assert sorted_df['version'].tolist() == expected_versions
+    assert sorted_df['name'].tolist() == ['v5', 'v4', 'v3', 'v2', 'v1']
+
+
+def test__sort_by_version_with_mixed_version_formats():
+    # Setup
+    data = pd.DataFrame({
+        'version': ['1.0a1', '1.0b2', '1.0rc3', '1.0', '1.0.post0'],
+        'name': ['alpha', 'beta', 'rc', 'stable', 'post'],
+    })
+
+    # Run
+    sorted_df = _sort_by_version(data, 'version', ascending=False)
+
+    # Assert
+    expected_versions = ['1.0.post0', '1.0', '1.0rc3', '1.0b2', '1.0a1']
+    assert sorted_df['version'].tolist() == expected_versions
+    assert sorted_df['name'].tolist() == ['post', 'stable', 'rc', 'beta', 'alpha']