Update README and use secrets for folder outputs (#29)

* renamed repo

* renamed repo

* renamed repo

* rename folder

* wip

* fix dry run

* fix output folder

* fix output folder

* fix dry run

* force input/output folders

* lint

* speed up dry run

* lint

* fix secret

* fix secret

* fix anaconda

* fix input

* fix readme

* fix input file

* update docstring

* update docstring

Author: gsheni

.github/workflows/daily_collection.yaml

@@ -22,6 +22,7 @@ on:
 
 jobs:
   collect:
+    environment: prod
     runs-on: ubuntu-latest-large
     timeout-minutes: 25
     steps:
@@ -34,28 +35,27 @@ jobs:
         cache-dependency-glob: |
           **/pyproject.toml
           **/__main__.py
-    - name: Install pip and dependencies
+    - name: Install dependencies
       run: |
-        uv pip install -U pip
         uv pip install .
     - name: Collect PyPI Downloads
       run: |
         uv run pymetrics collect-pypi \
-          --verbose \
           --max-days ${{ inputs.max_days_pypi || 30 }} \
           --add-metrics \
-          --output-folder gdrive://***REMOVED***
+          --output-folder ${{ secrets.PYPI_OUTPUT_FOLDER }}
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
         BIGQUERY_CREDENTIALS: ${{ secrets.BIGQUERY_CREDENTIALS }}
+        PYPI_OUTPUT_FOLDER: ${{ secrets.PYPI_OUTPUT_FOLDER }}
     - name: Collect Anaconda Downloads
       run: |
         uv run pymetrics collect-anaconda \
-          --output-folder gdrive://***REMOVED***-Z \
-          --max-days ${{ inputs.max_days_anaconda || 90 }} \
-          --verbose
+          --output-folder ${{ secrets.ANACONDA_OUTPUT_FOLDER }} \
+          --max-days ${{ inputs.max_days_anaconda || 90 }}
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
+        ANACONDA_OUTPUT_FOLDER: ${{ secrets.ANACONDA_OUTPUT_FOLDER }}
   alert:
     needs: [collect]
     runs-on: ubuntu-latest
@@ -69,9 +69,12 @@ jobs:
         activate-environment: true
     - name: Install pip and dependencies
       run: |
-        uv pip install -U pip
         uv pip install -e .[dev]
     - name: Slack alert if failure
-      run: uv run python -m pymetrics.slack_utils -r ${{ github.run_id }} -c ${{ github.event.inputs.slack_channel || 'sdv-alerts' }}
+      run: |
+        uv run python -m pymetrics.slack_utils \
+        -r ${{ github.run_id }} \
+        -c ${{ github.event.inputs.slack_channel || 'sdv-alerts' }} \
+        -m 'Daily Collection PyMetrics failed :fire: :dumpster-fire: :fire:'
       env:
         SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}

.github/workflows/daily_summarize.yaml

@@ -12,6 +12,7 @@ on:
 
 jobs:
   summarize:
+    environment: prod
     runs-on: ubuntu-latest-large
     timeout-minutes: 10
     steps:
@@ -25,15 +26,14 @@ jobs:
           **/pyproject.toml
           **/__main__.py
     - name: Install pip and dependencies
-      run: |
-        uv pip install -U pip
-        uv pip install .
+      run: uv pip install .
     - name: Run Summarize
       run: |
         uv run pymetrics summarize \
-          --output-folder gdrive://***REMOVED***
+          --output-folder ${{ secrets.PYPI_OUTPUT_FOLDER }}
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
+        PYPI_OUTPUT_FOLDER: ${{ secrets.PYPI_OUTPUT_FOLDER }}
     - uses: actions/checkout@v4
       with:
         repository: sdv-dev/sdv-dev.github.io
@@ -63,13 +63,12 @@ jobs:
         activate-environment: true
     - name: Install pip and dependencies
       run: |
-        uv pip install -U pip
         uv pip install .[dev]
     - name: Slack alert if failure
       run: |
         uv run python -m pymetrics.slack_utils \
         -r ${{ github.run_id }} \
         -c ${{ github.event.inputs.slack_channel || 'sdv-alerts' }} \
-        -m 'Summarize Analytics build failed :fire: :dumpster-fire: :fire:'
+        -m 'Daily Summarize PyMetrics failed :fire: :dumpster-fire: :fire:'
       env:
         SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}

.github/workflows/dryrun.yaml

@@ -10,6 +10,7 @@ concurrency:
   cancel-in-progress: true
 jobs:
   dry_run:
+    environment: stage
     runs-on: ubuntu-latest-large
     timeout-minutes: 25
     steps:
@@ -24,33 +25,32 @@ jobs:
           **/__main__.py
     - name: Install pip and dependencies
       run: |
-        uv pip install -U pip
         uv pip install .
     - name: Collect PyPI Downloads - Dry Run
       run: |
         uv run pymetrics collect-pypi \
-          --verbose \
           --max-days 30 \
           --add-metrics \
-          --output-folder gdrive://***REMOVED*** \
+          --output-folder ${{ secrets.PYPI_OUTPUT_FOLDER }} \
           --dry-run
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
         BIGQUERY_CREDENTIALS: ${{ secrets.BIGQUERY_CREDENTIALS }}
+        PYPI_OUTPUT_FOLDER: ${{ secrets.PYPI_OUTPUT_FOLDER }}
     - name: Collect Anaconda Downloads - Dry Run
       run: |
         uv run pymetrics collect-anaconda \
-          --output-folder gdrive://***REMOVED***-Z \
           --max-days 90 \
-          --verbose \
+          --output-folder ${{ secrets.ANACONDA_OUTPUT_FOLDER }} \
           --dry-run
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
+        ANACONDA_OUTPUT_FOLDER: ${{ secrets.ANACONDA_OUTPUT_FOLDER }}
     - name: Summarize - Dry Run
       run: |
         uv run pymetrics summarize \
-          --verbose \
-          --output-folder gdrive://***REMOVED*** \
+          --output-folder ${{ secrets.PYPI_OUTPUT_FOLDER }} \
           --dry-run
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
+        PYPI_OUTPUT_FOLDER: ${{ secrets.PYPI_OUTPUT_FOLDER }}

.github/workflows/lint.yaml

@@ -20,7 +20,6 @@ jobs:
         activate-environment: true
     - name: Install pip and dependencies
       run: |
-        uv pip install -U pip
         uv pip install .[dev]
     - name: Run lint checks
       run: uv run invoke lint

.github/workflows/unit.yaml

@@ -26,7 +26,6 @@ jobs:
           activate-environment: true
       - name: Install pip and dependencies
         run: |
-          uv pip install -U pip
           uv pip install -e .[test,dev]
       - name: Run summarize
         run: |

.gitignore

@@ -3,6 +3,7 @@ bigquery_creds.json
 client_secrets.json
 credentials.json
 sdv-dev.github.io/*
+uv.lock
 
 notebooks
 *.xlsx

MANIFEST.in

@@ -1,5 +1,11 @@
-# PyMetrics
+<div align="center">
+<br/>
+<p align="center">
+    <i>This repository is part of <a href="https://sdv.dev">The Synthetic Data Vault Project</a>, a project from <a href="https://datacebo.com">DataCebo</a>.</i>
+</p>
+<div align="left">
 
+# PyMetrics
 The PyMetrics project allows you to extract download metrics for Python libraries published on [PyPI](https://pypi.org/) and [Anaconda](https://www.anaconda.com/).
 
 The DataCebo team uses these scripts to report download counts for the libraries in the [SDV ecosystem](https://sdv.dev/) and other libraries.
@@ -13,8 +19,8 @@ engagement metrics.
 Currently, the download data is collected from the following distributions:
 * [PyPI](https://pypi.org/): Information about the project downloads from [PyPI](https://pypi.org/)
   obtained from the public BigQuery dataset, equivalent to the information shown on
-  [pepy.tech](https://pepy.tech) and [ClickPy](https://clickpy.clickhouse.com/)
-  - More information about the BigQuery dataset can be found on the [official PyPI documentation](https://packaging.python.org/en/latest/guides/analyzing-pypi-package-downloads/)
+  [pepy.tech](https://pepy.tech), [ClickPy](https://clickpy.clickhouse.com/) or [pypistats](https://pypistats.org/).
+  - More information about the BigQuery dataset can be found on the [official PyPI documentation](https://packaging.python.org/en/latest/guides/analyzing-pypi-package-downloads/).
 
 * [Anaconda](https://www.anaconda.com/): Information about conda package downloads for default and select Anaconda channels.
   - The conda package download data is provided by Anaconda, Inc. It includes package download counts
@@ -24,7 +30,6 @@ Currently, the download data is collected from the following distributions:
       - Replace `{username}` with the Anaconda channel (`conda-forge`)
       - Replace `{package_name}` with the specific package (`sdv`) in the Anaconda channel
     - For each file returned by the API endpoint, the current number of downloads is saved. Over time, a historical download recording can be built.
-  - Both of these sources were used to track Anaconda downloads because the package data for Anaconda does not match the download count on the website. This is due to missing download data. See: https://github.com/anaconda/anaconda-package-data/issues/45
 
 ### Future Data Sources
 In the future, we may expand the source distributions to include:
@@ -33,31 +38,28 @@ In the future, we may expand the source distributions to include:
 ## Workflows
 
 ### Daily Collection
-On a daily basis, this workflow collects download data from PyPI and Anaconda. The data is then published to Google Drive in CSV format (`pypi.csv`). In addition, it computes metrics for the PyPI downloads (see below).
+On a daily basis, this workflow collects download data from PyPI and Anaconda. The data is then published in CSV format (`pypi.csv`). In addition, it computes metrics for the PyPI downloads (see below).
 
 #### Metrics
 This PyPI download metrics are computed along several dimensions:
 
 - **By Month**: The number of downloads per month.
 - **By Version**: The number of downloads per version of the software, as determined by the software maintainers.
 - **By Python Version**: The number of downloads per minor Python version (eg. 3.8).
-- **By Full Python Version**: The number of downloads per full Python version (eg. 3.9.1).
 - **And more!**
 
 ### Daily Summarize
 
-On a daily basis, this workflow summarizes the PyPI download data from `pypi.csv` and calculates downloads for libraries.
-
-The summarized data is uploaded to a GitHub repo:
+On a daily basis, this workflow summarizes the PyPI download data from `pypi.csv` and calculates downloads for libraries. The summarized data is published to a GitHub repo:
 - [Downloads_Summary.xlsx](https://github.com/sdv-dev/sdv-dev.github.io/blob/gatsby-home/assets/Downloads_Summary.xlsx)
 
 #### SDV Calculation
 Installing the main SDV library also installs all the other libraries as dependencies. To calculate SDV downloads, we use an exclusive download methodology:
 
 1. Get download counts for `sdgym` and `sdv`.
-2. Adjust `sdv` downloads by subtracting `sdgym` downloads (since sdgym depends on sdv).
+2. Adjust `sdv` downloads by subtracting `sdgym` downloads (since `sdgym` depends on `sdv`).
 3. Get download counts for direct SDV dependencies: `rdt`, `copulas`, `ctgan`, `deepecho`, `sdmetrics`.
-4. Adjust downloads for each dependency by subtracting the `sdv` download count.
+4. Adjust downloads for each dependency by subtracting the `sdv` download count (since `sdv` has a direct dependency).
 5. Ensure no download count goes negative using `max(0, adjusted_count)` for each library.
 
 This methodology prevents double-counting downloads while providing an accurate representation of SDV usage.
@@ -72,6 +74,9 @@ For more information about the configuration, workflows, and metrics, see the re
 | :floppy_disk: | [COLLECTED DATA](docs/COLLECTED_DATA.md) | Explanation about the data that is being collected. |
 
 
+## Known Issues
+1. The conda package download data for Anaconda does not match the download count shown on the website. This is due to missing download data in the conda package download data. See this: https://github.com/anaconda/anaconda-package-data/issues/45
+
 ---
 
 <div align="center">

README.md

@@ -1,4 +1,3 @@
-output-folder: gdrive://***REMOVED***
 max-days: 7
 projects:
   - sdv

config.yaml

@@ -10,7 +10,7 @@
 import yaml
 
 from pymetrics.anaconda import collect_anaconda_downloads
-from pymetrics.main import collect_downloads
+from pymetrics.main import collect_pypi_downloads
 from pymetrics.summarize import summarize_downloads
 
 LOGGER = logging.getLogger(__name__)
@@ -48,10 +48,10 @@ def _load_config(config_path):
 def _collect_pypi(args):
     config = _load_config(args.config_file)
     projects = args.projects or config['projects']
-    output_folder = args.output_folder or config.get('output-folder', '.')
+    output_folder = args.output_folder
     max_days = args.max_days or config.get('max-days')
 
-    collect_downloads(
+    collect_pypi_downloads(
         projects=projects,
         start_date=args.start_date,
         output_folder=output_folder,
@@ -66,7 +66,7 @@ def _collect_pypi(args):
 def _collect_anaconda(args):
     config = _load_config(args.config_file)
     projects = config['projects']
-    output_folder = args.output_folder or config.get('output-folder', '.')
+    output_folder = args.output_folder
     collect_anaconda_downloads(
         projects=projects,
         output_folder=output_folder,
@@ -80,12 +80,11 @@ def _summarize(args):
     config = _load_config(args.config_file)
     projects = config['projects']
     vendors = config['vendors']
-    output_folder = args.output_folder or config.get('output-folder', '.')
+    output_folder = args.output_folder
 
     summarize_downloads(
         projects=projects,
         vendors=vendors,
-        input_file=args.input_file,
         output_folder=output_folder,
         dry_run=args.dry_run,
         verbose=args.verbose,
@@ -127,7 +126,7 @@ def _get_parser():
     action = parser.add_subparsers(title='action')
     action.required = True
 
-    # collect
+    # collect PyPI
     collect_pypi = action.add_parser(
         'collect-pypi', help='Collect download data from PyPi.', parents=[logging_args]
     )
@@ -137,7 +136,7 @@ def _get_parser():
         '-o',
         '--output-folder',
         type=str,
-        required=False,
+        required=True,
         help=(
             'Path to the folder where data will be stored. It can be a local path or a'
             ' Google Drive folder path in the format gdrive://<folder-id>'
@@ -191,7 +190,7 @@ def _get_parser():
         help='Compute the aggregation metrics and create the corresponding spreadsheets.',
     )
 
-    # collect
+    # summarize
     summarize = action.add_parser(
         'summarize', help='Summarize the downloads data.', parents=[logging_args]
     )
@@ -203,25 +202,18 @@ def _get_parser():
         default='summarize_config.yaml',
         help='Path to the configuration file.',
     )
-    summarize.add_argument(
-        '-i',
-        '--input-file',
-        type=str,
-        default=None,
-        help='Path to the pypi.csv. Default None, which means to use output-folder for pypi.csv',
-    )
     summarize.add_argument(
         '-o',
         '--output-folder',
         type=str,
-        required=False,
+        required=True,
         help=(
-            'Path to the folder where data will be outputted. It can be a local path or a'
+            'Path to the folder where data will be pypi.csv exists. It can be a local path or a'
             ' Google Drive folder path in the format gdrive://<folder-id>'
         ),
     )
 
-    # collect
+    # collect Anaconda
     collect_anaconda = action.add_parser(
         'collect-anaconda', help='Collect download data from Anaconda.', parents=[logging_args]
     )
@@ -237,7 +229,7 @@ def _get_parser():
         '-o',
         '--output-folder',
         type=str,
-        required=False,
+        required=True,
         help=(
             'Path to the folder where data will be outputted. It can be a local path or a'
             ' Google Drive folder path in the format gdrive://<folder-id>'