Update README and use secrets for folder outputs (#29)

* renamed repo

* renamed repo

* renamed repo

* rename folder

* wip

* fix dry run

* fix output folder

* fix output folder

* fix dry run

* force input/output folders

* lint

* speed up dry run

* lint

* fix secret

* fix secret

* fix anaconda

* fix input

* fix readme

* fix input file

* update docstring

* update docstring

Author: gsheni

.github/workflows/daily_collection.yaml

@@ -22,6 +22,7 @@ on:
 
 jobs:
   collect:
+    environment: prod
     runs-on: ubuntu-latest-large
     timeout-minutes: 25
     steps:
@@ -34,28 +35,27 @@ jobs:
         cache-dependency-glob: |
           **/pyproject.toml
           **/__main__.py
-    - name: Install pip and dependencies
+    - name: Install dependencies
       run: |
-        uv pip install -U pip
         uv pip install .
     - name: Collect PyPI Downloads
       run: |
         uv run pymetrics collect-pypi \
-          --verbose \
           --max-days ${{ inputs.max_days_pypi || 30 }} \
           --add-metrics \
-          --output-folder gdrive://10QHbqyvptmZX4yhu2Y38YJbVHqINRr0n
+          --output-folder ${{ secrets.PYPI_OUTPUT_FOLDER }}
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
         BIGQUERY_CREDENTIALS: ${{ secrets.BIGQUERY_CREDENTIALS }}
+        PYPI_OUTPUT_FOLDER: ${{ secrets.PYPI_OUTPUT_FOLDER }}
     - name: Collect Anaconda Downloads
       run: |
         uv run pymetrics collect-anaconda \
-          --output-folder gdrive://1UnDYovLkL4gletOF5328BG1X59mSHF-Z \
-          --max-days ${{ inputs.max_days_anaconda || 90 }} \
-          --verbose
+          --output-folder ${{ secrets.ANACONDA_OUTPUT_FOLDER }} \
+          --max-days ${{ inputs.max_days_anaconda || 90 }}
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
+        ANACONDA_OUTPUT_FOLDER: ${{ secrets.ANACONDA_OUTPUT_FOLDER }}
   alert:
     needs: [collect]
     runs-on: ubuntu-latest
@@ -69,9 +69,12 @@ jobs:
         activate-environment: true
     - name: Install pip and dependencies
       run: |
-        uv pip install -U pip
         uv pip install -e .[dev]
     - name: Slack alert if failure
-      run: uv run python -m pymetrics.slack_utils -r ${{ github.run_id }} -c ${{ github.event.inputs.slack_channel || 'sdv-alerts' }}
+      run: |
+        uv run python -m pymetrics.slack_utils \
+        -r ${{ github.run_id }} \
+        -c ${{ github.event.inputs.slack_channel || 'sdv-alerts' }} \
+        -m 'Daily Collection PyMetrics failed :fire: :dumpster-fire: :fire:'
       env:
         SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}

.github/workflows/daily_summarize.yaml

@@ -12,6 +12,7 @@ on:
 
 jobs:
   summarize:
+    environment: prod
     runs-on: ubuntu-latest-large
     timeout-minutes: 10
     steps:
@@ -25,15 +26,14 @@ jobs:
           **/pyproject.toml
           **/__main__.py
     - name: Install pip and dependencies
-      run: |
-        uv pip install -U pip
-        uv pip install .
+      run: uv pip install .
     - name: Run Summarize
       run: |
         uv run pymetrics summarize \
-          --output-folder gdrive://10QHbqyvptmZX4yhu2Y38YJbVHqINRr0n
+          --output-folder ${{ secrets.PYPI_OUTPUT_FOLDER }}
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
+        PYPI_OUTPUT_FOLDER: ${{ secrets.PYPI_OUTPUT_FOLDER }}
     - uses: actions/checkout@v4
       with:
         repository: sdv-dev/sdv-dev.github.io
@@ -63,13 +63,12 @@ jobs:
         activate-environment: true
     - name: Install pip and dependencies
       run: |
-        uv pip install -U pip
         uv pip install .[dev]
     - name: Slack alert if failure
       run: |
         uv run python -m pymetrics.slack_utils \
         -r ${{ github.run_id }} \
         -c ${{ github.event.inputs.slack_channel || 'sdv-alerts' }} \
-        -m 'Summarize Analytics build failed :fire: :dumpster-fire: :fire:'
+        -m 'Daily Summarize PyMetrics failed :fire: :dumpster-fire: :fire:'
       env:
         SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}

.github/workflows/dryrun.yaml

@@ -10,6 +10,7 @@ concurrency:
   cancel-in-progress: true
 jobs:
   dry_run:
+    environment: stage
     runs-on: ubuntu-latest-large
     timeout-minutes: 25
     steps:
@@ -24,33 +25,32 @@ jobs:
           **/__main__.py
     - name: Install pip and dependencies
       run: |
-        uv pip install -U pip
         uv pip install .
     - name: Collect PyPI Downloads - Dry Run
       run: |
         uv run pymetrics collect-pypi \
-          --verbose \
           --max-days 30 \
           --add-metrics \
-          --output-folder gdrive://10QHbqyvptmZX4yhu2Y38YJbVHqINRr0n \
+          --output-folder ${{ secrets.PYPI_OUTPUT_FOLDER }} \
           --dry-run
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
         BIGQUERY_CREDENTIALS: ${{ secrets.BIGQUERY_CREDENTIALS }}
+        PYPI_OUTPUT_FOLDER: ${{ secrets.PYPI_OUTPUT_FOLDER }}
     - name: Collect Anaconda Downloads - Dry Run
       run: |
         uv run pymetrics collect-anaconda \
-          --output-folder gdrive://1UnDYovLkL4gletOF5328BG1X59mSHF-Z \
           --max-days 90 \
-          --verbose \
+          --output-folder ${{ secrets.ANACONDA_OUTPUT_FOLDER }} \
           --dry-run
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
+        ANACONDA_OUTPUT_FOLDER: ${{ secrets.ANACONDA_OUTPUT_FOLDER }}
     - name: Summarize - Dry Run
       run: |
         uv run pymetrics summarize \
-          --verbose \
-          --output-folder gdrive://10QHbqyvptmZX4yhu2Y38YJbVHqINRr0n \
+          --output-folder ${{ secrets.PYPI_OUTPUT_FOLDER }} \
           --dry-run
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
+        PYPI_OUTPUT_FOLDER: ${{ secrets.PYPI_OUTPUT_FOLDER }}

.github/workflows/lint.yaml

@@ -20,7 +20,6 @@ jobs:
         activate-environment: true
     - name: Install pip and dependencies
       run: |
-        uv pip install -U pip
         uv pip install .[dev]
     - name: Run lint checks
       run: uv run invoke lint

.github/workflows/manual.yaml

@@ -32,7 +32,6 @@ jobs:
         activate-environment: true
     - name: Install pip and dependencies
       run: |
-        uv pip install -U pip
         uv pip install .
     - name: Collect Downloads Data
       run: |

.github/workflows/unit.yaml

@@ -26,7 +26,6 @@ jobs:
           activate-environment: true
       - name: Install pip and dependencies
         run: |
-          uv pip install -U pip
           uv pip install -e .[test,dev]
       - name: Run summarize
         run: |

.gitignore

@@ -3,6 +3,7 @@ bigquery_creds.json
 client_secrets.json
 credentials.json
 sdv-dev.github.io/*
+uv.lock
 
 notebooks
 *.xlsx

MANIFEST.in

@@ -1,5 +1,11 @@
-# PyMetrics
+<div align="center">
+<br/>
+<p align="center">
+    <i>This repository is part of <a href="https://sdv.dev">The Synthetic Data Vault Project</a>, a project from <a href="https://datacebo.com">DataCebo</a>.</i>
+</p>
+<div align="left">
 
+# PyMetrics
 The PyMetrics project allows you to extract download metrics for Python libraries published on [PyPI](https://pypi.org/) and [Anaconda](https://www.anaconda.com/).
 
 The DataCebo team uses these scripts to report download counts for the libraries in the [SDV ecosystem](https://sdv.dev/) and other libraries.
@@ -13,8 +19,8 @@ engagement metrics.
 Currently, the download data is collected from the following distributions:
 * [PyPI](https://pypi.org/): Information about the project downloads from [PyPI](https://pypi.org/)
   obtained from the public BigQuery dataset, equivalent to the information shown on
-  [pepy.tech](https://pepy.tech) and [ClickPy](https://clickpy.clickhouse.com/)
-  - More information about the BigQuery dataset can be found on the [official PyPI documentation](https://packaging.python.org/en/latest/guides/analyzing-pypi-package-downloads/)
+  [pepy.tech](https://pepy.tech), [ClickPy](https://clickpy.clickhouse.com/) or [pypistats](https://pypistats.org/).
+  - More information about the BigQuery dataset can be found on the [official PyPI documentation](https://packaging.python.org/en/latest/guides/analyzing-pypi-package-downloads/).
 
 * [Anaconda](https://www.anaconda.com/): Information about conda package downloads for default and select Anaconda channels.
   - The conda package download data is provided by Anaconda, Inc. It includes package download counts
@@ -24,7 +30,6 @@ Currently, the download data is collected from the following distributions:
       - Replace `{username}` with the Anaconda channel (`conda-forge`)
       - Replace `{package_name}` with the specific package (`sdv`) in the Anaconda channel
     - For each file returned by the API endpoint, the current number of downloads is saved. Over time, a historical download recording can be built.
-  - Both of these sources were used to track Anaconda downloads because the package data for Anaconda does not match the download count on the website. This is due to missing download data. See: https://github.com/anaconda/anaconda-package-data/issues/45
 
 ### Future Data Sources
 In the future, we may expand the source distributions to include:
@@ -33,31 +38,28 @@ In the future, we may expand the source distributions to include:
 ## Workflows
 
 ### Daily Collection
-On a daily basis, this workflow collects download data from PyPI and Anaconda. The data is then published to Google Drive in CSV format (`pypi.csv`). In addition, it computes metrics for the PyPI downloads (see below).
+On a daily basis, this workflow collects download data from PyPI and Anaconda. The data is then published in CSV format (`pypi.csv`). In addition, it computes metrics for the PyPI downloads (see below).
 
 #### Metrics
 This PyPI download metrics are computed along several dimensions:
 
 - **By Month**: The number of downloads per month.
 - **By Version**: The number of downloads per version of the software, as determined by the software maintainers.
 - **By Python Version**: The number of downloads per minor Python version (eg. 3.8).
-- **By Full Python Version**: The number of downloads per full Python version (eg. 3.9.1).
 - **And more!**
 
 ### Daily Summarize
 
-On a daily basis, this workflow summarizes the PyPI download data from `pypi.csv` and calculates downloads for libraries.
-
-The summarized data is uploaded to a GitHub repo:
+On a daily basis, this workflow summarizes the PyPI download data from `pypi.csv` and calculates downloads for libraries. The summarized data is published to a GitHub repo:
 - [Downloads_Summary.xlsx](https://github.com/sdv-dev/sdv-dev.github.io/blob/gatsby-home/assets/Downloads_Summary.xlsx)
 
 #### SDV Calculation
 Installing the main SDV library also installs all the other libraries as dependencies. To calculate SDV downloads, we use an exclusive download methodology:
 
 1. Get download counts for `sdgym` and `sdv`.
-2. Adjust `sdv` downloads by subtracting `sdgym` downloads (since sdgym depends on sdv).
+2. Adjust `sdv` downloads by subtracting `sdgym` downloads (since `sdgym` depends on `sdv`).
 3. Get download counts for direct SDV dependencies: `rdt`, `copulas`, `ctgan`, `deepecho`, `sdmetrics`.
-4. Adjust downloads for each dependency by subtracting the `sdv` download count.
+4. Adjust downloads for each dependency by subtracting the `sdv` download count (since `sdv` has a direct dependency).
 5. Ensure no download count goes negative using `max(0, adjusted_count)` for each library.
 
 This methodology prevents double-counting downloads while providing an accurate representation of SDV usage.
@@ -72,6 +74,9 @@ For more information about the configuration, workflows, and metrics, see the re
 | :floppy_disk: | [COLLECTED DATA](docs/COLLECTED_DATA.md) | Explanation about the data that is being collected. |
 
 
+## Known Issues
+1. The conda package download data for Anaconda does not match the download count shown on the website. This is due to missing download data in the conda package download data. See this: https://github.com/anaconda/anaconda-package-data/issues/45
+
 ---
 
 <div align="center">

README.md

@@ -1,4 +1,3 @@
-output-folder: gdrive://10QHbqyvptmZX4yhu2Y38YJbVHqINRr0n
 max-days: 7
 projects:
   - sdv