Add daily workflow to collect anaconda downloads (#21)

* wip

* lint

* fix start date

* add project print

* fix print

* update message

* update to use pyarrow dtypes

* fix string

* update to ubuntu-latest-largeA

* update to ubuntu

* fix engine

* docstring

* use category dtype

* remove pyarrow

* fix ns

* lint

* use pyarrow everywhere

* remove pyarrow dtypes

* add readme instructions

* fix manual

* cleanup

* fix manual

* fix manual

* fix max_days

* fix docs

* wip

* wip

* wip

* fix workflow

* fix workflow

* add message to workflow

* cleanup

* fix repo

* fix slack msg

* fix slack msg

* use extensions

* summarize fix"

* use uv

* fix uv

* use cache

* change token

* add unit tests

* add unit workflow

* add dry-run

* remove unused arg

* fix dry run

* use uv in lint

* add date

* cleanup readme

* Rename daily_collect.yaml to daily_collection.yaml

* Update daily_collection.yaml

* Update daily_summarize.yaml

* Update dryrun.yaml

* Update lint.yaml

* Update manual.yaml

* Update unit.yaml

* wip

* Address feedback 2

* add version parse

* use object dtype

* fix local write

* lint

* Update daily_summarize.yaml

* cleanup

* wip

* exclude pre-releases

* wip

* cleanup

* cleanup

* cleanup

* cleanup

* update workflow

* rename workflow

* fix unit tests

* define cache break

* force reinstall

* remove force install

* lint

* fix summarize config

* cleanup

* fix dry run

* Update dryrun.yaml

* Update dryrun.yaml

* fix dry run

* fix dry run

* fix dry run

* fix write

* remove breakpoint

* Update download_analytics/time_utils.py

Co-authored-by: Roy Wedge <[email protected]>

* fix tz

* fix based on feedback

* fix workflow

---------

Co-authored-by: Roy Wedge <[email protected]>

Author: gsheni

.github/workflows/daily_collection.yaml

@@ -7,35 +7,55 @@ on:
         description: Slack channel to post the error message to if the builds fail.
         required: false
         default: "sdv-alerts-debug"
+      max_days_pypi:
+        description: 'Maximum number of days to collect, starting from today for PyPI.'
+        required: false
+        type: number
+        default: 30
+      max_days_anaconda:
+        description: 'Maximum number of days to collect, starting from today for Anaconda'
+        required: false
+        type: number
+        default: 90
   schedule:
     - cron:  '0 0 * * *'
 
 jobs:
   collect:
     runs-on: ubuntu-latest-large
-    timeout-minutes: 20
+    timeout-minutes: 25
     steps:
     - uses: actions/checkout@v4
     - name: Install uv
       uses: astral-sh/setup-uv@v6
       with:
         enable-cache: true
         activate-environment: true
+        cache-dependency-glob: |
+          **/pyproject.toml
+          **/__main__.py
     - name: Install pip and dependencies
       run: |
         uv pip install -U pip
-        uv pip install -e .
-    - name: Collect Downloads Data
+        uv pip install .
+    - name: Collect PyPI Downloads
       run: |
-        uv run download-analytics collect \
+        uv run download-analytics collect-pypi \
           --verbose \
-          --max-days 30 \
+          --max-days ${{ inputs.max_days_pypi || 30 }} \
           --add-metrics \
           --output-folder gdrive://10QHbqyvptmZX4yhu2Y38YJbVHqINRr0n
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
         BIGQUERY_CREDENTIALS: ${{ secrets.BIGQUERY_CREDENTIALS }}
-
+    - name: Collect Anaconda Downloads
+      run: |
+        uv run download-analytics collect-anaconda \
+          --output-folder gdrive://1UnDYovLkL4gletOF5328BG1X59mSHF-Z \
+          --max-days ${{ inputs.max_days_anaconda || 90 }} \
+          --verbose
+      env:
+        PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
   alert:
     needs: [collect]
     runs-on: ubuntu-latest

.github/workflows/daily_summarize.yaml

@@ -21,6 +21,9 @@ jobs:
       with:
         enable-cache: true
         activate-environment: true
+        cache-dependency-glob: |
+          **/pyproject.toml
+          **/__main__.py
     - name: Install pip and dependencies
       run: |
         uv pip install -U pip

.github/workflows/dryrun.yaml

@@ -10,21 +10,25 @@ concurrency:
   cancel-in-progress: true
 jobs:
   dry_run:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-large
+    timeout-minutes: 25
     steps:
     - uses: actions/checkout@v4
     - name: Install uv
       uses: astral-sh/setup-uv@v6
       with:
         enable-cache: true
         activate-environment: true
+        cache-dependency-glob: |
+          **/pyproject.toml
+          **/__main__.py
     - name: Install pip and dependencies
       run: |
         uv pip install -U pip
         uv pip install .
-    - name: Collect Downloads Data - Dry Run
+    - name: Collect PyPI Downloads - Dry Run
       run: |
-        uv run download-analytics collect \
+        uv run download-analytics collect-pypi \
           --verbose \
           --max-days 30 \
           --add-metrics \
@@ -33,7 +37,16 @@ jobs:
       env:
         PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
         BIGQUERY_CREDENTIALS: ${{ secrets.BIGQUERY_CREDENTIALS }}
-    - name: Run Summarize - Dry Run
+    - name: Collect Anaconda Downloads - Dry Run
+      run: |
+        uv run download-analytics collect-anaconda \
+          --output-folder gdrive://1UnDYovLkL4gletOF5328BG1X59mSHF-Z \
+          --max-days 90 \
+          --verbose \
+          --dry-run
+      env:
+        PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }}
+    - name: Summarize - Dry Run
       run: |
         uv run download-analytics summarize \
           --verbose \

.github/workflows/manual.yaml

@@ -36,7 +36,7 @@ jobs:
         uv pip install .
     - name: Collect Downloads Data
       run: |
-        uv run download-analytics collect \
+        uv run download-analytics collect-pypi \
           --verbose \
           --projects ${{ github.event.inputs.projects }} \
           ${{ github.event.inputs.max_days && '--max-days ' || '' }} \

README.md

@@ -10,16 +10,24 @@ engagement metrics.
 
 ### Data sources
 
-Currently the download data is coming from the following distributions:
+Currently the download data is collected from the following distributions:
 
 * [PyPI](https://pypi.org/): Information about the project downloads from [PyPI](https://pypi.org/)
   obtained from the public Big Query dataset, equivalent to the information shown on
   [pepy.tech](https://pepy.tech).
+* [conda-forge](https://conda-forge.org/): Information about the project downloads from the
+  `conda-forge` channel on `conda`.
+  - The conda package download data provided by Anaconda. It includes package download counts
+    starting from January 2017. More information:
+    - https://github.com/anaconda/anaconda-package-data
+  - The conda package metadata data provided by Anaconda. There is a public API which allows for
+    the retrieval of package information, including current number of downloads.
+    - https://api.anaconda.org/package/{username}/{package_name}
+    - Replace {username} with the Anaconda username (`conda-forge`) and {package_name} with
+    the specific package name (`sdv`).
 
 In the future, we may also expand the source distributions to include:
 
-* [conda-forge](https://conda-forge.org/): Information about the project downloads from the
-  `conda-forge` channel on `conda`.
 * [github](https://github.com/): Information about the project downloads from github releases.
 
 For more information about how to configure and use the software, or about the data that is being

docs/DEVELOPMENT.md

@@ -33,14 +33,14 @@ For development, run `make install-develop` instead.
 ## Command Line Interface
 
 After the installation, a new `download-analytics` command will have been registered inside your
-`virtualenv`. This command can be used in conjunction with the `collect` action to collect
+`virtualenv`. This command can be used in conjunction with the `collect-pypi` action to collect
 downloads data from BigQuery and store the output locally or in Google Drive.
 
 Here is the entire list of arguments that the command line has:
 
 ```bash
-$ download-analytics collect --help
-usage: download-analytics collect [-h] [-v] [-l LOGFILE] [-o OUTPUT_FOLDER] [-a AUTHENTICATION_CREDENTIALS]
+$ download-analytics collect-pypi --help
+usage: download-analytics collect-pypi [-h] [-v] [-l LOGFILE] [-o OUTPUT_FOLDER] [-a AUTHENTICATION_CREDENTIALS]
                                   [-c CONFIG_FILE] [-p [PROJECTS [PROJECTS ...]]] [-s START_DATE]
                                   [-m MAX_DAYS] [-d] [-f] [-M]
 
@@ -73,7 +73,7 @@ and store the downloads data into a Google Drive folder alongside the correspond
 metric spreadsheets would look like this:
 
 ```bash
-$ download-analytics collect --verbose --projects sdv ctgan --start-date 2021-01-01 \
+$ download-analytics collect-pypi --verbose --projects sdv ctgan --start-date 2021-01-01 \
         --add-metrics --output-folder gdrive://10QHbqyvptmZX4yhu2Y38YJbVHqINRr0n
 ```
 

docs/SETUP.md

@@ -31,10 +31,10 @@ if contains the application KEY which should never be made public.
 
 Once the file is created, you can follow these steps:
 
-1. Run the `download-analytics collect` command. If the `settings.yaml` file has been properly
+1. Run the `download-analytics collect-pypi` command. If the `settings.yaml` file has been properly
    created, this will **open a new tab on your web browser**, where you need to authenticate.
 
-   | ![pydrive-collect](imgs/pydrive-collect.png "Run the `download-analytics collect` Command") |
+   | ![pydrive-collect](imgs/pydrive-collect.png "Run the `download-analytics collect-pypi` Command") |
    | - |
 
 2. Click on the Google account which you which to authenticate with. Notice that the account that
@@ -67,7 +67,7 @@ be provided to you by a privileged admin.
 Once you have this JSON file, you have two options:
 
 1. Pass the path to the authentication file with the `-a` or `--authentication-credentials`
-   argument to the `download-analytics collect` command.
+   argument to the `download-analytics collect-pypi` command.
 
    | ![bigquery-a](imgs/bigquery-a.png "Pass the credentials on command line") |
    | - |

download_analytics/__main__.py

@@ -9,6 +9,7 @@
 
 import yaml
 
+from download_analytics.anaconda import collect_anaconda_downloads
 from download_analytics.main import collect_downloads
 from download_analytics.summarize import summarize_downloads
 
@@ -44,7 +45,7 @@ def _load_config(config_path):
     return config
 
 
-def _collect(args):
+def _collect_pypi(args):
     config = _load_config(args.config_file)
     projects = args.projects or config['projects']
     output_folder = args.output_folder or config.get('output-folder', '.')
@@ -62,6 +63,19 @@ def _collect(args):
     )
 
 
+def _collect_anaconda(args):
+    config = _load_config(args.config_file)
+    projects = config['projects']
+    output_folder = args.output_folder or config.get('output-folder', '.')
+    collect_anaconda_downloads(
+        projects=projects,
+        output_folder=output_folder,
+        max_days=args.max_days,
+        dry_run=args.dry_run,
+        verbose=args.verbose,
+    )
+
+
 def _summarize(args):
     config = _load_config(args.config_file)
     projects = config['projects']
@@ -98,7 +112,12 @@ def _get_parser():
     logging_args.add_argument(
         '-l', '--logfile', help='If given, file where the logs will be written.'
     )
-
+    logging_args.add_argument(
+        '-d',
+        '--dry-run',
+        action='store_true',
+        help='Do not upload the results. Just calculate them.',
+    )
     parser = argparse.ArgumentParser(
         prog='download-analytics',
         description='Download Analytics Command Line Interface',
@@ -109,10 +128,12 @@ def _get_parser():
     action.required = True
 
     # collect
-    collect = action.add_parser('collect', help='Collect downloads data.', parents=[logging_args])
-    collect.set_defaults(action=_collect)
+    collect_pypi = action.add_parser(
+        'collect-pypi', help='Collect download data from PyPi.', parents=[logging_args]
+    )
+    collect_pypi.set_defaults(action=_collect_pypi)
 
-    collect.add_argument(
+    collect_pypi.add_argument(
         '-o',
         '--output-folder',
         type=str,
@@ -122,54 +143,48 @@ def _get_parser():
             ' Google Drive folder path in the format gdrive://<folder-id>'
         ),
     )
-    collect.add_argument(
+    collect_pypi.add_argument(
         '-a',
         '--authentication-credentials',
         type=str,
         required=False,
         help='Path to the GCP (BigQuery) credentials file to use.',
     )
-    collect.add_argument(
+    collect_pypi.add_argument(
         '-c',
         '--config-file',
         type=str,
         default='config.yaml',
         help='Path to the configuration file.',
     )
-    collect.add_argument(
+    collect_pypi.add_argument(
         '-p',
         '--projects',
         nargs='*',
         help='List of projects to collect. If not given use the configured ones.',
         default=None,
     )
-    collect.add_argument(
+    collect_pypi.add_argument(
         '-s',
         '--start-date',
         type=_valid_date,
         required=False,
         help='Date from which to start pulling data.',
     )
-    collect.add_argument(
+    collect_pypi.add_argument(
         '-m',
         '--max-days',
         type=int,
         required=False,
         help='Max days of data to pull if start-date is not given.',
     )
-    collect.add_argument(
-        '-d',
-        '--dry-run',
-        action='store_true',
-        help='Do not run the actual query, only simulate it.',
-    )
-    collect.add_argument(
+    collect_pypi.add_argument(
         '-f',
         '--force',
         action='store_true',
         help='Force the download even if the data already exists or there is a gap',
     )
-    collect.add_argument(
+    collect_pypi.add_argument(
         '-M',
         '--add-metrics',
         action='store_true',
@@ -205,11 +220,36 @@ def _get_parser():
             ' Google Drive folder path in the format gdrive://<folder-id>'
         ),
     )
-    summarize.add_argument(
-        '-d',
-        '--dry-run',
-        action='store_true',
-        help='Do not upload the summary results. Just calculate them.',
+
+    # collect
+    collect_anaconda = action.add_parser(
+        'collect-anaconda', help='Collect download data from Anaconda.', parents=[logging_args]
+    )
+    collect_anaconda.set_defaults(action=_collect_anaconda)
+    collect_anaconda.add_argument(
+        '-c',
+        '--config-file',
+        type=str,
+        default='config.yaml',
+        help='Path to the configuration file.',
+    )
+    collect_anaconda.add_argument(
+        '-o',
+        '--output-folder',
+        type=str,
+        required=False,
+        help=(
+            'Path to the folder where data will be outputted. It can be a local path or a'
+            ' Google Drive folder path in the format gdrive://<folder-id>'
+        ),
+    )
+    collect_anaconda.add_argument(
+        '-m',
+        '--max-days',
+        type=int,
+        required=False,
+        default=90,
+        help='Max days of data to pull.',
     )
     return parser