feat: various changes to make building of catalog working again (#30)

* fix: attempt to update GH actions workflow for building catalog

* feat: added some basic instructions based on catalog landing page

* fix: try to fix env availability

* Add changes

* fix: added pull step in attempt to fix failureto push to non-updated remote

* Add changes

* fix: added pull step attempt 2

* fix: added pull step attempt 3

* Add changes

* Add changes

* fix: restored original runner chunks (10 x 100)

* fix: minor improvements to doc, added badges

* fix: issue with autostashing intermittent changes

* Add changes

* Add changes

* fix: added problematic repos flagged by GH push protection

* Add changes

* Add changes

* Add changes

* Add changes

* Add changes

* Add changes

* Add changes

* Add changes

* fix: add filter argument to untar as suggested by warning

* fix: prevent possible exposure of tokens from snakemake --lint

* fix: added parameter to search only for repos that had recent updates

* fix: full update of all repos

* Add changes

* Add changes

* Add changes

* Add changes

* Add changes

* Update scripts/generate-catalog.py

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

* remove duplicate import

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Johannes Köster <johannes.koester@uni-due.de>
Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>

Author: 4 people

.github/workflows/generate.yml

@@ -2,10 +2,11 @@ name: Generate catalog
 
 on:
   schedule:
-   - cron: 0 5 * * *
+    - cron: 0 5 * * 1
   push:
     branches:
       - main
+      - dev
 
 jobs:
   generate-catalog:
@@ -16,25 +17,30 @@ jobs:
       max-parallel: 1
     steps:
       - uses: actions/checkout@v4
-        with:
-          ref: main
-      
+
       - name: deployment
-        uses: mamba-org/provision-with-micromamba@main
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: environment.yml
+
+      - name: Pull latest changes
+        run: |
+          git pull --rebase origin ${{ github.ref }}
 
       - name: generate-catalog
         shell: bash -l {0}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           OFFSET: ${{ matrix.offset }}
+          LATEST_COMMIT: 7
         run: |
           python scripts/generate-catalog.py
 
       - name: Commit files
         run: |
           git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git config --local user.name "github-actions[bot]"
-          git commit -m "Add changes" -a
+          git commit -m "Add changes" -a || echo "No changes to commit"
 
       - name: Push changes
         uses: ad-m/github-push-action@master
@@ -51,13 +57,17 @@ jobs:
       max-parallel: 1
     steps:
       - uses: actions/checkout@v4
-        with:
-          ref: main
-      
+
       - name: deployment
-        uses: mamba-org/provision-with-micromamba@main
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: environment.yml
 
-      - name: generate-catalog
+      - name: Pull latest changes
+        run: |
+          git pull --rebase origin ${{ github.ref }}
+
+      - name: cleanup-catalog
         shell: bash -l {0}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -69,10 +79,10 @@ jobs:
         run: |
           git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git config --local user.name "github-actions[bot]"
-          git commit -m "Add changes" -a
+          git commit -m "Add changes" -a || echo "No changes to commit"
 
       - name: Push changes
         uses: ad-m/github-push-action@master
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          branch: ${{ github.ref }}
+          branch: ${{ github.ref }}

.github/workflows/test-repo.yml

@@ -54,7 +54,7 @@ jobs:
             Results: https://github.com/snakemake/snakemake-workflow-catalog/actions/workflows/test-repo.yml
       - uses: actions/checkout@v1
       - name: deployment
-        uses: mamba-org/provision-with-micromamba@main
+        uses: mamba-org/setup-micromamba@v2
       - name: generate-catalog
         shell: bash -l {0}
         env:

README.md

@@ -1,2 +1,80 @@
 # snakemake-workflow-catalog
+
+[![Generate catalog](https://github.com/snakemake/snakemake-workflow-catalog/actions/workflows/generate.yml/badge.svg)](https://github.com/snakemake/snakemake-workflow-catalog/actions/workflows/generate.yml)
+[![pages-build-deployment](https://github.com/snakemake/snakemake-workflow-catalog/actions/workflows/pages/pages-build-deployment/badge.svg)](https://github.com/snakemake/snakemake-workflow-catalog/actions/workflows/pages/pages-build-deployment)
+![GitHub last commit](https://img.shields.io/github/last-commit/snakemake/snakemake-workflow-catalog?label=latest%20update)
+![GitHub Issues or Pull Requests](https://img.shields.io/github/issues/snakemake/snakemake-workflow-catalog)
+
 A statically generated catalog of available Snakemake workflows
+
+This repository serves as a centralized collection of workflows designed to facilitate reproducible and scalable data analyses using the [**Snakemake**](https://snakemake.github.io/) workflow management system.
+
+## Purpose
+
+The Snakemake Workflow Catalog aims to provide a regularly updated list of high-quality workflows that can be easily reused and adapted for various data analysis tasks. By leveraging the power of [**Snakemake**](https://snakemake.github.io/), these workflows promote:
+
+- Reproducibility: Snakemake workflows produce consistent results, making it easier to share and validate scientific findings.
+- Scalability: Snakemake workflows can be executed on various computing environments, from local machines to high-performance computing clusters and cloud services.
+- Modularity: Snakemake workflows are structured to allow easy customization and extension, enabling users to adapt them to their specific needs.
+
+## Workflows
+
+Workflows are automatically added to the Workflow Catalog. This is done by regularly searching Github repositories for matching workflow structures. The catalog includes workflows based on the following criteria.
+
+### All workflows
+
+- The workflow is contained in a public Github repository.
+- The repository has a `README.md` file, containing the words "snakemake" and "workflow" (case insensitive).
+- The repository contains a workflow definition named either `Snakefile` or `workflow/Snakefile`.
+- If the repository contains a folder `rules` or `workflow/rules`, that folder must at least contain one file ending on `.smk`.
+- The repository is small enough to be cloned into a [Github Actions](https://docs.github.com/en/actions/about-github-actions/understanding-github-actions) job (very large files should be handled via [Git LFS](https://docs.github.com/en/repositories/working-with-files/managing-large-files), so that they can be stripped out during cloning).
+- The repository is not blacklisted here.
+
+### Standardized usage workflows
+
+In order to additionally appear in the "standardized usage" area, repositories additionally have to:
+
+- have their main workflow definition named `workflow/Snakefile` (unlike for plain inclusion (see above), which also allows just `Snakefile` in the root of the repository),
+- provide configuration instructions under `config/README.md`
+- contain a `YAML` file `.snakemake-workflow-catalog.yml` in their root directory, which configures the usage instructions displayed by this workflow catalog.
+
+Typical content of the `.snakemake-workflow-catalog.yml` file:
+
+```bash
+usage:
+  mandatory-flags: # optional definition of additional flags
+    desc: # describe your flags here in a few sentences (they will be inserted below the example commands)
+    flags: # put your flags here
+  software-stack-deployment: # definition of software deployment method (at least one of conda, singularity, or singularity+conda)
+    conda: true # whether pipeline works with --use-conda
+    singularity: true # whether pipeline works with --use-singularity
+    singularity+conda: true # whether pipeline works with --use-singularity --use-conda
+    report: true # add this to confirm that the workflow allows to use 'snakemake --report report.zip' to generate a report containing all results and explanations
+```
+
+Once included in the standardized usage area you can link directly to the usage instructions for your repository via the URL `https://snakemake.github.io/snakemake-workflow-catalog?usage=<owner>/<repo>`.
+
+### Release handling
+
+If your workflow provides Github releases, the catalog will always just scrape the latest non-preview release. Hence, in order to update your workflow's records here, you need to release a new version on Github.
+
+## Contributing
+
+Contributions to the Snakemake Workflow Catalog are welcome!
+Ideas can be discussed on the [catalog's Issues page](https://github.com/snakemake/snakemake-workflow-catalog/issues) first, and contributions made through Github Pull Requests.
+
+## Using workflows from the catalog
+
+To get started with a workflow from the catalog:
+
+1. Clone the repository or download the specific workflow directory.
+2. Review the documentation provided with the workflow to understand its requirements and usage.
+3. Configure the workflow by editing the `config.yml` files as needed.
+4. Execute the workflow using Snakemake.
+
+For more detailed instructions, please refer to the documentation within each workflow directory.
+
+## License
+
+The Snakemake Workflow Catalog is open-source and available under the MIT License.
+For more information and to explore the available workflows, visit https://snakemake.github.io/snakemake-workflow-catalog/.

blacklist.txt

@@ -1,3 +1,6 @@
 snakemake/snakemake-wrappers
 snakemake/snakemake
-snakemake/snakemake-workflow-catalog
+snakemake/snakemake-workflow-catalog
+tdayris/fair_bowtie2_mapping
+tdayris/fair_fastqc_multiqc
+GiulioCentorame/FADS-by-breastfeeding

data.js

@@ -3,10 +3,10 @@ channels:
   - conda-forge
   - bioconda
 dependencies:
-  - jinja2
-  - pygithub
-  - gitpython
-  - snakemake
-  - snakefmt
-  - python >=3.9
-  - ratelimit
+  - jinja2=3.1.4
+  - pygithub=2.5.0
+  - gitpython=3.1.43
+  - snakemake=8.25.5
+  - snakefmt=0.10.2
+  - python=3.12.8
+  - ratelimit=2.2.1

environment.yml

@@ -1,22 +1,16 @@
 import logging
-import tempfile
 import subprocess as sp
 import os
-from pathlib import Path
 import json
 import calendar
 import time
-import urllib
-import tarfile
 
 from ratelimit import limits, sleep_and_retry
 from jinja2 import Environment
 from github import Github
 from github.ContentFile import ContentFile
 from github.GithubException import UnknownObjectException, RateLimitExceededException
-import git
 from jinja2 import Environment, FileSystemLoader, select_autoescape
-import yaml
 
 logging.basicConfig(level=logging.INFO)
 
@@ -48,6 +42,7 @@
     .split()[-1]
 )
 
+
 def rate_limit_wait(api_type):
     curr_timestamp = calendar.timegm(time.gmtime())
     reset_timestamp = calendar.timegm(get_rate_limit(api_type).reset.timetuple())

scripts/common.py

@@ -3,22 +3,32 @@
 import subprocess as sp
 import os
 from pathlib import Path
-import json
 import time
 import urllib
 import tarfile
+import re
+from datetime import timedelta, datetime
 
-from jinja2 import Environment
 import git
 from jinja2 import Environment, FileSystemLoader, select_autoescape
 import yaml
 
-from common import store_data, check_repo_exists, call_rate_limit_aware, g, previous_repos, previous_skips, blacklist, snakefmt_version, offset
+from common import (
+    store_data,
+    call_rate_limit_aware,
+    g,
+    previous_repos,
+    previous_skips,
+    blacklist,
+    snakefmt_version,
+    offset,
+)
 
 logging.basicConfig(level=logging.INFO)
 
 test_repo = os.environ.get("TEST_REPO")
-offset = int(offset / 100 * 1000)
+latest_commit = int(os.environ.get("LATEST_COMMIT"))
+offset = int(offset * 10)
 
 env = Environment(
     autoescape=select_autoescape(["html"]), loader=FileSystemLoader("templates")
@@ -96,8 +106,11 @@ def __init__(
     total_count = 1
     offset = 0
 else:
+    date_threshold = datetime.today() - timedelta(latest_commit)
+    date_threshold = datetime.strftime(date_threshold, "%Y-%m-%d")
     repo_search = g.search_repositories(
-        "snakemake workflow in:readme archived:false", sort="updated"
+        f"snakemake workflow in:readme archived:false pushed:>={date_threshold}",
+        sort="updated",
     )
     time.sleep(5)
     total_count = call_rate_limit_aware(
@@ -170,7 +183,7 @@ def __init__(
                 fileobj=urllib.request.urlopen(tarball_url), mode="r|gz"
             )
             root_dir = get_tarfile().getmembers()[0].name
-            get_tarfile().extractall(path=tmp)
+            get_tarfile().extractall(path=tmp, filter="tar")
             tmp /= root_dir
         else:
             # no latest release, clone main branch
@@ -195,7 +208,8 @@ def __init__(
 
         if rules.exists() and rules.is_dir():
             if not any(
-                rule_file.suffix == ".smk" for rule_file in rules.iterdir()
+                rule_file.suffix == ".smk"
+                for rule_file in rules.iterdir()
                 if rule_file.is_file()
             ):
                 log_skip("rule modules are not using .smk extension")
@@ -233,6 +247,7 @@ def __init__(
             )
         except sp.CalledProcessError as e:
             linting = e.stderr.decode()
+            linting = re.sub("gh[pousr]\\_[a-zA-Z0-9_]{36}@?", "", linting)
             if test_repo is not None:
                 logging.error(linting)
 
@@ -252,15 +267,22 @@ def __init__(
             if test_repo is not None:
                 logging.error(formatting)
 
-    topics = call_rate_limit_aware(
-        repo.get_topics
-    )
+    topics = call_rate_limit_aware(repo.get_topics)
 
     if config_readme is not None:
         config_readme = call_rate_limit_aware(lambda: g.render_markdown(config_readme))
 
     repos.append(
-        Repo(repo, linting, formatting, config_readme, settings, release, updated_at, topics).__dict__
+        Repo(
+            repo,
+            linting,
+            formatting,
+            config_readme,
+            settings,
+            release,
+            updated_at,
+            topics,
+        ).__dict__
     )
 
 if test_repo is None:
@@ -270,7 +292,9 @@ def __init__(
 
     def add_old(old_repos, current_repos):
         visited = set(repo["full_name"] for repo in current_repos)
-        current_repos.extend(repo for repo_name, repo in old_repos.items() if repo_name not in visited)
+        current_repos.extend(
+            repo for repo_name, repo in old_repos.items() if repo_name not in visited
+        )
 
     logging.info("Adding all old repos not covered by the current query.")
     add_old(previous_repos, repos)