Merge pull request #2 from sourmash-bio/upd_aug2

WIP: working on stuff, round2

Author: ctb

.github/workflows/gh-pages.yml

@@ -0,0 +1,22 @@
+name: build and deploy mkdocs to github pages
+
+permissions:
+  pages: write
+  contents: write
+
+on:
+  push:
+    branches:
+      - main
+      - upd_aug2
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: pip install mkdocs-material
+      - run: mkdocs gh-deploy --force

.gitignore

@@ -1,4 +1,5 @@
 *~
 __pycache__
-outputs
 .snakemake
+site/
+*.pickle

Makefile

@@ -1,6 +1,18 @@
-all: Snakefile
+.PHONY: build format update_sourmash
+
+all: Snakefile build preview
+
+build:
 	snakemake --delete-all-output
-	snakemake -p
+	snakemake -p -j 1
+
+preview: build
+	rm -fr preview/generated
+	mkdir -p preview/generated/
+	cp outputs/md/*.md preview/generated/
+
+update_ctb_sourmash: build
+	cp -r outputs/md/ ~/dev/sourmash/doc/databases-md/
 
 format:
 	black scripts

README.md

@@ -1 +1,37 @@
 # 2025-sourmash-databases-doc-template
+
+This repo uses Jinja2 templating to automatically produce Markdown
+files describing the available sourmash databases. It also produces
+automated scripts that check that the relevant files are available for
+download.
+
+## Basic instructions
+
+Run `make`. Then look at [outputs/md](outputs/md) to see output markdown.
+
+## Previewing formatting with mkdocs
+
+Run `mkdocs serve` to see the generated files.
+
+## Updating the database list in the sourmash docs
+
+Copy all the files in `outputs/md/` into the sourmash repo under `doc/databases-md/`.
+
+## Adding databases
+
+Edit [scripts/databases.py](scripts/databases.py) to add databases.
+
+You'll also need to:
+* add the new db to 'collections' list at top of `scripts/make-md.py`.
+* update `mkdocs.yml` if you want to preview the new db;
+* update `doc/databases.md` in sourmash to include a direct link to the generated file.
+
+## Validating database links
+
+Run `outputs/scripts/check-urls.py` to check that all the database and
+taxonomy URLs are valid.
+
+## Notes on database naming conventions:
+
+* all NCBI databases will be date-stamped
+* all GTDB databases will be version-stamped

Snakefile

@@ -9,6 +9,9 @@ templates = [
     Templates_To_Output('gtdb220',
                         'complete',
                         'outputs/md/gtdb220.md'),
+    Templates_To_Output('gtdb226',
+                        'complete',
+                        'outputs/md/gtdb226.md'),
     Templates_To_Output('ncbi_viruses_2025_01',
                         'complete',
                         'outputs/md/ncbi_viruses_2025_01.md'),
@@ -17,6 +20,7 @@ templates = [
                         'outputs/md/ncbi_euks_2025_01.md'),
 ]
 
+# retrieve path from a Templates_To_Output object
 def get_template_path(w):
     print('gtp', w)
     for t in templates:
@@ -25,6 +29,7 @@ def get_template_path(w):
             print('get_template_path', t, x)
             return x
 
+# retrieve name from a Templates_To_Output object
 def get_template_name(w):
     for t in templates:
         if t.output_md == f'outputs/md/{w.db}.md':
@@ -37,19 +42,22 @@ def get_template_name(w):
 rule default:
     input:
         [ t.output_md for t in templates ],
+        "outputs/scripts/check-urls.py",
+        "outputs/md/databases.md",
 
 
 rule make_db_descr:
     input:
         script='scripts/make-list.py',
+        dbfoo='scripts/databases.py',
     output:
         pickle='outputs/collections.pickle',
     shell: """
         {input.script} --save-pickle {output.pickle}
     """
 
 
-rule make_gtdb:
+rule make_db_md:
     input:
         script='scripts/make-md.py',
         pickle='outputs/collections.pickle',
@@ -62,3 +70,31 @@ rule make_gtdb:
         {input.script} {input.pickle} {params.template} \
             --set-collection {wildcards.db} -o {output}
     """
+
+rule make_check_script:
+    input:
+        script="scripts/make-file.py",
+        pickle='outputs/collections.pickle',
+        template="templates/check-urls.py",
+    output:
+        "outputs/scripts/check-urls.py",
+    params:
+        template_name="check-urls.py"
+    shell: """
+        {input.script} {input.pickle} {params.template_name} -o {output}
+        chmod +x {output}
+    """
+
+rule make_databases:
+    input:
+        script="scripts/make-file.py",
+        pickle='outputs/collections.pickle',
+        template="templates/databases.md",
+    output:
+        "outputs/md/databases.md",
+    params:
+        template_name="databases.md",
+    shell: """
+        {input.script} {input.pickle} {params.template_name} -o {output}
+        chmod +x {output}
+    """

mkdocs.yml

@@ -0,0 +1,36 @@
+site_name: database-template-preview
+docs_dir: ./preview/
+
+nav:
+- Home: index.md
+- Databases: generated/databases.md
+- GTDB RS220: generated/gtdb220.md
+- GTDB RS226: generated/gtdb226.md
+- NCBI eukaryotes (Jan 2025): generated/ncbi_euks_2025_01.md
+- NCBI viruses (Jan 2025): generated/ncbi_viruses_2025_01.md
+
+markdown_extensions:
+  - toc:
+      permalink: true
+  - def_list
+  - attr_list
+  - admonition
+
+plugins:
+  - search
+
+theme:
+  name: material
+  features:
+    - content.code.annotate
+    - navigation.sections
+    - navigation.expand
+    - navigation.indexes
+    - navigation.top
+    - navigation.footer
+    - navigation.tracking
+    - search.highlight
+    - search.share
+    - search.suggest
+    - content.code.copy
+    

outputs/md/databases.md

@@ -0,0 +1,19 @@
+<!-- automatically generated by code in https://github.com/sourmash-bio/2025-sourmash-databases-doc-template/ -->
+<!-- template file: templates/databases.md -->
+
+
+
+[GTDB RS220](gtdb220.md) -- Bacterial and Archaeal genomes from GTDB RS220.
+
+
+
+[GTDB RS226](gtdb226.md) -- Bacterial and Archaeal genomes from GTDB RS226.
+
+
+
+[NCBI Viruses (Jan 2025)](ncbi_viruses_2025_01.md) -- All viruses from NCBI (NCBI:txid10239) as of January 2025.
+
+
+
+[NCBI Eukaryotes (Jan 2025)](ncbi_euks_2025_01.md) -- All eukaryotic reference genomes from NCBI (NCBI:txid2759) as of January 2025.
+

outputs/md/gtdb220.md

@@ -1,18 +1,69 @@
-# Collection: GTDB RS220 - All Bacteria and Archaea from GTDB RS220
+<!-- automatically generated by code in https://github.com/sourmash-bio/2025-sourmash-databases-doc-template/ -->
+<!-- template file: templates/complete.md -->
+
+# Collection: GTDB RS220
+
+Bacterial and Archaeal genomes from GTDB RS220.
 
 Links:
+
 * [Announcement](https://forum.gtdb.ecogenomic.org/t/announcing-gtdb-r09-rs220/595)
 
 ## Database files:
 
 Files:
 
-* zip: [gtdb-rs220-k21.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db/gtdb-rs220/gtdb-rs220-k21.zip) - all GTDB genomes - DNA, k=21, scaled=1000
-* zip: [gtdb-rs220-k31.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db/gtdb-rs220/gtdb-rs220-k31.zip) - all GTDB genomes - DNA, k=31, scaled=1000
-* zip: [gtdb-rs220-k51.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db/gtdb-rs220/gtdb-rs220-k51.zip) - all GTDB genomes - DNA, k=51, scaled=1000
+* zip: [gtdb-rs220-k21.dna.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-rs220-k21.dna.zip) - all GTDB genomes. - DNA, k=21, scaled=1000 (17.0 GB)
+* zip: [gtdb-rs220-k31.dna.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-rs220-k31.dna.zip) - all GTDB genomes. - DNA, k=31, scaled=1000 (17.0 GB)
+* zip: [gtdb-rs220-k51.dna.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-rs220-k51.dna.zip) - all GTDB genomes. - DNA, k=51, scaled=1000 (17.0 GB)
+
+
+* zip: [gtdb-reps-rs220-k21.dna.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-reps-rs220-k21.dna.zip) - all GTDB species representative genomes. - DNA, k=21, scaled=1000 (2.8 GB)
+* zip: [gtdb-reps-rs220-k31.dna.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-reps-rs220-k31.dna.zip) - all GTDB species representative genomes. - DNA, k=31, scaled=1000 (2.8 GB)
+* zip: [gtdb-reps-rs220-k51.dna.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-reps-rs220-k51.dna.zip) - all GTDB species representative genomes. - DNA, k=51, scaled=1000 (2.8 GB)
 
 
 
 ## Taxonomy files:
 
-* [GTDB taxonomy for RS220](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db/gtdb-rs220.lineages.csv)
+* [GTDB taxonomy for RS220.](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-rs220.lineages.csv)
+
+
+## Advanced
+
+### Download via curl using the command line
+
+```shell
+# download gtdb-rs220-k21.dna.zip
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-rs220-k21.dna.zip
+
+# download gtdb-rs220-k31.dna.zip
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-rs220-k31.dna.zip
+
+# download gtdb-rs220-k51.dna.zip
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-rs220-k51.dna.zip
+
+# download gtdb-reps-rs220-k21.dna.zip
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-reps-rs220-k21.dna.zip
+
+# download gtdb-reps-rs220-k31.dna.zip
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-reps-rs220-k31.dna.zip
+
+# download gtdb-reps-rs220-k51.dna.zip
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-reps-rs220-k51.dna.zip
+
+# download taxonomy file
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-rs220.lineages.csv
+```
+
+### A list of all the URLs
+
+```
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-rs220-k21.dna.zip
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-rs220-k31.dna.zip
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-rs220-k51.dna.zip
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-reps-rs220-k21.dna.zip
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-reps-rs220-k31.dna.zip
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-reps-rs220-k51.dna.zip
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs220/gtdb-rs220.lineages.csv
+```

outputs/md/gtdb226.md

@@ -0,0 +1,69 @@
+<!-- automatically generated by code in https://github.com/sourmash-bio/2025-sourmash-databases-doc-template/ -->
+<!-- template file: templates/complete.md -->
+
+# Collection: GTDB RS226
+
+Bacterial and Archaeal genomes from GTDB RS226.
+
+Links:
+
+* [Announcement](https://forum.gtdb.ecogenomic.org/t/announcing-gtdb-r10-rs226/724)
+
+## Database files:
+
+Files:
+
+* zip: [gtdb-rs226-k21.dna.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-rs226-k21.dna.zip) - all GTDB genomes. - DNA, k=21, scaled=1000 (21.0 GB)
+* zip: [gtdb-rs226-k31.dna.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-rs226-k31.dna.zip) - all GTDB genomes. - DNA, k=31, scaled=1000 (21.0 GB)
+* zip: [gtdb-rs226-k51.dna.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-rs226-k51.dna.zip) - all GTDB genomes. - DNA, k=51, scaled=1000 (21.0 GB)
+
+
+* zip: [gtdb-reps-rs226-k21.dna.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-reps-rs226-k21.dna.zip) - all GTDB species representative genomes. - DNA, k=21, scaled=1000 (21.0 GB)
+* zip: [gtdb-reps-rs226-k31.dna.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-reps-rs226-k31.dna.zip) - all GTDB species representative genomes. - DNA, k=31, scaled=1000 (21.0 GB)
+* zip: [gtdb-reps-rs226-k51.dna.zip](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-reps-rs226-k51.dna.zip) - all GTDB species representative genomes. - DNA, k=51, scaled=1000 (21.0 GB)
+
+
+
+## Taxonomy files:
+
+* [GTDB taxonomy for RS226.](https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-rs226.lineages.csv)
+
+
+## Advanced
+
+### Download via curl using the command line
+
+```shell
+# download gtdb-rs226-k21.dna.zip
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-rs226-k21.dna.zip
+
+# download gtdb-rs226-k31.dna.zip
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-rs226-k31.dna.zip
+
+# download gtdb-rs226-k51.dna.zip
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-rs226-k51.dna.zip
+
+# download gtdb-reps-rs226-k21.dna.zip
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-reps-rs226-k21.dna.zip
+
+# download gtdb-reps-rs226-k31.dna.zip
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-reps-rs226-k31.dna.zip
+
+# download gtdb-reps-rs226-k51.dna.zip
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-reps-rs226-k51.dna.zip
+
+# download taxonomy file
+curl -O --no-clobber https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-rs226.lineages.csv
+```
+
+### A list of all the URLs
+
+```
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-rs226-k21.dna.zip
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-rs226-k31.dna.zip
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-rs226-k51.dna.zip
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-reps-rs226-k21.dna.zip
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-reps-rs226-k31.dna.zip
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-reps-rs226-k51.dna.zip
+https://farm.cse.ucdavis.edu/~ctbrown/sourmash-db.new/gtdb-rs226/gtdb-rs226.lineages.csv
+```