refactor: use templates and change CI flow

Author: Midnighter

.github/workflows/main.yml

@@ -1,4 +1,4 @@
-name: Singularity container build
+name: Build Singularity images
 
 on:
   push:
@@ -14,32 +14,31 @@ jobs:
       matrix:
         python-version:
           - '3.10'
+        singularity-version:
+          - '3.8.3'
     steps:
       - name: Checkout bot code
         uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
       - name: Cache downloads
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: |
             ~/.cache/pip
-            /usr/local/bin/singularity
-            /usr/local/go/
-          key: cache-${{ matrix.python-version }}
-      - name: Install singularity
-        run: |
-          bash install_singularity.sh
+          key: ${{ runner.os }}_Python-${{ matrix.python-version }}_Singularity-${{ matrix.singularity-version }}
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
       - name: Install Python dependencies
         run: |
           python -m pip install --requirement requirements.in
-      - name: Get list of containers to build
+      - name: Fetch images and generate build script
         run: |
-          python get_container_list.py
-      - name: Build containers
+          ./populate_build.py
+      - name: Set up Singularity ${{ matrix.singularity-version }}
+        uses: eWaterCycle/setup-singularity@v7
+        with:
+          singularity-version: ${{ matrix.singularity-version }}
+      - name: Build images
         run: |
-          echo "${{ secrets.DEPOT_GALAXYPROJECT_ORG }}" > ssh_key && chmod 600 ssh_key
-          cat build.sh
-          bash build.sh
+          ./build.sh

build.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SOURCE='docker://quay.io/biocontainers'
+DESTINATION='[email protected]:/srv/nginx/depot.galaxyproject.org/root/singularity/'

image_template.sh

@@ -0,0 +1,6 @@
+
+singularity build ${img} "$${SOURCE}/${img}" > /dev/null 2>&1 \
+    && rsync -azq -e 'ssh -i ssh_key -o StrictHostKeyChecking=no' ./${img} "$${DESTINATION}" \
+    && rm ${img} \
+    && singularity cache clean --type blob --force \
+    && echo 'Container ${img} built (${idx}/${total}).'

install_singularity.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 
-"""Provide a command line tool for finding new container images and building them."""
+"""Provide a command line tool for building Singularity images from BioContainers."""
 
 
 import argparse
@@ -11,6 +11,7 @@
 from functools import partial
 from html.parser import HTMLParser
 from pathlib import Path
+from string import Template
 from typing import List, Tuple, Dict, Optional, Iterable
 
 import aiometer
@@ -313,13 +314,13 @@ def parse_denylist(filename: Path) -> List[str]:
         return [entry for line in handle.readlines() if (entry := line.strip())]
 
 
-def generate_build_script(filename: Path, images: List[str]) -> None:
-    """Generate a build script with one new image per line."""
-    with filename.open("w") as handle:
+def generate_build_script(filename: Path, images: List[str], template: Path) -> None:
+    """Generate a build script from provided templates."""
+    with template.open() as handle:
+        img_template = Template(handle.read())
+    with filename.open("a") as handle:
         for idx, img in enumerate(images, start=1):
-            handle.write(
-                f"sudo singularity build {img} docker://quay.io/biocontainers/{img} > /dev/null 2>&1 && rsync -azq -e 'ssh -i ssh_key -o StrictHostKeyChecking=no' ./{img} [email protected]:/srv/nginx/depot.galaxyproject.org/root/singularity/ && rm {img} && echo 'Container {img} built ({idx}/{len(images)}).'\n"
-            )
+            handle.write(img_template.substitute(img=img, idx=idx, total=len(images)))
 
 
 def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
@@ -345,6 +346,15 @@ def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
         type=Path,
         help=f"Output the Singularity build script (default '{default_build_script}').",
     )
+    default_image_template = Path("image_template.sh")
+    parser.add_argument(
+        "--image-template",
+        metavar="PATH",
+        default=default_image_template,
+        type=Path,
+        help=f"The template for building a single Singularity image (default "
+        f"'{default_image_template}'). Uses Python `string.Template` syntax.",
+    )
     default_quay_api = "https://quay.io/api/v1/"
     parser.add_argument(
         "--quay-api",
@@ -384,6 +394,8 @@ def main(argv: Optional[List[str]] = None) -> None:
         handlers=[RichHandler(markup=True, rich_tracebacks=True)],
     )
     assert args.denylist.is_file(), f"File not found '{args.denylist}'."
+    assert args.build_script.is_file(), f"File not found '{args.build_script}'."
+    assert args.image_template.is_file(), f"File not found '{args.image_template}'."
     args.build_script.parent.mkdir(parents=True, exist_ok=True)
     logger.info("Fetching quay.io BioContainers images.")
     quay_images = asyncio.run(QuayImageFetcher.fetch_all(api_url=args.quay_api))
@@ -400,7 +412,7 @@ def main(argv: Optional[List[str]] = None) -> None:
         logger.warning("No new images found.")
         return
     logger.info(f"{len(images):,} new images found. Generating build script.")
-    generate_build_script(args.build_script, images)
+    generate_build_script(args.build_script, images, args.image_template)
 
 
 if __name__ == "__main__":