update script to select latest biocontainer (#629)

* update script to select latest biocontainer
* updating black formatting

Signed-off-by: vsoch <[email protected]>

Author: vsoch

actions/cache-update/scripts/update-from-cache.py

@@ -66,7 +66,6 @@ def get_parser():
 
 
 def main():
-
     parser = get_parser()
 
     # If an error occurs while parsing the arguments, the interpreter will exit with value 2

actions/cache-update/scripts/update-from-listing.py

@@ -37,7 +37,6 @@ def get_parser():
 
 
 def main():
-
     parser = get_parser()
 
     # If an error occurs while parsing the arguments, the interpreter will exit with value 2
@@ -72,7 +71,6 @@ def main():
     # Do a diff between lists, get tag for container if not added
     # This checks the registry and ensures the path doesn't already exist
     for container in iter_tags(containers, existing, args.registry):
-
         # We assume we cannot get aliases here
         try:
             add_container(

example/biocontainer-match.py

@@ -5,6 +5,9 @@
 # You should still set up your module_base before doing this (as we need
 # to write module files.
 
+# This script also requires pipelib
+# pip install pipelib
+
 # **ALWAYS** dry run first!
 #      python biocontainer-match.py --dry-run
 # At the root of your depot do:
@@ -22,10 +25,26 @@
 import re
 import sys
 
+import pipelib.pipeline as pipeline
+import pipelib.pipelines as pipelines
+import pipelib.steps as step
+
 from shpc.logger import logger
 from shpc.main import get_client
 from shpc.main.container import ContainerConfig
 
+# A pipeline to process docker tags
+steps = (
+    # Filter out those that look like commits
+    pipelines.git.RemoveCommits,
+    # Scrub commits from version string
+    step.filters.CleanCommit(),
+    # Parse versions, return sorted ascending, and taking version major.minor.patch into account
+    step.container.ContainerTagSort(),
+)
+
+p = pipeline.Pipeline(steps)
+
 
 def get_parser():
     parser = argparse.ArgumentParser(
@@ -51,7 +70,6 @@ def get_parser():
 
 
 def main():
-
     parser = get_parser()
 
     # If an error occurs while parsing the arguments, the interpreter will exit with value 2
@@ -65,22 +83,34 @@ def main():
     depot = os.path.abspath(args.containers)
     cli = get_client()
 
-    # Keep a record of repos we've seen and don't repeat
-    seen = set()
-
     # Ensure we start with the populated modules
     cli.registry.iter_modules()
 
+    # Organize paths based on container uri
+    paths = {}
+
     # Find all paths that match the pattern of a name:tag
+    # Organize by tag so we can later find the newest
     for path in os.listdir(depot):
         if not re.search("^(.*):(.*)$", path) or os.path.isdir(path):
             continue
         repo, tag = path.split(":", 1)
+        if repo not in paths:
+            paths[repo] = set()
+        paths[repo].add(tag)
 
-        # Don't need to parse twice
-        if repo in seen:
+    # Find all paths that match the pattern of a name:tag
+    for repo, tags in paths.items():
+        # The updated and transformed items
+        sorted_tags = p.run(tags, unwrap=False)
+        if not sorted_tags:
             continue
 
+        latest = sorted_tags[0]
+        tag = latest._original
+        path = f"{repo}:{tag}"
+        print(f"⭐️ Found {len(tags)} for {repo}, latest is {tag}")
+
         container = f"{args.namespace}/{repo}"
         tagged = f"{container}:{tag}"
 
@@ -96,7 +126,6 @@ def main():
 
         if args.dry_run:
             print(f"Would be installing {path} to {tagged}")
-            seen.add(repo)
             continue
 
         # Note we need to install the correct version but using aliases
@@ -121,7 +150,6 @@ def main():
             container_image=path,
             keep_path=True,
         )
-        seen.add(repo)
 
 
 if __name__ == "__main__":

setup.py

@@ -61,7 +61,6 @@ def get_reqs(lookup=None, key="INSTALL_REQUIRES"):
 ################################################################################
 
 if __name__ == "__main__":
-
     INSTALL_REQUIRES = get_reqs(lookup)
     TESTS_REQUIRES = get_reqs(lookup, "TESTS_REQUIRES")
     INSTALL_REQUIRES_ALL = get_reqs(lookup, "INSTALL_REQUIRES_ALL")

shpc/client/add.py

@@ -6,7 +6,6 @@
 
 
 def main(args, parser, extra, subparser):
-
     from shpc.main import get_client
 
     shpc.utils.ensure_no_extra(extra)

shpc/client/check.py

@@ -6,7 +6,6 @@
 
 
 def main(args, parser, extra, subparser):
-
     from shpc.main import get_client
 
     shpc.utils.ensure_no_extra(extra)

shpc/client/config.py

@@ -10,7 +10,6 @@
 
 
 def main(args, parser, extra, subparser):
-
     from shpc.main import get_client
 
     shpc.utils.ensure_no_extra(extra)

shpc/client/docgen.py

@@ -6,7 +6,6 @@
 
 
 def main(args, parser, extra, subparser):
-
     from shpc.main import get_client
 
     shpc.utils.ensure_no_extra(extra)

shpc/client/get.py

@@ -6,7 +6,6 @@
 
 
 def main(args, parser, extra, subparser):
-
     from shpc.main import get_client
 
     shpc.utils.ensure_no_extra(extra)

shpc/client/inspect.py

@@ -9,7 +9,6 @@
 
 
 def main(args, parser, extra, subparser):
-
     from shpc.main import get_client
 
     shpc.utils.ensure_no_extra(extra)
@@ -35,7 +34,6 @@ def main(args, parser, extra, subparser):
 
     # Case 3: pretty print the whole thing
     else:
-
         # Inspect Singularity formatted metadata
         if "attributes" in metadata:
             for key, value in metadata.get("attributes", {}).items():