|
| 1 | +from pathlib import Path |
1 | 2 | import re |
2 | | -from typing import List |
| 3 | +import tempfile |
| 4 | +from typing import Iterable, List |
3 | 5 | from urllib.parse import urlparse |
| 6 | +import subprocess as sp |
4 | 7 | from snakedeploy.logger import logger |
5 | 8 |
|
6 | | -from github import Github |
7 | | - |
8 | | - |
9 | | -def update_snakemake_wrappers(snakefiles: List[str], git_ref: str): |
10 | | - """Set all snakemake wrappers to the given git ref (e.g. tag or branch).""" |
11 | | - |
12 | | - if git_ref is None: |
13 | | - logger.info("Obtaining latest release of snakemake-wrappers...") |
14 | | - github = Github() |
15 | | - repo = github.get_repo("snakemake/snakemake-wrappers") |
16 | | - releases = repo.get_releases() |
17 | | - git_ref = releases[0].tag_name |
18 | | - |
19 | | - for snakefile in snakefiles: |
20 | | - with open(snakefile, "r") as infile: |
21 | | - snakefile_content = infile.read() |
22 | | - |
23 | | - def update_spec(matchobj): |
24 | | - spec = matchobj.group("spec") |
25 | | - url = urlparse(spec) |
26 | | - if not url.scheme: |
27 | | - old_git_ref, rest = spec.split("/", 1) |
28 | | - return ( |
29 | | - matchobj.group("def") |
30 | | - + matchobj.group("quote") |
31 | | - + f"{git_ref}/{rest}" |
32 | | - + matchobj.group("quote") |
33 | | - ) |
34 | | - else: |
35 | | - return matchobj.group() |
36 | | - |
37 | | - logger.info(f"Updating snakemake-wrappers in {snakefile} to {git_ref}...") |
38 | | - snakefile_content = re.sub( |
39 | | - "(?P<def>wrapper:\\n?\\s*)(?P<quote>['\"])(?P<spec>.+)(?P=quote)", |
40 | | - update_spec, |
41 | | - snakefile_content, |
| 9 | + |
| 10 | +def get_latest_git_tag(path: Path, repo: Path) -> str | None: |
| 11 | + """Get the latest git tag of any file in the given directory or below. |
| 12 | + Thereby ignore later git tags outside of the given directory. |
| 13 | + """ |
| 14 | + |
| 15 | + # get the latest git commit that changed the given dir: |
| 16 | + commit = ( |
| 17 | + sp.run( |
| 18 | + ["git", "rev-list", "-1", "HEAD", "--", str(path)], |
| 19 | + stdout=sp.PIPE, |
| 20 | + cwd=repo, |
| 21 | + check=True, |
| 22 | + ) |
| 23 | + .stdout.decode() |
| 24 | + .strip() |
| 25 | + ) |
| 26 | + # get the first git tag that includes this commit: |
| 27 | + # Note: We want the EARLIEST tag containing the commit, which represents |
| 28 | + # the first version where this wrapper reached its current state |
| 29 | + tags = ( |
| 30 | + sp.run( |
| 31 | + ["git", "tag", "--sort", "creatordate", "--contains", commit], |
| 32 | + check=True, |
| 33 | + cwd=repo, |
| 34 | + stdout=sp.PIPE, |
| 35 | + ) |
| 36 | + .stdout.decode() |
| 37 | + .strip() |
| 38 | + .splitlines() |
| 39 | + ) |
| 40 | + if not tags: |
| 41 | + return None |
| 42 | + else: |
| 43 | + return tags[0] |
| 44 | + |
| 45 | + |
| 46 | +def get_sparse_checkout_patterns() -> Iterable[str]: |
| 47 | + for wrapper_pattern in ("*", "*/*"): |
| 48 | + for filetype in ("wrapper.*", "environment.yaml"): |
| 49 | + yield f"/*/{wrapper_pattern}/{filetype}" |
| 50 | + yield "/meta/*/*/test/Snakefile" |
| 51 | + |
| 52 | + |
| 53 | +class WrapperRepo: |
| 54 | + def __init__(self): |
| 55 | + self.tmpdir = tempfile.TemporaryDirectory() |
| 56 | + logger.info("Cloning snakemake-wrappers repository...") |
| 57 | + sp.run( |
| 58 | + [ |
| 59 | + "git", |
| 60 | + "clone", |
| 61 | + "--filter=blob:none", |
| 62 | + "--no-checkout", |
| 63 | + "https://github.com/snakemake/snakemake-wrappers.git", |
| 64 | + ".", |
| 65 | + ], |
| 66 | + cwd=self.tmpdir.name, |
| 67 | + check=True, |
| 68 | + ) |
| 69 | + sp.run( |
| 70 | + ["git", "config", "core.sparseCheckoutCone", "false"], |
| 71 | + cwd=self.tmpdir.name, |
| 72 | + check=True, |
42 | 73 | ) |
43 | | - with open(snakefile, "w") as outfile: |
44 | | - outfile.write(snakefile_content) |
| 74 | + sp.run(["git", "sparse-checkout", "disable"], cwd=self.tmpdir.name, check=True) |
| 75 | + sp.run( |
| 76 | + ["git", "sparse-checkout", "set", "--no-cone"] |
| 77 | + + list(get_sparse_checkout_patterns()), |
| 78 | + cwd=self.tmpdir.name, |
| 79 | + check=True, |
| 80 | + ) |
| 81 | + sp.run(["git", "read-tree", "-mu", "HEAD"], cwd=self.tmpdir.name, check=True) |
| 82 | + self.repo_dir = Path(self.tmpdir.name) |
| 83 | + |
| 84 | + def get_wrapper_version(self, spec: str) -> str | None: |
| 85 | + if not (self.repo_dir / spec).exists(): |
| 86 | + return None |
| 87 | + return get_latest_git_tag(Path(spec), self.repo_dir) |
| 88 | + |
| 89 | + def __enter__(self): |
| 90 | + return self |
| 91 | + |
| 92 | + def __exit__(self, exc_type, exc_value, traceback): |
| 93 | + self.tmpdir.cleanup() |
| 94 | + |
| 95 | + |
| 96 | +def update_snakemake_wrappers(snakefiles: List[str]): |
| 97 | + """Update all snakemake wrappers to their specific latest versions.""" |
| 98 | + |
| 99 | + with WrapperRepo() as wrapper_repo: |
| 100 | + for snakefile in snakefiles: |
| 101 | + with open(snakefile, "r") as infile: |
| 102 | + snakefile_content = infile.read() |
| 103 | + |
| 104 | + def update_spec(matchobj): |
| 105 | + spec = matchobj.group("spec") |
| 106 | + url = urlparse(spec) |
| 107 | + if not url.scheme: |
| 108 | + parts = spec.split("/", 1) |
| 109 | + if len(parts) != 2: |
| 110 | + logger.warning( |
| 111 | + f"Could not parse wrapper specification '{spec}' " |
| 112 | + "(expected version/cat/name or version/cat/name/subcommand). " |
| 113 | + "Leaving unchanged." |
| 114 | + ) |
| 115 | + return matchobj.group() |
| 116 | + old_git_ref, rest = parts |
| 117 | + git_ref = wrapper_repo.get_wrapper_version(rest) |
| 118 | + if git_ref is None: |
| 119 | + logger.warning( |
| 120 | + f"Could not determine latest version of wrapper '{rest}'. " |
| 121 | + "Leaving unchanged." |
| 122 | + ) |
| 123 | + return matchobj.group() |
| 124 | + elif git_ref != old_git_ref: |
| 125 | + logger.info( |
| 126 | + f"Updated wrapper '{rest}' from {old_git_ref} to {git_ref}." |
| 127 | + ) |
| 128 | + else: |
| 129 | + logger.info( |
| 130 | + f"Wrapper '{rest}' is already at latest version {git_ref}." |
| 131 | + ) |
| 132 | + return ( |
| 133 | + matchobj.group("def") |
| 134 | + + matchobj.group("quote") |
| 135 | + + f"{git_ref}/{rest}" |
| 136 | + + matchobj.group("quote") |
| 137 | + ) |
| 138 | + else: |
| 139 | + return matchobj.group() |
| 140 | + |
| 141 | + logger.info( |
| 142 | + f"Updating snakemake-wrappers and meta-wrappers in {snakefile}..." |
| 143 | + ) |
| 144 | + snakefile_content = re.sub( |
| 145 | + "(?P<def>(meta_)?wrapper:\\n?\\s*)(?P<quote>['\"])(?P<spec>.+)(?P=quote)", |
| 146 | + update_spec, |
| 147 | + snakefile_content, |
| 148 | + ) |
| 149 | + with open(snakefile, "w") as outfile: |
| 150 | + outfile.write(snakefile_content) |
0 commit comments