Set up and build in own venv if requested

Author: simoncozens

Lib/gftools/packager/build.py

@@ -6,7 +6,8 @@
 import subprocess
 import tempfile
 from pathlib import Path
-from typing import List, Optional
+from typing import Dict, List
+from venv import EnvBuilder
 
 import git
 import yaml
@@ -130,6 +131,52 @@ def __init__(
         self.progressbar = None
         self.source_dir = tempfile.TemporaryDirectory()
 
+    def setup_venv(self, source_dir: Path):
+        venv_task = self.progressbar.add_task(
+            "[yellow]Setup venv", total=100, visible=False
+        )
+        self.progressbar.update(
+            venv_task,
+            description="[yellow] Setting up venv for " + self.name + "...",
+            completed=0,
+            visible=True,
+        )
+        if (source_dir / "Makefile").exists:
+            with contextlib.chdir(source_dir):
+                rc = self.run_command_with_callback(
+                    ["make", "venv"],
+                    lambda line: self.progressbar.update(venv_task, advance=1),
+                )
+        elif (source_dir / "requirements.txt").exists:
+            builder = EnvBuilder(system_site_packages=True, with_pip=True)
+            builder.create(str(source_dir / "venv"))
+            self.progressbar.update(venv_task, completed=10)
+            with contextlib.chdir(source_dir):
+                rc = self.run_command_with_callback(
+                    ["venv/bin/pip", "install", "-r", "requirements.txt"],
+                    lambda line: self.progressbar.update(venv_task, advance=1),
+                )
+        else:
+            raise ValueError(
+                "--their-venv was provided but no Makefile or requirements.txt upstream"
+            )
+        if rc != 0:
+            self.progressbar.console.print(
+                "[red]Error setting up venv for " + self.name
+            )
+            raise ValueError("Venv setup failed")
+        self.progressbar.remove_task(venv_task)
+
+    def local_overrides(
+        self, upstream: Path, downstream: Path, overrides: Dict[str, str]
+    ):
+        for source, dest in overrides.items():
+            if (downstream / source).exists():
+                dest_path = upstream / dest
+                os.makedirs(dest_path.parent, exist_ok=True)
+                self.progressbar.console.print("[grey]Using our " + source)
+                shutil.copy(downstream / source, dest_path)
+
     def build(self):
         with Progress(
             progress.TimeElapsedColumn(),
@@ -142,16 +189,19 @@ def build(self):
             with tempfile.TemporaryDirectory() as source_dir:
                 source_dir = Path(source_dir)
                 self.clone_source(source_dir)
-                # Do we have our own local config.yaml?
-                if (self.family_path / "config.yaml").exists():
-                    # If so, copy it over
-                    os.makedirs(source_dir / "sources", exist_ok=True)
-                    shutil.copy(
-                        self.family_path / "config.yaml", source_dir / "sources"
-                    )
+                self.local_overrides(
+                    source_dir,
+                    self.family_path,
+                    {
+                        "config.yaml": "sources/config.yaml",
+                        "requirements.txt": "requirements.txt",
+                    },
+                )
 
                 if not (source_dir / "sources").exists():
                     raise ValueError(f"Could not find sources directory in {self.name}")
+                if self.their_venv:
+                    self.setup_venv(source_dir)
 
                 # Locate the config.yaml file or first source
                 arg = find_config_yaml(source_dir)
@@ -164,14 +214,18 @@ def build(self):
                     arg = sources[0]
 
                 with contextlib.chdir(source_dir):
-                    buildcmd = ["gftools-builder", str(arg)]
+                    if self.their_venv:
+                        buildcmd = ["venv/bin/gftools-builder", str(arg)]
+                    else:
+                        buildcmd = ["gftools-builder", str(arg)]
                     self.run_build_command(buildcmd)
                     self.copy_files()
 
-    def run_build_command(self, buildcmd):
-        build_task = self.progressbar.add_task("[green]Build " + self.name, total=1)
+    def run_command_with_callback(self, cmd, callback):
         process = subprocess.Popen(
-            buildcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
         )
         sel = selectors.DefaultSelector()
         sel.register(process.stdout, selectors.EVENT_READ)
@@ -185,12 +239,8 @@ def run_build_command(self, buildcmd):
                 if not line:
                     ok = False
                     break
-                if key.fileobj is process.stdout and (
-                    m := re.match(r"^\[(\d+)/(\d+)\]", line.decode("utf8"))
-                ):
-                    self.progressbar.update(
-                        build_task, completed=int(m.group(1)), total=int(m.group(2))
-                    )
+                if key.fileobj is process.stdout:
+                    callback(line)
                 elif key.fileobj is process.stderr:
                     stderrlines.append(line)
                 else:
@@ -201,7 +251,18 @@ def run_build_command(self, buildcmd):
                 self.progressbar.console.print(line.decode("utf-8"), end="")
             for line in stderrlines:
                 self.progressbar.console.print("[red]" + line.decode("utf8"), end="")
+        return rc
+
+    def run_build_command(self, buildcmd):
+        build_task = self.progressbar.add_task("[green]Build " + self.name, total=1)
+
+        def progress_callback(line):
+            if m := re.match(r"^\[(\d+)/(\d+)\]", line.decode("utf8")):
+                self.progressbar.update(
+                    build_task, completed=int(m.group(1)), total=int(m.group(2))
+                )
 
+        if self.run_command_with_callback(buildcmd, progress_callback) != 0:
             self.progressbar.console.print("[red]Error building " + self.name)
             raise ValueError("Build failed")
         else: