Merge pull request ceph#63887 from phlogistonjohn/jjm-bwc-improvements

Assorted build-with-container improvements

Author: dmick

src/script/build-with-container.py

@@ -70,6 +70,9 @@
 import contextlib
 import enum
 import glob
+import hashlib
+import functools
+import json
 import logging
 import os
 import pathlib
@@ -78,6 +81,7 @@
 import shutil
 import subprocess
 import sys
+import time
 
 log = logging.getLogger()
 
@@ -149,6 +153,18 @@ class DidNotExecute(Exception):
     pass
 
 
+_CONTAINER_SOURCES = [
+    "Dockerfile.build",
+    "src/script/lib-build.sh",
+    "src/script/run-make.sh",
+    "ceph.spec.in",
+    "do_cmake.sh",
+    "install-deps.sh",
+    "run-make-check.sh",
+    "src/script/buildcontainer-setup.sh",
+]
+
+
 def _cmdstr(cmd):
     return " ".join(shlex.quote(c) for c in cmd)
 
@@ -218,6 +234,9 @@ def _git_command(ctx, args):
     return cmd
 
 
+# Assume that the git version will not be changing after the 1st time
+# the command is run.
+@functools.cache
 def _git_current_branch(ctx):
     cmd = _git_command(ctx, ["rev-parse", "--abbrev-ref", "HEAD"])
     res = _run(cmd, check=True, capture_output=True)
@@ -234,6 +253,20 @@ def _git_current_sha(ctx, short=True):
     return res.stdout.decode("utf8").strip()
 
 
+@functools.cache
+def _hash_sources(bsize=4096):
+    hh = hashlib.sha256()
+    buf = bytearray(bsize)
+    for path in sorted(_CONTAINER_SOURCES):
+        with open(path, "rb") as fh:
+            while True:
+                rlen = fh.readinto(buf)
+                hh.update(buf[:rlen])
+                if rlen < len(buf):
+                    break
+    return f"sha256:{hh.hexdigest()}"
+
+
 class Steps(StrEnum):
     DNF_CACHE = "dnfcache"
     BUILD_CONTAINER = "build-container"
@@ -396,6 +429,7 @@ class Builder:
 
     def __init__(self):
         self._did_steps = set()
+        self._reported_failed = False
 
     def wants(self, step, ctx, *, force=False, top=False):
         log.info("want to execute build step: %s", step)
@@ -407,13 +441,35 @@ def wants(self, step, ctx, *, force=False, top=False):
             return
         if not self._did_steps:
             prepare_env_once(ctx)
-        self._steps[step](ctx)
-        self._did_steps.add(step)
-        log.info("step done: %s", step)
+        with self._timer(step):
+            self._steps[step](ctx)
+            self._did_steps.add(step)
 
     def available_steps(self):
         return [str(k) for k in self._steps]
 
+    @contextlib.contextmanager
+    def _timer(self, step):
+        ns = argparse.Namespace(start=time.monotonic())
+        status = "not-started"
+        try:
+            yield ns
+            status = "completed"
+        except Exception:
+            status = "failed"
+            raise
+        finally:
+            ns.end = time.monotonic()
+            ns.duration = int(ns.end - ns.start)
+            hrs, _rest = map(int, divmod(ns.duration, 3600))
+            mins, secs = map(int, divmod(_rest, 60))
+            ns.duration_hms = f"{hrs:02}:{mins:02}:{secs:02}"
+            if not self._reported_failed:
+                log.info(
+                    "step done: %s %s in %s", step, status, ns.duration_hms
+                )
+            self._reported_failed = status == "failed"
+
     @classmethod
     def set(self, step):
         def wrap(f):
@@ -462,6 +518,7 @@ def build_container(ctx):
         "--pull",
         "-t",
         ctx.image_name,
+        f"--label=io.ceph.build-with-container.src={_hash_sources()}",
         f"--build-arg=JENKINS_HOME={ctx.cli.homedir}",
         f"--build-arg=CEPH_BASE_BRANCH={ctx.base_branch()}",
     ]
@@ -482,32 +539,58 @@ def build_container(ctx):
         _run(cmd, check=True, ctx=ctx)
 
 
-@Builder.set(Steps.CONTAINER)
-def get_container(ctx):
-    """Build or fetch a container image that we will build in."""
+def _check_cached_image(ctx):
     inspect_cmd = [
         ctx.container_engine,
         "image",
         "inspect",
         ctx.image_name,
     ]
+    res = _run(inspect_cmd, check=False, capture_output=True)
+    if res.returncode != 0:
+        log.info("Container image %s not present", ctx.image_name)
+        return False, False
+
+    log.info("Container image %s present", ctx.image_name)
+    ctr_info = json.loads(res.stdout)[0]
+    labels = {}
+    if "Labels" in ctr_info:
+        labels = ctr_info["Labels"]
+    elif "Labels" in ctr_info.get("ContainerConfig", {}):
+        labels = ctr_info["ContainerConfig"]["Labels"]
+    elif "Labels" in ctr_info.get("Config", {}):
+        labels = ctr_info["Config"]["Labels"]
+    saved_hash = labels.get("io.ceph.build-with-container.src", "")
+    curr_hash = _hash_sources()
+    if saved_hash == curr_hash:
+        log.info("Container passes source check")
+        return True, True
+    log.info("Container sources do not match: %s", curr_hash)
+    return True, False
+
+
+@Builder.set(Steps.CONTAINER)
+def get_container(ctx):
+    """Build or fetch a container image that we will build in."""
     pull_cmd = [
         ctx.container_engine,
         "pull",
         ctx.image_name,
     ]
     allowed = ctx.cli.image_sources or ImageSource
     if ImageSource.CACHE in allowed:
-        res = _run(inspect_cmd, check=False, capture_output=True)
-        if res.returncode == 0:
-            log.info("Container image %s present", ctx.image_name)
+        log.info("Checking for cached image")
+        present, hash_ok = _check_cached_image(ctx)
+        if present and hash_ok or len(allowed) == 1:
             return
-        log.info("Container image %s not present", ctx.image_name)
     if ImageSource.PULL in allowed:
+        log.info("Checking for image in remote repository")
         res = _run(pull_cmd, check=False, capture_output=True)
         if res.returncode == 0:
             log.info("Container image %s pulled successfully", ctx.image_name)
-            return
+            present, hash_ok = _check_cached_image(ctx)
+            if present and hash_ok:
+                return
     log.info("Container image %s needed", ctx.image_name)
     if ImageSource.BUILD in allowed:
         ctx.build.wants(Steps.BUILD_CONTAINER, ctx)
@@ -598,6 +681,11 @@ def bc_make_source_rpm(ctx):
         _run(cmd, check=True, ctx=ctx)
 
 
+def _glob_search(ctx, pattern):
+    overlay = ctx.overlay()
+    return glob.glob(pattern, root_dir=overlay.upper if overlay else None)
+
+
 @Builder.set(Steps.RPM)
 def bc_build_rpm(ctx):
     """Build RPMs from SRPM."""
@@ -618,7 +706,7 @@ def bc_build_rpm(ctx):
                 ctx.cli.ceph_version
             )
             srpm_glob = f"ceph-{srpm_version}.*.src.rpm"
-    paths = glob.glob(srpm_glob)
+    paths = _glob_search(ctx, srpm_glob)
     if len(paths) > 1:
         raise RuntimeError(
             "too many matching source rpms"
@@ -628,8 +716,11 @@ def bc_build_rpm(ctx):
     if not paths:
         # no matches. build a new srpm
         ctx.build.wants(Steps.SOURCE_RPM, ctx)
-        paths = glob.glob(srpm_glob)
-        assert paths
+        paths = _glob_search(ctx, srpm_glob)
+        if not paths:
+            raise RuntimeError(
+                f"unable to find source rpm(s) matching {srpm_glob}"
+            )
     srpm_path = pathlib.Path(ctx.cli.homedir) / paths[0]
     topdir = pathlib.Path(ctx.cli.homedir) / "rpmbuild"
     if ctx.cli.build_dir:
@@ -640,7 +731,7 @@ def bc_build_rpm(ctx):
         'rpmbuild',
         '--rebuild',
         f'-D_topdir {topdir}',
-    ] + list(ctx.cli.rpmbuild_arg) + [str(srpm_path)]
+    ] + list(ctx.cli.rpmbuild_arg or []) + [str(srpm_path)]
     rpmbuild_cmd = ' '.join(shlex.quote(cmd) for cmd in rpmbuild_args)
     cmd = _container_cmd(
         ctx,

src/script/buildcontainer-setup.sh

@@ -5,6 +5,7 @@ install_container_deps() {
     # set JENKINS_HOME in order to have the build container look as much
     # like an existing jenkins build environment as possible
     export JENKINS_HOME=/ceph
+    export WITH_CRIMSON=true
     prepare
 }