unix: write out and depend on expanded Dockerfile

Previously, if we changed an included Dockerfile, derived Dockerfile
and their images wouldn't be invalidated by make dependencies.

We refactor the code to always write out the expanded Dockerfile so
any content changes result in image invalidation.

Author: indygreg

cpython-unix/Makefile

@@ -5,10 +5,6 @@ OUTDIR := $(ROOT)/build
 BUILD := $(HERE)/build.py
 NULL :=
 
-COMMON_DEPENDS := \
-#    $(BUILD) \
-    $(NULL)
-
 ifndef PYBUILD_TARGET_TRIPLE
     $(error PYBUILD_TARGET_TRIPLE not defined)
 endif
@@ -49,6 +45,9 @@ $(shell $(RUN_BUILD) placeholder_archive makefiles)
 include $(OUTDIR)/Makefile.$(HOST_PLATFORM).$(TARGET_TRIPLE).$(PYBUILD_PYTHON_MAJOR_VERSION)
 include $(OUTDIR)/versions/VERSION.*
 
+# Always write out expanded Dockerfiles.
+$(shell $(RUN_BUILD) placeholder_archive dockerfiles)
+
 BASE_TOOLCHAIN_DEPENDS := \
     $(if $(NEED_BINUTILS),$(OUTDIR)/binutils-$(BINUTILS_VERSION)-$(HOST_PLATFORM).tar) \
     $(if $(NEED_GCC),$(OUTDIR)/gcc-$(GCC_VERSION)-$(HOST_PLATFORM).tar) \
@@ -68,7 +67,7 @@ PYTHON_DEP_DEPENDS := \
 default: $(OUTDIR)/cpython-$(PYBUILD_PYTHON_VERSION)-$(PACKAGE_SUFFIX).tar
 
 ifndef PYBUILD_NO_DOCKER
-$(OUTDIR)/image-%.tar: $(HERE)/%.Dockerfile $(COMMON_DEPENDS)
+$(OUTDIR)/image-%.tar: $(OUTDIR)/%.Dockerfile
 	$(RUN_BUILD) --toolchain image-$*
 endif
 

cpython-unix/build.py

@@ -22,7 +22,7 @@
     parse_setup_line,
     STDLIB_TEST_PACKAGES,
 )
-from pythonbuild.docker import build_docker_image, get_image
+from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles
 from pythonbuild.downloads import DOWNLOADS
 from pythonbuild.logging import log, set_logger
 from pythonbuild.utils import (
@@ -959,7 +959,9 @@ def main():
 
     settings = get_target_settings(TARGETS_CONFIG, target_triple)
 
-    if args.action == "makefiles":
+    if args.action == "dockerfiles":
+        log_name = "dockerfiles"
+    elif args.action == "makefiles":
         log_name = "makefiles"
     elif args.action.startswith("image-"):
         log_name = "image-%s" % action
@@ -978,12 +980,19 @@ def main():
 
     with log_path.open("wb") as log_fh:
         set_logger(action, log_fh)
-        if action == "makefiles":
+        if action == "dockerfiles":
+            write_dockerfiles(SUPPORT, BUILD)
+        elif action == "makefiles":
             write_triples_makefiles(get_targets(TARGETS_CONFIG), BUILD, SUPPORT)
             write_package_versions(BUILD / "versions")
 
         elif action.startswith("image-"):
-            build_docker_image(client, ROOT, BUILD, action[6:])
+            image_name = action[6:]
+            image_path = BUILD / ("%s.Dockerfile" % image_name)
+            with image_path.open("rb") as fh:
+                image_data = fh.read()
+
+            build_docker_image(client, image_data, BUILD, image_name)
 
         elif action == "binutils":
             build_binutils(client, get_image(client, ROOT, BUILD, "gcc"), host_platform)

pythonbuild/docker.py

@@ -13,21 +13,26 @@
 import jinja2
 
 from .logging import log, log_raw
+from .utils import write_if_different
 
 
-def build_docker_image(client, source_dir: pathlib.Path, image_dir: pathlib.Path, name):
-    image_path = image_dir / ("image-%s" % name)
+def write_dockerfiles(source_dir: pathlib.Path, dest_dir: pathlib.Path):
+    env = jinja2.Environment(loader=jinja2.FileSystemLoader(str(source_dir)))
 
-    env = jinja2.Environment(
-        loader=jinja2.FileSystemLoader(str(source_dir / "cpython-unix"))
-    )
+    for f in os.listdir(source_dir):
+        if not f.endswith(".Dockerfile"):
+            continue
 
-    tmpl = env.get_template("%s.Dockerfile" % name)
-    data = tmpl.render()
+        tmpl = env.get_template(f)
+        data = tmpl.render()
 
-    return ensure_docker_image(
-        client, io.BytesIO(data.encode("utf")), image_path=image_path
-    )
+        write_if_different(dest_dir / f, data.encode("utf-8"))
+
+
+def build_docker_image(client, image_data: bytes, image_dir: pathlib.Path, name):
+    image_path = image_dir / ("image-%s" % name)
+
+    return ensure_docker_image(client, io.BytesIO(image_data), image_path=image_path)
 
 
 def ensure_docker_image(client, fh, image_path=None):