unix: derive set of packages to copy from config file

The code was mostly redundant with the config file annotations.

But it was incomplete and there were some minor name mismatches. So
we close the gaps as part of this change.

Author: indygreg

cpython-unix/Makefile

@@ -206,7 +206,7 @@ TIX_DEPENDS = \
     $(HERE)/build-tix.sh \
     $(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar \
     $(OUTDIR)/tk-$(TK_VERSION)-$(PACKAGE_SUFFIX).tar \
-    $(if $(NEED_X11),$(OUTDIR)/libX11-$(LIBX11_VERSION)-$(PACKAGE_SUFFIX).tar) \
+    $(if $(NEED_LIBX11),$(OUTDIR)/libX11-$(LIBX11_VERSION)-$(PACKAGE_SUFFIX).tar) \
     $(NULL)
 
 $(OUTDIR)/tix-$(TIX_VERSION)-$(PACKAGE_SUFFIX).tar: $(TIX_DEPENDS)
@@ -215,7 +215,7 @@ $(OUTDIR)/tix-$(TIX_VERSION)-$(PACKAGE_SUFFIX).tar: $(TIX_DEPENDS)
 TK_DEPENDS = \
     $(HERE)/build-tk.sh \
     $(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar \
-    $(if $(NEED_X11),$(OUTDIR)/libX11-$(LIBX11_VERSION)-$(PACKAGE_SUFFIX).tar) \
+    $(if $(NEED_LIBX11),$(OUTDIR)/libX11-$(LIBX11_VERSION)-$(PACKAGE_SUFFIX).tar) \
     $(NULL)
 
 $(OUTDIR)/tk-$(TK_VERSION)-$(PACKAGE_SUFFIX).tar: $(TK_DEPENDS)
@@ -250,7 +250,7 @@ $(OUTDIR)/zlib-$(ZLIB_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HE
 
 PYTHON_DEPENDS := \
     $(if $(NEED_BDB),$(OUTDIR)/bdb-$(BDB_VERSION)-$(PACKAGE_SUFFIX).tar) \
-    $(OUTDIR)/bzip2-$(BZIP2_VERSION)-$(PACKAGE_SUFFIX).tar \
+    $(if $(NEED_BZIP2),$(OUTDIR)/bzip2-$(BZIP2_VERSION)-$(PACKAGE_SUFFIX).tar) \
     $(if $(NEED_GDBM),$(OUTDIR)/gdbm-$(GDBM_VERSION)-$(PACKAGE_SUFFIX).tar) \
     $(if $(NEED_GETTEXT),$(OUTDIR)/gettext-$(GETTEXT_VERSION)-$(PACKAGE_SUFFIX).tar) \
     $(if $(NEED_LIBEDIT),$(OUTDIR)/libedit-$(LIBEDIT_VERSION)-$(PACKAGE_SUFFIX).tar) \
@@ -262,8 +262,8 @@ PYTHON_DEPENDS := \
     $(if $(NEED_SQLITE),$(OUTDIR)/sqlite-$(SQLITE_VERSION)-$(PACKAGE_SUFFIX).tar) \
     $(if $(NEED_TIX),$(OUTDIR)/tix-$(TIX_VERSION)-$(PACKAGE_SUFFIX).tar) \
     $(if $(NEED_UUID),$(OUTDIR)/uuid-$(UUID_VERSION)-$(PACKAGE_SUFFIX).tar) \
-    $(OUTDIR)/xz-$(XZ_VERSION)-$(PACKAGE_SUFFIX).tar \
-    $(OUTDIR)/zlib-$(ZLIB_VERSION)-$(PACKAGE_SUFFIX).tar \
+    $(if $(NEED_XZ),$(OUTDIR)/xz-$(XZ_VERSION)-$(PACKAGE_SUFFIX).tar) \
+    $(if $(NEED_ZLIB),$(OUTDIR)/zlib-$(ZLIB_VERSION)-$(PACKAGE_SUFFIX).tar) \
     $(HERE)/disabled-static-modules.$(PYBUILD_PYTHON_MAJOR_VERSION).$(HOST_PLATFORM) \
     $(HERE)/required-extensions.$(PYBUILD_PYTHON_MAJOR_VERSION).$(HOST_PLATFORM) \
     $(HERE)/static-modules.$(PYBUILD_PYTHON_MAJOR_VERSION).$(HOST_PLATFORM) \

cpython-unix/build.py

@@ -29,6 +29,7 @@
     add_licenses_to_extension_entry,
     download_entry,
     get_targets,
+    target_needs,
     validate_python_json,
     write_package_versions,
     write_triples_makefiles,
@@ -820,53 +821,11 @@ def build_cpython(
             musl="musl" in target_triple,
         )
 
-        # TODO support bdb/gdbm toggle
-        packages = {
-            "bzip2",
-            "xz",
-            "zlib",
-        }
-
-        bdb = host_platform != "macos"
-        if bdb:
-            packages.add("bdb")
-
-        libedit = target_triple not in ("aarch64-apple-ios", "x86_64-apple-ios")
-        if libedit:
-            packages.add("libedit")
-
-        libffi = target_triple not in ("aarch64-apple-ios", "x86_64-apple-ios")
-        if libffi:
-            packages.add("libffi")
-
-        if libressl:
-            packages.add("libressl")
-        else:
-            packages.add("openssl")
-
-        # We use the system ncurses on macOS for now.
-        ncurses = host_platform != "macos"
-        if ncurses:
-            packages.add("ncurses")
-
-        readline = host_platform != "macos"
-        if readline:
-            packages.add("readline")
-
-        sqlite = target_triple not in ("aarch64-apple-ios", "x86_64-apple-ios")
-        if sqlite:
-            packages.add("sqlite")
-
-        tk = target_triple not in ("aarch64-apple-ios", "x86_64-apple-ios")
-        if tk:
-            packages |= {"tcl", "tix", "tk"}
-
-        uuid = target_triple not in ("aarch64-apple-ios", "x86_64-apple-ios")
-        if uuid:
-            packages.add("uuid")
-
-        if host_platform == "linux64":
-            packages |= {"libX11", "libXau", "libxcb", "xorgproto"}
+        packages = target_needs(TARGETS_CONFIG, target_triple)
+        # Toolchain packages are handled specially.
+        packages.discard("binutils")
+        packages.discard("clang")
+        packages.discard("gcc")
 
         for p in sorted(packages):
             build_env.install_artifact_archive(BUILD, p, target_triple, optimizations)

cpython-unix/targets.yml

@@ -12,91 +12,132 @@ aarch64-apple-darwin:
   host_platforms:
     - darwin
   needs:
+    - bzip2
     - libedit
     - libffi
     - ncurses
     - sqlite
     - tix
     - uuid
+    - xz
+    - zlib
 
 aarch64-apple-ios:
   host_platforms:
     - darwin
   needs:
+    - bzip2
+    - libffi
     - sqlite
+    - xz
+    - zlib
 
 arm64-apple-tvos:
   host_platforms:
     - darwin
   needs:
+    - bzip2
     - sqlite
+    - xz
+    - zlib
 
 thumb7k-apple-watchos:
   host_platforms:
     - darwin
   needs:
+    - bzip2
     - sqlite
+    - xz
+    - zlib
 
 x86_64-apple-darwin:
   host_platforms:
     - darwin
   needs:
+    - bzip2
     - libedit
     - libffi
     - ncurses
     - sqlite
     - tix
     - uuid
+    - xz
+    - zlib
 
 x86_64-apple-ios:
   host_platforms:
     - darwin
   needs:
+    - bzip2
+    - libffi
     - sqlite
+    - xz
+    - zlib
 
 x86_64-apple-tvos:
   host_platforms:
     - darwin
   needs:
+    - bzip2
     - sqlite
+    - xz
 
 x86_64-apple-watchos:
   host_platforms:
     - darwin
   needs:
+    - bzip2
     - sqlite
+    - xz
+    - zlib
 
 x86_64-unknown-linux-gnu:
   host_platforms:
     - linux
   needs:
     - bdb
     - binutils
+    - bzip2
     - gcc
     - gdbm
     - libedit
     - libffi
+    - libX11
+    - libXau
+    - libxcb
     - ncurses
-    - x11
     - readline
     - sqlite
+    - tcl
+    - tk
     - tix
     - uuid
+    - xorgproto
+    - xz
+    - zlib
 
 x86_64-unknown-linux-musl:
   host_platforms:
     - linux
   needs:
     - bdb
     - binutils
+    - bzip2
     - gcc
     - gdbm
     - libedit
     - libffi
+    - libX11
+    - libXau
+    - libxcb
     - musl
     - ncurses
-    - x11
     - readline
     - sqlite
+    - tcl
+    - tk
     - tix
     - uuid
+    - xorgproto
+    - xz
+    - zlib

pythonbuild/utils.py

@@ -38,6 +38,20 @@ def supported_targets(yaml_path: pathlib.Path):
     return targets
 
 
+def target_needs(yaml_path: pathlib.Path, target: str):
+    """Obtain the dependencies needed to build the specified target."""
+    settings = get_targets(yaml_path)[target]
+
+    needs = set(settings["needs"])
+
+    if "PYBUILD_LIBRESSL" in os.environ:
+        needs.add("libressl")
+    else:
+        needs.add("openssl")
+
+    return needs
+
+
 def release_tag_from_git():
     return (
         subprocess.check_output(