Support for seed packages having dependencies

- bump setuptools/wheel
- no OS dependent dependencies allowed

Signed-off-by: Bernat Gabor <[email protected]>

Author: gaborbernat

src/virtualenv/seed/embed/base_embed.py

@@ -4,6 +4,7 @@
 
 from six import add_metaclass
 
+from virtualenv.seed.wheels.embed import get_wheel_deps
 from virtualenv.util.path import Path
 from virtualenv.util.six import ensure_str, ensure_text
 
@@ -49,6 +50,13 @@ def distribution_to_versions(self):
             if getattr(self, "no_{}".format(distribution)) is False
         }
 
+    def get_expanded_distributions(self, creator):
+        distributions = self.distribution_to_versions()
+        for dist, value in list(distributions.items()):
+            for dep in get_wheel_deps(dist, creator.interpreter.version_release_str):
+                distributions[dep] = value
+        return distributions
+
     @classmethod
     def add_parser_arguments(cls, parser, interpreter, app_data):
         group = parser.add_mutually_exclusive_group()

src/virtualenv/seed/embed/pip_invoke.py

@@ -18,7 +18,7 @@ def run(self, creator):
         if not self.enabled:
             return
         for_py_version = creator.interpreter.version_release_str
-        with self.get_pip_install_cmd(creator.exe, for_py_version) as cmd:
+        with self.get_pip_install_cmd(creator.exe, for_py_version, creator) as cmd:
             env = pip_wheel_env_run(self.extra_search_dir, self.app_data)
             self._execute(cmd, env)
 
@@ -32,12 +32,12 @@ def _execute(cmd, env):
         return process
 
     @contextmanager
-    def get_pip_install_cmd(self, exe, for_py_version):
+    def get_pip_install_cmd(self, exe, for_py_version, creator):
         cmd = [str(exe), "-m", "pip", "-q", "install", "--only-binary", ":all:", "--disable-pip-version-check"]
         if not self.download:
             cmd.append("--no-index")
         folders = set()
-        for dist, version in self.distribution_to_versions().items():
+        for dist, version in self.get_expanded_distributions(creator).items():
             wheel = get_wheel(
                 distribution=dist,
                 version=version,

src/virtualenv/seed/embed/via_app_data/via_app_data.py

@@ -117,9 +117,9 @@ def _get(distribution, version):
                 with lock:
                     name_to_whl[distribution] = result
 
+        distributions = self.get_expanded_distributions(creator)
         threads = list(
-            Thread(target=_get, args=(distribution, version))
-            for distribution, version in self.distribution_to_versions().items()
+            Thread(target=_get, args=(distribution, version)) for distribution, version in distributions.items()
         )
         for thread in threads:
             thread.start()

src/virtualenv/seed/wheels/embed/__init__.py

@@ -47,6 +47,16 @@
     },
 }
 MAX = "3.10"
+VERSION_DEPS = {
+    "3.10": {"pip": [], "setuptools": [], "wheel": []},
+    "3.9": {"pip": [], "setuptools": [], "wheel": []},
+    "3.8": {"pip": [], "setuptools": [], "wheel": []},
+    "3.7": {"pip": [], "setuptools": [], "wheel": []},
+    "3.6": {"pip": [], "setuptools": [], "wheel": []},
+    "3.5": {"pip": [], "setuptools": [], "wheel": []},
+    "3.4": {"pip": [], "setuptools": [], "wheel": []},
+    "2.7": {"pip": [], "setuptools": [], "wheel": []},
+}
 
 
 def get_embed_wheel(distribution, for_py_version):
@@ -57,6 +67,7 @@ def get_embed_wheel(distribution, for_py_version):
 __all__ = (
     "get_embed_wheel",
     "BUNDLE_SUPPORT",
+    "VERSION_DEPS",
     "MAX",
     "BUNDLE_FOLDER",
 )

src/virtualenv/seed/wheels/periodic_update.py

@@ -46,10 +46,12 @@ def periodic_update(distribution, for_py_version, wheel, search_dirs, app_data,
     u_log_older_than_hour = now - u_log.completed > timedelta(hours=1) if u_log.completed is not None else False
     for _, group in groupby(u_log.versions, key=lambda v: v.wheel.version_tuple[0:2]):
         version = next(group)  # use only latest patch version per minor, earlier assumed to be buggy
-        if wheel is not None and Path(version.filename).name == wheel.name:
+        updated_wheel = Wheel(app_data.house / version.filename)
+        if wheel is not None and (
+            Path(version.filename).name == wheel.name or wheel.version_tuple > updated_wheel.version_tuple
+        ):
             break
         if u_log.periodic is False or (u_log_older_than_hour and version.use(now)):
-            updated_wheel = Wheel(app_data.house / version.filename)
             logging.debug("using %supdated wheel %s", "periodically " if updated_wheel else "", updated_wheel)
             wheel = updated_wheel
             break

tasks/update_embedded.py

@@ -7,11 +7,13 @@
 import shutil
 import subprocess
 import sys
-from collections import OrderedDict, defaultdict
+from collections import defaultdict
+from concurrent.futures import as_completed
+from concurrent.futures.thread import ThreadPoolExecutor
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from textwrap import dedent
-from threading import Thread
+from typing import Dict
 
 STRICT = "UPGRADE_ADVISORY" not in os.environ
 
@@ -20,43 +22,62 @@
 DEST = Path(__file__).resolve().parents[1] / "src" / "virtualenv" / "seed" / "wheels" / "embed"
 
 
-def download(ver, dest, package):
-    subprocess.call(
-        [
-            sys.executable,
-            "-m",
-            "pip",
-            "--disable-pip-version-check",
-            "download",
-            "--only-binary=:all:",
-            "--python-version",
-            ver,
-            "-d",
-            dest,
-            package,
-        ],
-    )
+def download(ver: str, into: Path):
+    dest = into / ver
+    dest.mkdir()
+    cmd = [
+        sys.executable,
+        "-m",
+        "pip",
+        "--disable-pip-version-check",
+        "download",
+        "--only-binary=:all:",
+        "--python-version",
+        ver,
+    ]
+    subprocess.check_call(cmd + ["-d", str(dest), *BUNDLED])
+    deps = {}
+    for pkg in BUNDLED:
+        to_folder = dest / pkg
+        to_folder.mkdir()
+        try:
+            subprocess.check_call(cmd + ["-d", str(to_folder), pkg])
+            found = sorted(list(wheel.name.split("-")[0] for wheel in to_folder.iterdir()))
+            found.remove(pkg)
+            deps[pkg] = found
+        finally:
+            shutil.rmtree(to_folder)
+    return ver, dest, deps
 
 
 def run():
+    if sys.version_info < (3, 7):
+        raise RuntimeError("requires python 3.7 or later")
     old_batch = {i.name for i in DEST.iterdir() if i.suffix == ".whl"}
     with TemporaryDirectory() as temp:
-        temp_path = Path(temp)
-        folders = {}
-        targets = []
-        for support in SUPPORT:
-            support_ver = ".".join(str(i) for i in support)
-            into = temp_path / support_ver
-            into.mkdir()
-            folders[into] = support_ver
-            for package in BUNDLED:
-                thread = Thread(target=download, args=(support_ver, str(into), package))
-                targets.append(thread)
-                thread.start()
-        for thread in targets:
-            thread.join()
-        new_batch = {i.name: i for f in folders.keys() for i in Path(f).iterdir()}
-
+        temp = Path(temp)
+        bundle_support: Dict[str, Dict[str, str]] = {}
+        version_deps = {}
+        new_batch = {}
+        with ThreadPoolExecutor(max_workers=4) as executor:
+            for future in as_completed({executor.submit(download, ".".join(str(i) for i in s), temp) for s in SUPPORT}):
+                try:
+                    version, dest, deps = future.result()
+                except Exception as exc:
+                    raise exc
+                else:
+                    bundle = {}
+                    for wheel in dest.iterdir():
+                        name = wheel.name.split("-")[0]
+                        bundle[name] = wheel.name
+                        new_batch[wheel.name] = wheel
+                    wheels = {wheel.name.split("-")[0]: wheel.name for wheel in dest.iterdir()}
+                    wheels = {k: v for k, v in sorted(wheels.items())}
+                    bundle_support[version] = wheels
+                    version_deps[version] = deps
+        sort_by_version = lambda i: tuple(int(j) for j in i[0].split("."))  # noqa
+        bundle_support = {k: v for k, v in sorted(bundle_support.items(), key=sort_by_version, reverse=True)}
+        version_deps = {k: v for k, v in sorted(version_deps.items(), key=sort_by_version, reverse=True)}
         new_packages = new_batch.keys() - old_batch
         remove_packages = old_batch - new_batch.keys()
 
@@ -78,13 +99,6 @@ def run():
         for key, versions in removed.items():
             print("* removed embedded {} of {}".format(key, fmt_version(versions)))
 
-        support_table = OrderedDict((".".join(str(j) for j in i), list()) for i in SUPPORT)
-        for package in sorted(new_batch.keys()):
-            for folder, version in sorted(folders.items()):
-                if (folder / package).exists():
-                    support_table[version].append(package)
-        support_table = {k: OrderedDict((i.split("-")[0], i) for i in v) for k, v in support_table.items()}
-
         msg = dedent(
             """
         from __future__ import absolute_import, unicode_literals
@@ -93,8 +107,9 @@ def run():
         from virtualenv.util.path import Path
 
         BUNDLE_FOLDER = Path(__file__).absolute().parent
-        BUNDLE_SUPPORT = {{ {0} }}
+        BUNDLE_SUPPORT = {0}
         MAX = {1}
+        VERSION_DEPS = {2}
 
 
         def get_embed_wheel(distribution, for_py_version):
@@ -105,16 +120,13 @@ def get_embed_wheel(distribution, for_py_version):
         __all__ = (
             "get_embed_wheel",
             "BUNDLE_SUPPORT",
+            "VERSION_DEPS",
             "MAX",
             "BUNDLE_FOLDER",
         )
 
         """.format(
-                ",".join(
-                    "{!r}: {{ {} }}".format(v, ",".join("{!r}: {!r}".format(p, f) for p, f in l.items()))
-                    for v, l in support_table.items()
-                ),
-                repr(next(iter(support_table.keys()))),
+                repr(bundle_support), repr(next(iter(bundle_support.keys()))), repr(version_deps),
             ),
         )
         dest_target = DEST / "__init__.py"