Don't require virtualenv binary

With Python 3 we can use python3 -m venv to create a virtualenv.

The main dififculty is that we're missing activate_this.py which we use
to activate the environment. But we can do the relevant path / env manipulations
ourselves and so cut down the number of dependencies.

Author: jgraham

.taskcluster.yml

@@ -57,7 +57,7 @@ tasks:
             owner: ${owner}
             source: ${event.repository.clone_url}
           payload:
-            image: webplatformtests/wpt:0.53
+            image: webplatformtests/wpt:0.54
             maxRunTime: 7200
             artifacts:
               public/results:

tools/ci/tc/tasks/test.yml

@@ -4,7 +4,7 @@ components:
     workerType: ci
     schedulerId: taskcluster-github
     deadline: "24 hours"
-    image: webplatformtests/wpt:0.53
+    image: webplatformtests/wpt:0.54
     maxRunTime: 7200
     artifacts:
       public/results:
@@ -116,6 +116,7 @@ components:
       - python3.7
       - python3.7-distutils
       - python3.7-dev
+      - python3.7-venv
 
   tox-python3_10:
     env:
@@ -125,6 +126,7 @@ components:
       - python3.10
       - python3.10-distutils
       - python3.10-dev
+      - python3.10-venv
 
   tests-affected:
     options:

tools/docker/Dockerfile

@@ -31,6 +31,7 @@ RUN apt-get -qqy update \
     python3 \
     python3-dev \
     python3-pip \
+    python3-venv \
     software-properties-common \
     qemu-kvm \
     tzdata \
@@ -64,7 +65,6 @@ RUN apt-get -qqy install \
 RUN apt-get -y autoremove
 
 RUN pip install --upgrade pip
-RUN pip install virtualenv
 
 ENV TZ "UTC"
 RUN echo "${TZ}" > /etc/timezone \

tools/wpt/virtualenv.py

@@ -1,9 +1,12 @@
 # mypy: allow-untyped-defs
 
+import logging
 import os
 import shutil
+import site
 import sys
-import logging
+import sysconfig
+from pathlib import Path
 from shutil import which
 
 # The `pkg_resources` module is provided by `setuptools`, which is itself a
@@ -27,9 +30,7 @@ def __init__(self, path, skip_virtualenv_setup):
         self.path = path
         self.skip_virtualenv_setup = skip_virtualenv_setup
         if not skip_virtualenv_setup:
-            self.virtualenv = which("virtualenv")
-            if not self.virtualenv:
-                raise ValueError("virtualenv must be installed and on the PATH")
+            self.virtualenv = [sys.executable, "-m", "venv"]
             self._working_set = None
 
     @property
@@ -47,7 +48,7 @@ def create(self):
         if os.path.exists(self.path):
             shutil.rmtree(self.path)
             self._working_set = None
-        call(self.virtualenv, self.path, "-p", sys.executable)
+        call(*self.virtualenv, self.path)
 
     @property
     def bin_path(self):
@@ -100,9 +101,37 @@ def activate(self):
             # https://github.com/web-platform-tests/wpt/issues/27377
             # https://github.com/python/cpython/pull/9516
             os.environ.pop('__PYVENV_LAUNCHER__', None)
-        path = os.path.join(self.bin_path, "activate_this.py")
-        with open(path) as f:
-            exec(f.read(), {"__file__": path})
+
+        # Setup the path and site packages as if we'd launched with the virtualenv active
+        bin_dir = os.path.join(self.path, "bin")
+        os.environ["PATH"] = os.pathsep.join([bin_dir] + os.environ.get("PATH", "").split(os.pathsep))
+        os.environ["VIRTUAL_ENV"] = self.path
+
+        prev_length = len(sys.path)
+
+        schemes = sysconfig.get_scheme_names()
+        if "venv" in schemes:
+            scheme = "venv"
+        else:
+            scheme = "nt_user" if os.name == "nt" else "posix_user"
+        sys_paths = sysconfig.get_paths(scheme)
+        data_path = sys_paths["data"]
+        added = set()
+        # Add the venv library paths as sitedirs.
+        # This converts system paths like /usr/local/lib/python3.10/site-packages
+        # to venv-relative paths like {self.path}/lib/python3.10/site-packages and adds
+        # those paths as site dirs to be used for module import.
+        for key in ["purelib", "platlib"]:
+            host_path = Path(sys_paths[key])
+            relative_path = host_path.relative_to(data_path)
+            site_dir = os.path.normpath(os.path.normcase(Path(self.path) / relative_path))
+            if site_dir not in added:
+                site.addsitedir(site_dir)
+                added.add(site_dir)
+        sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
+
+        sys.real_prefix = sys.prefix
+        sys.prefix = self.path
 
     def start(self):
         if not self.exists or self.broken_link: