Perform pip install as part of bootstrap.py

Also, install pip packages in local `out/python_deps` directory instead
of installing system wide.

This means we can consistently expect dev dependencies to be available
in test code without needing to include an opt out mechanism.

We were already doing this for `psutil`, but for `websockify` we made
it optional.

This means that only python scripts that explicitly add
`out/python_deps` to their python path will be able use
the packages, and in particular it means that the emscripten compiler
itself won't end up implicitly/accidentally depending on them.

Author: sbc100

.circleci/config.yml

@@ -170,7 +170,6 @@ commands:
           name: clear cache
           command: |
             ./emcc --clear-cache
-      - pip-install
       - run: apt-get install -q -y ninja-build
       - run:
           name: embuilder build ALL
@@ -216,12 +215,8 @@ commands:
           # Must be absolute path or relative path from working_directory
           at: ~/
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
       - emsdk-env
       - bootstrap
-      - pip-install
       - when:
           # We only set EMTEST_RETRY_FLAKY on pull requests.  When we run
           # normal CI jobs on branches like main we still want to be able to
@@ -339,9 +334,6 @@ commands:
           at: ~/
       - checkout
       - remove-linux-binaries
-      - run:
-          name: submodule update
-          command: git submodule update --init
   run-tests-firefox:
     description: "Runs emscripten tests under firefox"
     parameters:
@@ -791,10 +783,6 @@ jobs:
     executor: linux-python
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       - run:
           name: install jsc
@@ -816,10 +804,6 @@ jobs:
     executor: linux-python
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       - run:
           name: install spidermonkey
@@ -846,10 +830,6 @@ jobs:
       EMTEST_SKIP_V8: "1"
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       # `install-node-version` only changes the NODE_JS_TEST (the version of
       # node used to run test), not NODE_JS (the version of node used to run the
@@ -989,10 +969,6 @@ jobs:
       EMTEST_LACKS_WEBGPU: "1"
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       - run-tests-chrome:
           title: "browser"
@@ -1082,10 +1058,6 @@ jobs:
     executor: focal
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       - run-tests-firefox:
           title: "browser64"
@@ -1114,8 +1086,6 @@ jobs:
           name: Add python to bash path
           command: echo "export PATH=\"$PATH:/c/Python27amd64/\"" >> $BASH_ENV
       - install-emsdk
-      - pip-install:
-          python: "$EMSDK_PYTHON"
       - run-tests-firefox-windows:
           title: "browser on firefox on windows"
           # skip browser.test_glbook, as it requires mingw32-make, which is not
@@ -1178,8 +1148,6 @@ jobs:
       # note we do *not* build all libraries and freeze the cache; as we run
       # only limited tests here, it's more efficient to build on demand
       - install-emsdk
-      - pip-install:
-          python: "$EMSDK_PYTHON"
       - run-tests:
           title: "crossplatform tests"
           test_targets: "--crossplatform-only"
@@ -1207,8 +1175,6 @@ jobs:
       - setup-macos
       - install-emsdk
       - freeze-cache
-      # TODO: We can't currently do pip install here since numpy and other packages
-      # are currently missing arm64 macos binaries.
       - run-tests:
           title: "crossplatform tests"
           test_targets: "--crossplatform-only"

bootstrap.py

@@ -39,6 +39,9 @@
      'test/third_party/googletest',
      'test/third_party/wasi-test-suite',
    ], ['git', 'submodule', 'update', '--init']),
+  ('pip3 install', [
+     'requirements-dev.txt',
+   ], [sys.executable, '-m', 'pip', 'install', '--target', 'out/python_deps', '-r', 'requirements-dev.txt']),
 ]
 
 

test/runner.py

@@ -43,8 +43,15 @@
 from common import errlog
 from tools import shared, config, utils
 
+# Add `out/python_deps` to pythoon path and make sure we can import out dev dependencies.
+sys.path.insert(0, utils.path_from_root('out/python_deps'))
+
+try:
+  import psutil  # noqa: F401
+  import websockify  # noqa: F401
+except ModuleNotFoundError as e:
+  raise Exception('Unable to import python dev dependencies (psutil/websockify). Run "./bootstrap" (or "python3 -m pip -r requirements-dev.txt --target out/python_deps") to install') from e
 
-sys.path.append(utils.path_from_root('third_party/websockify'))
 
 logger = logging.getLogger("runner")
 

test/test_sockets.py

@@ -9,6 +9,7 @@
 import shutil
 import sys
 import time
+import websockify
 from subprocess import Popen
 from typing import List
 
@@ -25,22 +26,9 @@
 
 npm_checked = False
 
-EMTEST_SKIP_PYTHON_DEV_PACKAGES = int(os.getenv('EMTEST_SKIP_PYTHON_DEV_PACKAGES', '0'))
 EMTEST_SKIP_NODE_DEV_PACKAGES = int(os.getenv('EMTEST_SKIP_NODE_DEV_PACKAGES', '0'))
 
 
-def requires_python_dev_packages(func):
-  assert callable(func)
-
-  @common.wraps(func)
-  def decorated(self, *args, **kwargs):
-    if EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      return self.skipTest('python websockify based tests are disabled by EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
-    return func(self, *args, **kwargs)
-
-  return decorated
-
-
 def clean_processes(processes):
   for p in processes:
     if getattr(p, 'exitcode', None) is None and getattr(p, 'returncode', None) is None:
@@ -77,11 +65,6 @@ def __enter__(self):
       process = Popen([os.path.abspath('server')])
       self.processes.append(process)
 
-    try:
-      import websockify  # type: ignore
-    except ModuleNotFoundError:
-      raise Exception('Unable to import module websockify. Run "python3 -m pip install websockify" or set environment variable EMTEST_SKIP_PYTHON_DEV_PACKAGES=1 to skip this test.') from None
-
     # start the websocket proxy
     print('running websockify on %d, forward to tcp %d' % (self.listen_port, self.target_port), file=sys.stderr)
     # source_is_ipv6=True here signals to websockify that it should prefer ipv6 address when
@@ -202,8 +185,6 @@ def setUpClass(cls):
   def test_sockets_echo(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
 
@@ -230,8 +211,6 @@ def test_sdl2_sockets_echo(self):
   def test_sockets_async_echo(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
 
@@ -252,8 +231,6 @@ def test_sockets_async_bad_port(self):
   def test_sockets_echo_bigdata(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
     sockets_include = '-I' + test_file('sockets')
@@ -271,7 +248,6 @@ def test_sockets_echo_bigdata(self, harness_class, port, args):
       self.btest_exit('test_sockets_echo_bigdata.c', cflags=[sockets_include, '-DSOCKK=%d' % harness.listen_port] + args)
 
   @no_windows('This test is Unix-specific.')
-  @requires_python_dev_packages
   @requires_dev_dependency('ws')
   def test_sockets_partial(self):
     for harness in [
@@ -282,7 +258,6 @@ def test_sockets_partial(self):
         self.btest_exit('sockets/test_sockets_partial_client.c', assert_returncode=165, cflags=['-DSOCKK=%d' % harness.listen_port])
 
   @no_windows('This test is Unix-specific.')
-  @requires_python_dev_packages
   @requires_dev_dependency('ws')
   def test_sockets_select_server_down(self):
     for harness in [
@@ -293,7 +268,6 @@ def test_sockets_select_server_down(self):
         self.btest_exit('sockets/test_sockets_select_server_down_client.c', cflags=['-DSOCKK=%d' % harness.listen_port])
 
   @no_windows('This test is Unix-specific.')
-  @requires_python_dev_packages
   @requires_dev_dependency('ws')
   def test_sockets_select_server_closes_connection_rw(self):
     for harness in [
@@ -326,8 +300,6 @@ def test_enet(self):
   def test_nodejs_sockets_echo(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
 
@@ -340,7 +312,6 @@ def test_nodejs_sockets_connect_failure(self):
     self.do_runf('sockets/test_sockets_echo_client.c', r'connect failed: (Connection refused|Host is unreachable)', regex=True, cflags=['-DSOCKK=666'], assert_returncode=NON_ZERO)
 
   @requires_native_clang
-  @requires_python_dev_packages
   def test_nodejs_sockets_echo_subprotocol(self):
     # Test against a Websockified server with compile time configured WebSocket subprotocol. We use a Websockified
     # server because as long as the subprotocol list contains binary it will configure itself to accept binary
@@ -353,7 +324,6 @@ def test_nodejs_sockets_echo_subprotocol(self):
       self.assertContained(['connect: ws://127.0.0.1:59168, base64,binary', 'connect: ws://127.0.0.1:59168/, base64,binary'], out)
 
   @requires_native_clang
-  @requires_python_dev_packages
   def test_nodejs_sockets_echo_subprotocol_runtime(self):
     # Test against a Websockified server with runtime WebSocket configuration. We specify both url and subprotocol.
     # In this test we have *deliberately* used the wrong port '-DSOCKK=12345' to configure the echo_client.c, so