Perform pip install as part of bootstrap.py

.circleci/config.yml

@@ -170,7 +170,6 @@ commands:
           name: clear cache
           command: |
             ./emcc --clear-cache
-      - pip-install
       - run: apt-get install -q -y ninja-build
       - run:
           name: embuilder build ALL
@@ -216,12 +215,8 @@ commands:
           # Must be absolute path or relative path from working_directory
           at: ~/
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
       - emsdk-env
       - bootstrap
-      - pip-install
       - when:
           # We only set EMTEST_RETRY_FLAKY on pull requests.  When we run
           # normal CI jobs on branches like main we still want to be able to
@@ -339,9 +334,6 @@ commands:
           at: ~/
       - checkout
       - remove-linux-binaries
-      - run:
-          name: submodule update
-          command: git submodule update --init
   run-tests-firefox:
     description: "Runs emscripten tests under firefox"
     parameters:
@@ -791,10 +783,6 @@ jobs:
     executor: linux-python
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       - run:
           name: install jsc
@@ -816,10 +804,6 @@ jobs:
     executor: linux-python
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       - run:
           name: install spidermonkey
@@ -846,10 +830,6 @@ jobs:
       EMTEST_SKIP_V8: "1"
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       # `install-node-version` only changes the NODE_JS_TEST (the version of
       # node used to run test), not NODE_JS (the version of node used to run the
@@ -989,10 +969,6 @@ jobs:
       EMTEST_LACKS_WEBGPU: "1"
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       - run-tests-chrome:
           title: "browser"
@@ -1082,10 +1058,6 @@ jobs:
     executor: focal
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       - run-tests-firefox:
           title: "browser64"
@@ -1114,8 +1086,6 @@ jobs:
           name: Add python to bash path
           command: echo "export PATH=\"$PATH:/c/Python27amd64/\"" >> $BASH_ENV
       - install-emsdk
-      - pip-install:
-          python: "$EMSDK_PYTHON"
       - run-tests-firefox-windows:
           title: "browser on firefox on windows"
           # skip browser.test_glbook, as it requires mingw32-make, which is not
@@ -1178,8 +1148,6 @@ jobs:
       # note we do *not* build all libraries and freeze the cache; as we run
       # only limited tests here, it's more efficient to build on demand
       - install-emsdk
-      - pip-install:
-          python: "$EMSDK_PYTHON"
       - run-tests:
           title: "crossplatform tests"
           test_targets: "--crossplatform-only"
@@ -1207,8 +1175,6 @@ jobs:
       - setup-macos
       - install-emsdk
       - freeze-cache
-      # TODO: We can't currently do pip install here since numpy and other packages
-      # are currently missing arm64 macos binaries.
       - run-tests:
           title: "crossplatform tests"
           test_targets: "--crossplatform-only"

bootstrap.py

@@ -39,6 +39,9 @@
      'test/third_party/googletest',
      'test/third_party/wasi-test-suite',
    ], ['git', 'submodule', 'update', '--init']),
+  ('pip3 install', [
+     'requirements-dev.txt',
+   ], [sys.executable, '-m', 'pip', 'install', '--target', 'out/python_deps', '-r', 'requirements-dev.txt']),
 ]
 
 

test/runner.py

@@ -43,8 +43,15 @@
 from common import errlog
 from tools import shared, config, utils
 
+# Add `out/python_deps` to pythoon path and make sure we can import out dev dependencies.
+sys.path.insert(0, utils.path_from_root('out/python_deps'))
+
+try:
+  import psutil  # noqa: F401
+  import websockify  # noqa: F401
+except ModuleNotFoundError as e:
+  raise Exception('Unable to import python dev dependencies (psutil/websockify). Run "./bootstrap" (or "python3 -m pip -r requirements-dev.txt --target out/python_deps") to install') from e
 
-sys.path.append(utils.path_from_root('third_party/websockify'))
 
 logger = logging.getLogger("runner")
 

test/test_sockets.py

@@ -9,6 +9,7 @@
 import shutil
 import sys
 import time
+import websockify  # type: ignore
 from subprocess import Popen
 from typing import List
 
@@ -25,22 +26,9 @@
 
 npm_checked = False
 
-EMTEST_SKIP_PYTHON_DEV_PACKAGES = int(os.getenv('EMTEST_SKIP_PYTHON_DEV_PACKAGES', '0'))
 EMTEST_SKIP_NODE_DEV_PACKAGES = int(os.getenv('EMTEST_SKIP_NODE_DEV_PACKAGES', '0'))
 
 
-def requires_python_dev_packages(func):
-  assert callable(func)
-
-  @common.wraps(func)
-  def decorated(self, *args, **kwargs):
-    if EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      return self.skipTest('python websockify based tests are disabled by EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
-    return func(self, *args, **kwargs)
-
-  return decorated
-
-
 def clean_processes(processes):
   for p in processes:
     if getattr(p, 'exitcode', None) is None and getattr(p, 'returncode', None) is None:
@@ -77,11 +65,6 @@ def __enter__(self):
       process = Popen([os.path.abspath('server')])
       self.processes.append(process)
 
-    try:
-      import websockify  # type: ignore
-    except ModuleNotFoundError:
-      raise Exception('Unable to import module websockify. Run "python3 -m pip install websockify" or set environment variable EMTEST_SKIP_PYTHON_DEV_PACKAGES=1 to skip this test.') from None
-
     # start the websocket proxy
     print('running websockify on %d, forward to tcp %d' % (self.listen_port, self.target_port), file=sys.stderr)
     # source_is_ipv6=True here signals to websockify that it should prefer ipv6 address when
@@ -202,8 +185,6 @@ def setUpClass(cls):
   def test_sockets_echo(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
 
@@ -230,8 +211,6 @@ def test_sdl2_sockets_echo(self):
   def test_sockets_async_echo(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
 
@@ -252,8 +231,6 @@ def test_sockets_async_bad_port(self):
   def test_sockets_echo_bigdata(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
     sockets_include = '-I' + test_file('sockets')
@@ -271,7 +248,6 @@ def test_sockets_echo_bigdata(self, harness_class, port, args):
       self.btest_exit('test_sockets_echo_bigdata.c', cflags=[sockets_include, '-DSOCKK=%d' % harness.listen_port] + args)
 
   @no_windows('This test is Unix-specific.')
-  @requires_python_dev_packages
   @requires_dev_dependency('ws')
   def test_sockets_partial(self):
     for harness in [
@@ -282,7 +258,6 @@ def test_sockets_partial(self):
         self.btest_exit('sockets/test_sockets_partial_client.c', assert_returncode=165, cflags=['-DSOCKK=%d' % harness.listen_port])
 
   @no_windows('This test is Unix-specific.')
-  @requires_python_dev_packages
   @requires_dev_dependency('ws')
   def test_sockets_select_server_down(self):
     for harness in [
@@ -293,7 +268,6 @@ def test_sockets_select_server_down(self):
         self.btest_exit('sockets/test_sockets_select_server_down_client.c', cflags=['-DSOCKK=%d' % harness.listen_port])
 
   @no_windows('This test is Unix-specific.')
-  @requires_python_dev_packages
   @requires_dev_dependency('ws')
   def test_sockets_select_server_closes_connection_rw(self):
     for harness in [
@@ -326,8 +300,6 @@ def test_enet(self):
   def test_nodejs_sockets_echo(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
 
@@ -340,7 +312,6 @@ def test_nodejs_sockets_connect_failure(self):
     self.do_runf('sockets/test_sockets_echo_client.c', r'connect failed: (Connection refused|Host is unreachable)', regex=True, cflags=['-DSOCKK=666'], assert_returncode=NON_ZERO)
 
   @requires_native_clang
-  @requires_python_dev_packages
   def test_nodejs_sockets_echo_subprotocol(self):
     # Test against a Websockified server with compile time configured WebSocket subprotocol. We use a Websockified
     # server because as long as the subprotocol list contains binary it will configure itself to accept binary
@@ -353,7 +324,6 @@ def test_nodejs_sockets_echo_subprotocol(self):
       self.assertContained(['connect: ws://127.0.0.1:59168, base64,binary', 'connect: ws://127.0.0.1:59168/, base64,binary'], out)
 
   @requires_native_clang
-  @requires_python_dev_packages
   def test_nodejs_sockets_echo_subprotocol_runtime(self):
     # Test against a Websockified server with runtime WebSocket configuration. We specify both url and subprotocol.
     # In this test we have *deliberately* used the wrong port '-DSOCKK=12345' to configure the echo_client.c, so