Perform pip install as part of bootstrap.py

Also, install pip packages in local `out/python_deps` directory instead
of installing system wide.  This means that only python scripts that
explicitly add `out/python_deps` to their python path will be able use
the packages, and in particular it means that the emscripten compiler
itself won't end up implicitly/accidentally depending on them.

Author: sbc100

.circleci/config.yml

@@ -170,7 +170,6 @@ commands:
           name: clear cache
           command: |
             ./emcc --clear-cache
-      - pip-install
       - run: apt-get install -q -y ninja-build
       - run:
           name: embuilder build ALL
@@ -216,12 +215,8 @@ commands:
           # Must be absolute path or relative path from working_directory
           at: ~/
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
       - emsdk-env
       - bootstrap
-      - pip-install
       - when:
           # We only set EMTEST_RETRY_FLAKY on pull requests.  When we run
           # normal CI jobs on branches like main we still want to be able to
@@ -339,9 +334,6 @@ commands:
           at: ~/
       - checkout
       - remove-linux-binaries
-      - run:
-          name: submodule update
-          command: git submodule update --init
   run-tests-firefox:
     description: "Runs emscripten tests under firefox"
     parameters:
@@ -791,10 +783,6 @@ jobs:
     executor: linux-python
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       - run:
           name: install jsc
@@ -816,10 +804,6 @@ jobs:
     executor: linux-python
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       - run:
           name: install spidermonkey
@@ -846,10 +830,6 @@ jobs:
       EMTEST_SKIP_V8: "1"
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       # `install-node-version` only changes the NODE_JS_TEST (the version of
       # node used to run test), not NODE_JS (the version of node used to run the
@@ -989,10 +969,6 @@ jobs:
       EMTEST_LACKS_WEBGPU: "1"
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       - run-tests-chrome:
           title: "browser"
@@ -1082,10 +1058,6 @@ jobs:
     executor: focal
     steps:
       - checkout
-      - run:
-          name: submodule update
-          command: git submodule update --init
-      - pip-install
       - install-emsdk
       - run-tests-firefox:
           title: "browser64"
@@ -1114,8 +1086,6 @@ jobs:
           name: Add python to bash path
           command: echo "export PATH=\"$PATH:/c/Python27amd64/\"" >> $BASH_ENV
       - install-emsdk
-      - pip-install:
-          python: "$EMSDK_PYTHON"
       - run-tests-firefox-windows:
           title: "browser on firefox on windows"
           # skip browser.test_glbook, as it requires mingw32-make, which is not
@@ -1178,8 +1148,6 @@ jobs:
       # note we do *not* build all libraries and freeze the cache; as we run
       # only limited tests here, it's more efficient to build on demand
       - install-emsdk
-      - pip-install:
-          python: "$EMSDK_PYTHON"
       - run-tests:
           title: "crossplatform tests"
           test_targets: "--crossplatform-only"
@@ -1207,8 +1175,6 @@ jobs:
       - setup-macos
       - install-emsdk
       - freeze-cache
-      # TODO: We can't currently do pip install here since numpy and other packages
-      # are currently missing arm64 macos binaries.
       - run-tests:
           title: "crossplatform tests"
           test_targets: "--crossplatform-only"

bootstrap.py

@@ -39,6 +39,9 @@
      'test/third_party/googletest',
      'test/third_party/wasi-test-suite',
    ], ['git', 'submodule', 'update', '--init']),
+  ('pip3 install', [
+     'requirements-dev.txt',
+   ], [sys.executable, '-m', 'pip', 'install', '--target', 'out/python_deps', '-r', 'requirements-dev.txt']),
 ]
 
 

test/test_sockets.py

@@ -25,22 +25,9 @@
 
 npm_checked = False
 
-EMTEST_SKIP_PYTHON_DEV_PACKAGES = int(os.getenv('EMTEST_SKIP_PYTHON_DEV_PACKAGES', '0'))
 EMTEST_SKIP_NODE_DEV_PACKAGES = int(os.getenv('EMTEST_SKIP_NODE_DEV_PACKAGES', '0'))
 
 
-def requires_python_dev_packages(func):
-  assert callable(func)
-
-  @common.wraps(func)
-  def decorated(self, *args, **kwargs):
-    if EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      return self.skipTest('python websockify based tests are disabled by EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
-    return func(self, *args, **kwargs)
-
-  return decorated
-
-
 def clean_processes(processes):
   for p in processes:
     if getattr(p, 'exitcode', None) is None and getattr(p, 'returncode', None) is None:
@@ -57,6 +44,15 @@ def clean_processes(processes):
         pass
 
 
+def import_websockify():
+  sys.path.append(path_from_root('out/python_deps'))
+  try:
+    import websockify  # type: ignore
+    return websockify
+  except ModuleNotFoundError:
+    raise Exception('Unable to import module websockify. Run "./bootstrap" (or "python3 -m pip -r requirements-dev.txt --target out/python_deps" to install') from None
+
+
 class WebsockifyServerHarness:
   def __init__(self, filename, args, listen_port, do_server_check=True):
     self.processes = []
@@ -77,10 +73,7 @@ def __enter__(self):
       process = Popen([os.path.abspath('server')])
       self.processes.append(process)
 
-    try:
-      import websockify  # type: ignore
-    except ModuleNotFoundError:
-      raise Exception('Unable to import module websockify. Run "python3 -m pip install websockify" or set environment variable EMTEST_SKIP_PYTHON_DEV_PACKAGES=1 to skip this test.') from None
+    websockify = import_websockify()
 
     # start the websocket proxy
     print('running websockify on %d, forward to tcp %d' % (self.listen_port, self.target_port), file=sys.stderr)
@@ -202,8 +195,6 @@ def setUpClass(cls):
   def test_sockets_echo(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
 
@@ -230,8 +221,6 @@ def test_sdl2_sockets_echo(self):
   def test_sockets_async_echo(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
 
@@ -252,8 +241,6 @@ def test_sockets_async_bad_port(self):
   def test_sockets_echo_bigdata(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
     sockets_include = '-I' + test_file('sockets')
@@ -271,7 +258,6 @@ def test_sockets_echo_bigdata(self, harness_class, port, args):
       self.btest_exit('test_sockets_echo_bigdata.c', cflags=[sockets_include, '-DSOCKK=%d' % harness.listen_port] + args)
 
   @no_windows('This test is Unix-specific.')
-  @requires_python_dev_packages
   @requires_dev_dependency('ws')
   def test_sockets_partial(self):
     for harness in [
@@ -282,7 +268,6 @@ def test_sockets_partial(self):
         self.btest_exit('sockets/test_sockets_partial_client.c', assert_returncode=165, cflags=['-DSOCKK=%d' % harness.listen_port])
 
   @no_windows('This test is Unix-specific.')
-  @requires_python_dev_packages
   @requires_dev_dependency('ws')
   def test_sockets_select_server_down(self):
     for harness in [
@@ -293,7 +278,6 @@ def test_sockets_select_server_down(self):
         self.btest_exit('sockets/test_sockets_select_server_down_client.c', cflags=['-DSOCKK=%d' % harness.listen_port])
 
   @no_windows('This test is Unix-specific.')
-  @requires_python_dev_packages
   @requires_dev_dependency('ws')
   def test_sockets_select_server_closes_connection_rw(self):
     for harness in [
@@ -326,8 +310,6 @@ def test_enet(self):
   def test_nodejs_sockets_echo(self, harness_class, port, args):
     if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG:
       self.skipTest('requires native clang')
-    if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES:
-      self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1')
     if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES:
       self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1')
 
@@ -340,7 +322,6 @@ def test_nodejs_sockets_connect_failure(self):
     self.do_runf('sockets/test_sockets_echo_client.c', r'connect failed: (Connection refused|Host is unreachable)', regex=True, cflags=['-DSOCKK=666'], assert_returncode=NON_ZERO)
 
   @requires_native_clang
-  @requires_python_dev_packages
   def test_nodejs_sockets_echo_subprotocol(self):
     # Test against a Websockified server with compile time configured WebSocket subprotocol. We use a Websockified
     # server because as long as the subprotocol list contains binary it will configure itself to accept binary
@@ -353,7 +334,6 @@ def test_nodejs_sockets_echo_subprotocol(self):
       self.assertContained(['connect: ws://127.0.0.1:59168, base64,binary', 'connect: ws://127.0.0.1:59168/, base64,binary'], out)
 
   @requires_native_clang
-  @requires_python_dev_packages
   def test_nodejs_sockets_echo_subprotocol_runtime(self):
     # Test against a Websockified server with runtime WebSocket configuration. We specify both url and subprotocol.
     # In this test we have *deliberately* used the wrong port '-DSOCKK=12345' to configure the echo_client.c, so