From 0bc31adc9306d6db76296723e6adbe75c34c4094 Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Wed, 10 Sep 2025 09:53:39 -0700 Subject: [PATCH] Perform `pip install` as part of `bootstrap.py` Also, install pip packages in local `out/python_deps` directory instead of installing system wide. This means we can consistently expect dev dependencies to be available in test code without needing to include an opt out mechanism. We were already doing this for `psutil`, but for `websockify` we made it optional. This means that only python scripts that explicitly add `out/python_deps` to their python path will be able use the packages, and in particular it means that the emscripten compiler itself won't end up implicitly/accidentally depending on them. --- .circleci/config.yml | 34 ---------------------------------- bootstrap.py | 3 +++ test/runner.py | 9 ++++++++- test/test_sockets.py | 32 +------------------------------- 4 files changed, 12 insertions(+), 66 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 189927694f601..726545e883349 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -170,7 +170,6 @@ commands: name: clear cache command: | ./emcc --clear-cache - - pip-install - run: apt-get install -q -y ninja-build - run: name: embuilder build ALL @@ -216,12 +215,8 @@ commands: # Must be absolute path or relative path from working_directory at: ~/ - checkout - - run: - name: submodule update - command: git submodule update --init - emsdk-env - bootstrap - - pip-install - when: # We only set EMTEST_RETRY_FLAKY on pull requests. When we run # normal CI jobs on branches like main we still want to be able to @@ -339,9 +334,6 @@ commands: at: ~/ - checkout - remove-linux-binaries - - run: - name: submodule update - command: git submodule update --init run-tests-firefox: description: "Runs emscripten tests under firefox" parameters: @@ -791,10 +783,6 @@ jobs: executor: linux-python steps: - checkout - - run: - name: submodule update - command: git submodule update --init - - pip-install - install-emsdk - run: name: install jsc @@ -816,10 +804,6 @@ jobs: executor: linux-python steps: - checkout - - run: - name: submodule update - command: git submodule update --init - - pip-install - install-emsdk - run: name: install spidermonkey @@ -846,10 +830,6 @@ jobs: EMTEST_SKIP_V8: "1" steps: - checkout - - run: - name: submodule update - command: git submodule update --init - - pip-install - install-emsdk # `install-node-version` only changes the NODE_JS_TEST (the version of # node used to run test), not NODE_JS (the version of node used to run the @@ -989,10 +969,6 @@ jobs: EMTEST_LACKS_WEBGPU: "1" steps: - checkout - - run: - name: submodule update - command: git submodule update --init - - pip-install - install-emsdk - run-tests-chrome: title: "browser" @@ -1082,10 +1058,6 @@ jobs: executor: focal steps: - checkout - - run: - name: submodule update - command: git submodule update --init - - pip-install - install-emsdk - run-tests-firefox: title: "browser64" @@ -1114,8 +1086,6 @@ jobs: name: Add python to bash path command: echo "export PATH=\"$PATH:/c/Python27amd64/\"" >> $BASH_ENV - install-emsdk - - pip-install: - python: "$EMSDK_PYTHON" - run-tests-firefox-windows: title: "browser on firefox on windows" # skip browser.test_glbook, as it requires mingw32-make, which is not @@ -1178,8 +1148,6 @@ jobs: # note we do *not* build all libraries and freeze the cache; as we run # only limited tests here, it's more efficient to build on demand - install-emsdk - - pip-install: - python: "$EMSDK_PYTHON" - run-tests: title: "crossplatform tests" test_targets: "--crossplatform-only" @@ -1207,8 +1175,6 @@ jobs: - setup-macos - install-emsdk - freeze-cache - # TODO: We can't currently do pip install here since numpy and other packages - # are currently missing arm64 macos binaries. - run-tests: title: "crossplatform tests" test_targets: "--crossplatform-only" diff --git a/bootstrap.py b/bootstrap.py index 059415053c5ed..5cd39928fb544 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -39,6 +39,9 @@ 'test/third_party/googletest', 'test/third_party/wasi-test-suite', ], ['git', 'submodule', 'update', '--init']), + ('pip3 install', [ + 'requirements-dev.txt', + ], [sys.executable, '-m', 'pip', 'install', '--target', 'out/python_deps', '-r', 'requirements-dev.txt']), ] diff --git a/test/runner.py b/test/runner.py index ce76aba756863..cec91d50aae14 100755 --- a/test/runner.py +++ b/test/runner.py @@ -43,8 +43,15 @@ from common import errlog from tools import shared, config, utils +# Add `out/python_deps` to pythoon path and make sure we can import out dev dependencies. +sys.path.insert(0, utils.path_from_root('out/python_deps')) + +try: + import psutil # noqa: F401 + import websockify # noqa: F401 +except ModuleNotFoundError as e: + raise Exception('Unable to import python dev dependencies (psutil/websockify). Run "./bootstrap" (or "python3 -m pip -r requirements-dev.txt --target out/python_deps") to install') from e -sys.path.append(utils.path_from_root('third_party/websockify')) logger = logging.getLogger("runner") diff --git a/test/test_sockets.py b/test/test_sockets.py index ded5493a8964d..379f328d442ce 100644 --- a/test/test_sockets.py +++ b/test/test_sockets.py @@ -9,6 +9,7 @@ import shutil import sys import time +import websockify # type: ignore from subprocess import Popen from typing import List @@ -25,22 +26,9 @@ npm_checked = False -EMTEST_SKIP_PYTHON_DEV_PACKAGES = int(os.getenv('EMTEST_SKIP_PYTHON_DEV_PACKAGES', '0')) EMTEST_SKIP_NODE_DEV_PACKAGES = int(os.getenv('EMTEST_SKIP_NODE_DEV_PACKAGES', '0')) -def requires_python_dev_packages(func): - assert callable(func) - - @common.wraps(func) - def decorated(self, *args, **kwargs): - if EMTEST_SKIP_PYTHON_DEV_PACKAGES: - return self.skipTest('python websockify based tests are disabled by EMTEST_SKIP_PYTHON_DEV_PACKAGES=1') - return func(self, *args, **kwargs) - - return decorated - - def clean_processes(processes): for p in processes: if getattr(p, 'exitcode', None) is None and getattr(p, 'returncode', None) is None: @@ -77,11 +65,6 @@ def __enter__(self): process = Popen([os.path.abspath('server')]) self.processes.append(process) - try: - import websockify # type: ignore - except ModuleNotFoundError: - raise Exception('Unable to import module websockify. Run "python3 -m pip install websockify" or set environment variable EMTEST_SKIP_PYTHON_DEV_PACKAGES=1 to skip this test.') from None - # start the websocket proxy print('running websockify on %d, forward to tcp %d' % (self.listen_port, self.target_port), file=sys.stderr) # source_is_ipv6=True here signals to websockify that it should prefer ipv6 address when @@ -202,8 +185,6 @@ def setUpClass(cls): def test_sockets_echo(self, harness_class, port, args): if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG: self.skipTest('requires native clang') - if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES: - self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1') if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES: self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1') @@ -230,8 +211,6 @@ def test_sdl2_sockets_echo(self): def test_sockets_async_echo(self, harness_class, port, args): if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG: self.skipTest('requires native clang') - if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES: - self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1') if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES: self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1') @@ -252,8 +231,6 @@ def test_sockets_async_bad_port(self): def test_sockets_echo_bigdata(self, harness_class, port, args): if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG: self.skipTest('requires native clang') - if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES: - self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1') if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES: self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1') sockets_include = '-I' + test_file('sockets') @@ -271,7 +248,6 @@ def test_sockets_echo_bigdata(self, harness_class, port, args): self.btest_exit('test_sockets_echo_bigdata.c', cflags=[sockets_include, '-DSOCKK=%d' % harness.listen_port] + args) @no_windows('This test is Unix-specific.') - @requires_python_dev_packages @requires_dev_dependency('ws') def test_sockets_partial(self): for harness in [ @@ -282,7 +258,6 @@ def test_sockets_partial(self): self.btest_exit('sockets/test_sockets_partial_client.c', assert_returncode=165, cflags=['-DSOCKK=%d' % harness.listen_port]) @no_windows('This test is Unix-specific.') - @requires_python_dev_packages @requires_dev_dependency('ws') def test_sockets_select_server_down(self): for harness in [ @@ -293,7 +268,6 @@ def test_sockets_select_server_down(self): self.btest_exit('sockets/test_sockets_select_server_down_client.c', cflags=['-DSOCKK=%d' % harness.listen_port]) @no_windows('This test is Unix-specific.') - @requires_python_dev_packages @requires_dev_dependency('ws') def test_sockets_select_server_closes_connection_rw(self): for harness in [ @@ -326,8 +300,6 @@ def test_enet(self): def test_nodejs_sockets_echo(self, harness_class, port, args): if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG: self.skipTest('requires native clang') - if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES: - self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1') if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES: self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1') @@ -340,7 +312,6 @@ def test_nodejs_sockets_connect_failure(self): self.do_runf('sockets/test_sockets_echo_client.c', r'connect failed: (Connection refused|Host is unreachable)', regex=True, cflags=['-DSOCKK=666'], assert_returncode=NON_ZERO) @requires_native_clang - @requires_python_dev_packages def test_nodejs_sockets_echo_subprotocol(self): # Test against a Websockified server with compile time configured WebSocket subprotocol. We use a Websockified # server because as long as the subprotocol list contains binary it will configure itself to accept binary @@ -353,7 +324,6 @@ def test_nodejs_sockets_echo_subprotocol(self): self.assertContained(['connect: ws://127.0.0.1:59168, base64,binary', 'connect: ws://127.0.0.1:59168/, base64,binary'], out) @requires_native_clang - @requires_python_dev_packages def test_nodejs_sockets_echo_subprotocol_runtime(self): # Test against a Websockified server with runtime WebSocket configuration. We specify both url and subprotocol. # In this test we have *deliberately* used the wrong port '-DSOCKK=12345' to configure the echo_client.c, so