diff --git a/.circleci/config.yml b/.circleci/config.yml index 189927694f601..726545e883349 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -170,7 +170,6 @@ commands: name: clear cache command: | ./emcc --clear-cache - - pip-install - run: apt-get install -q -y ninja-build - run: name: embuilder build ALL @@ -216,12 +215,8 @@ commands: # Must be absolute path or relative path from working_directory at: ~/ - checkout - - run: - name: submodule update - command: git submodule update --init - emsdk-env - bootstrap - - pip-install - when: # We only set EMTEST_RETRY_FLAKY on pull requests. When we run # normal CI jobs on branches like main we still want to be able to @@ -339,9 +334,6 @@ commands: at: ~/ - checkout - remove-linux-binaries - - run: - name: submodule update - command: git submodule update --init run-tests-firefox: description: "Runs emscripten tests under firefox" parameters: @@ -791,10 +783,6 @@ jobs: executor: linux-python steps: - checkout - - run: - name: submodule update - command: git submodule update --init - - pip-install - install-emsdk - run: name: install jsc @@ -816,10 +804,6 @@ jobs: executor: linux-python steps: - checkout - - run: - name: submodule update - command: git submodule update --init - - pip-install - install-emsdk - run: name: install spidermonkey @@ -846,10 +830,6 @@ jobs: EMTEST_SKIP_V8: "1" steps: - checkout - - run: - name: submodule update - command: git submodule update --init - - pip-install - install-emsdk # `install-node-version` only changes the NODE_JS_TEST (the version of # node used to run test), not NODE_JS (the version of node used to run the @@ -989,10 +969,6 @@ jobs: EMTEST_LACKS_WEBGPU: "1" steps: - checkout - - run: - name: submodule update - command: git submodule update --init - - pip-install - install-emsdk - run-tests-chrome: title: "browser" @@ -1082,10 +1058,6 @@ jobs: executor: focal steps: - checkout - - run: - name: submodule update - command: git submodule update --init - - pip-install - install-emsdk - run-tests-firefox: title: "browser64" @@ -1114,8 +1086,6 @@ jobs: name: Add python to bash path command: echo "export PATH=\"$PATH:/c/Python27amd64/\"" >> $BASH_ENV - install-emsdk - - pip-install: - python: "$EMSDK_PYTHON" - run-tests-firefox-windows: title: "browser on firefox on windows" # skip browser.test_glbook, as it requires mingw32-make, which is not @@ -1178,8 +1148,6 @@ jobs: # note we do *not* build all libraries and freeze the cache; as we run # only limited tests here, it's more efficient to build on demand - install-emsdk - - pip-install: - python: "$EMSDK_PYTHON" - run-tests: title: "crossplatform tests" test_targets: "--crossplatform-only" @@ -1207,8 +1175,6 @@ jobs: - setup-macos - install-emsdk - freeze-cache - # TODO: We can't currently do pip install here since numpy and other packages - # are currently missing arm64 macos binaries. - run-tests: title: "crossplatform tests" test_targets: "--crossplatform-only" diff --git a/bootstrap.py b/bootstrap.py index 059415053c5ed..5cd39928fb544 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -39,6 +39,9 @@ 'test/third_party/googletest', 'test/third_party/wasi-test-suite', ], ['git', 'submodule', 'update', '--init']), + ('pip3 install', [ + 'requirements-dev.txt', + ], [sys.executable, '-m', 'pip', 'install', '--target', 'out/python_deps', '-r', 'requirements-dev.txt']), ] diff --git a/test/runner.py b/test/runner.py index ce76aba756863..cec91d50aae14 100755 --- a/test/runner.py +++ b/test/runner.py @@ -43,8 +43,15 @@ from common import errlog from tools import shared, config, utils +# Add `out/python_deps` to pythoon path and make sure we can import out dev dependencies. +sys.path.insert(0, utils.path_from_root('out/python_deps')) + +try: + import psutil # noqa: F401 + import websockify # noqa: F401 +except ModuleNotFoundError as e: + raise Exception('Unable to import python dev dependencies (psutil/websockify). Run "./bootstrap" (or "python3 -m pip -r requirements-dev.txt --target out/python_deps") to install') from e -sys.path.append(utils.path_from_root('third_party/websockify')) logger = logging.getLogger("runner") diff --git a/test/test_sockets.py b/test/test_sockets.py index ded5493a8964d..379f328d442ce 100644 --- a/test/test_sockets.py +++ b/test/test_sockets.py @@ -9,6 +9,7 @@ import shutil import sys import time +import websockify # type: ignore from subprocess import Popen from typing import List @@ -25,22 +26,9 @@ npm_checked = False -EMTEST_SKIP_PYTHON_DEV_PACKAGES = int(os.getenv('EMTEST_SKIP_PYTHON_DEV_PACKAGES', '0')) EMTEST_SKIP_NODE_DEV_PACKAGES = int(os.getenv('EMTEST_SKIP_NODE_DEV_PACKAGES', '0')) -def requires_python_dev_packages(func): - assert callable(func) - - @common.wraps(func) - def decorated(self, *args, **kwargs): - if EMTEST_SKIP_PYTHON_DEV_PACKAGES: - return self.skipTest('python websockify based tests are disabled by EMTEST_SKIP_PYTHON_DEV_PACKAGES=1') - return func(self, *args, **kwargs) - - return decorated - - def clean_processes(processes): for p in processes: if getattr(p, 'exitcode', None) is None and getattr(p, 'returncode', None) is None: @@ -77,11 +65,6 @@ def __enter__(self): process = Popen([os.path.abspath('server')]) self.processes.append(process) - try: - import websockify # type: ignore - except ModuleNotFoundError: - raise Exception('Unable to import module websockify. Run "python3 -m pip install websockify" or set environment variable EMTEST_SKIP_PYTHON_DEV_PACKAGES=1 to skip this test.') from None - # start the websocket proxy print('running websockify on %d, forward to tcp %d' % (self.listen_port, self.target_port), file=sys.stderr) # source_is_ipv6=True here signals to websockify that it should prefer ipv6 address when @@ -202,8 +185,6 @@ def setUpClass(cls): def test_sockets_echo(self, harness_class, port, args): if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG: self.skipTest('requires native clang') - if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES: - self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1') if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES: self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1') @@ -230,8 +211,6 @@ def test_sdl2_sockets_echo(self): def test_sockets_async_echo(self, harness_class, port, args): if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG: self.skipTest('requires native clang') - if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES: - self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1') if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES: self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1') @@ -252,8 +231,6 @@ def test_sockets_async_bad_port(self): def test_sockets_echo_bigdata(self, harness_class, port, args): if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG: self.skipTest('requires native clang') - if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES: - self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1') if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES: self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1') sockets_include = '-I' + test_file('sockets') @@ -271,7 +248,6 @@ def test_sockets_echo_bigdata(self, harness_class, port, args): self.btest_exit('test_sockets_echo_bigdata.c', cflags=[sockets_include, '-DSOCKK=%d' % harness.listen_port] + args) @no_windows('This test is Unix-specific.') - @requires_python_dev_packages @requires_dev_dependency('ws') def test_sockets_partial(self): for harness in [ @@ -282,7 +258,6 @@ def test_sockets_partial(self): self.btest_exit('sockets/test_sockets_partial_client.c', assert_returncode=165, cflags=['-DSOCKK=%d' % harness.listen_port]) @no_windows('This test is Unix-specific.') - @requires_python_dev_packages @requires_dev_dependency('ws') def test_sockets_select_server_down(self): for harness in [ @@ -293,7 +268,6 @@ def test_sockets_select_server_down(self): self.btest_exit('sockets/test_sockets_select_server_down_client.c', cflags=['-DSOCKK=%d' % harness.listen_port]) @no_windows('This test is Unix-specific.') - @requires_python_dev_packages @requires_dev_dependency('ws') def test_sockets_select_server_closes_connection_rw(self): for harness in [ @@ -326,8 +300,6 @@ def test_enet(self): def test_nodejs_sockets_echo(self, harness_class, port, args): if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG: self.skipTest('requires native clang') - if harness_class == WebsockifyServerHarness and EMTEST_SKIP_PYTHON_DEV_PACKAGES: - self.skipTest('requires python websockify and EMTEST_SKIP_PYTHON_DEV_PACKAGES=1') if harness_class == CompiledServerHarness and EMTEST_SKIP_NODE_DEV_PACKAGES: self.skipTest('requires node ws and EMTEST_SKIP_NODE_DEV_PACKAGES=1') @@ -340,7 +312,6 @@ def test_nodejs_sockets_connect_failure(self): self.do_runf('sockets/test_sockets_echo_client.c', r'connect failed: (Connection refused|Host is unreachable)', regex=True, cflags=['-DSOCKK=666'], assert_returncode=NON_ZERO) @requires_native_clang - @requires_python_dev_packages def test_nodejs_sockets_echo_subprotocol(self): # Test against a Websockified server with compile time configured WebSocket subprotocol. We use a Websockified # server because as long as the subprotocol list contains binary it will configure itself to accept binary @@ -353,7 +324,6 @@ def test_nodejs_sockets_echo_subprotocol(self): self.assertContained(['connect: ws://127.0.0.1:59168, base64,binary', 'connect: ws://127.0.0.1:59168/, base64,binary'], out) @requires_native_clang - @requires_python_dev_packages def test_nodejs_sockets_echo_subprotocol_runtime(self): # Test against a Websockified server with runtime WebSocket configuration. We specify both url and subprotocol. # In this test we have *deliberately* used the wrong port '-DSOCKK=12345' to configure the echo_client.c, so