From 9d3b8dcbf2df7d882a5ebfd00180d0aa4ebf867f Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 12 Oct 2024 21:28:30 -0500 Subject: [PATCH 01/10] Bump to Python 3.9+ support --- .evergreen/config.yml | 16 ++++++++-------- README.md | 4 ++-- docs/installing-running-locally.md | 2 +- docs/integration-guide.md | 12 ++++++------ pyproject.toml | 5 ++--- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index e61bd2f..e65c160 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -54,7 +54,7 @@ functions: continue_on_err: true command: | cat /etc/resolv.conf - # Create virtualenv using a CPython 3.8+ binary. + # Create virtualenv using a CPython 3.9+ binary. - command: subprocess.exec type: setup params: @@ -757,7 +757,7 @@ axes: run_on: windows-64-vsMulti-small batchtime: 10080 # 7 days variables: - PYTHON3_BINARY: "C:/python/Python38/python.exe" + PYTHON3_BINARY: "C:/python/Python39/python.exe" PYTHON_BIN_DIR: "Scripts" # The 'runtime' axis specifies the language runtime to use. @@ -765,14 +765,14 @@ axes: - id: runtime display_name: runtime values: - - id: python39 + - id: python3-linux display_name: CPython-3 variables: PYTHON_BINARY: "/opt/python/3.9/bin/python3" - - id: python38-windows - display_name: CPython-3.8-Windows + - id: python3-windows + display_name: CPython-3-Windows variables: - PYTHON_BINARY: "C:/python/Python38/python.exe" + PYTHON_BINARY: "C:/python/Python39/python.exe" - id: ruby-32 display_name: Ruby 3.2 variables: @@ -864,7 +864,7 @@ buildvariants: matrix_spec: driver: ["pymongo-master"] platform: ["ubuntu-22.04"] - runtime: ["python39"] + runtime: ["python3-linux"] display_name: "${driver} ${platform} ${runtime}" tasks: - ".all" @@ -872,7 +872,7 @@ buildvariants: matrix_spec: driver: ["pymongo-master"] platform: ["windows-64"] - runtime: ["python38-windows"] + runtime: ["python3-windows"] display_name: "${driver} ${platform} ${runtime}" tasks: # Kind tasks can't run on Windows, so exclude them. diff --git a/README.md b/README.md index 1826086..fa98a0a 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ API](https://docs.atlas.mongodb.com/api/) via a fluent interface. The `atlasclient` and also contains the test harnesses necessary to run Atlas Planned Maintenance specification tests. -Astrolabe supports Python 3.8+. +Astrolabe supports Python 3.9+. ## Installation @@ -36,7 +36,7 @@ python -m pip install . ## Dependencies -Astrolabe supports CPython 3.8+. +Astrolabe supports CPython 3.9+. Astrolabe requires [Click](https://pypi.org/project/click/), [requests](https://pypi.org/project/requests/), diff --git a/docs/installing-running-locally.md b/docs/installing-running-locally.md index 8f2b01b..eb82f58 100644 --- a/docs/installing-running-locally.md +++ b/docs/installing-running-locally.md @@ -10,7 +10,7 @@ Kubernetes test scenarios. `astrolabe` runs on Linux, OSX and Windows. -Running `astrolabe` requires Python 3.8 or later. To check the version +Running `astrolabe` requires Python 3.9 or later. To check the version of Python you are running, do: ``` bash diff --git a/docs/integration-guide.md b/docs/integration-guide.md index d1248ff..7f40a1c 100644 --- a/docs/integration-guide.md +++ b/docs/integration-guide.md @@ -153,14 +153,14 @@ systems are **strongly advised** to use the custom distro for running their tests. See `faq-why-custom-distro` for details. The Atlas Planned Maintenance tests can be run on all platforms which -have a Python 3.8+ binary installed. Each entry to the `platform` axis +have a Python 3.9+ binary installed. Each entry to the `platform` axis has the following fields: - `id` (required): unique identifier for this `platform` axis entry. - `display_name` (optional): plaintext name for this platform that will be used to display test runs. - `run_on` (required): evergreen distro name for this platform -- `variables.PYTHON3_BINARY` (required): path to the Python 3.8+ binary +- `variables.PYTHON3_BINARY` (required): path to the Python 3.9+ binary on the distro. This is used to run `astrolabe`. - `variables.PYTHON_BIN_DIR` (required): name of directory in which Python install executables. This is always `bin` on \*nix systems and @@ -172,11 +172,11 @@ platform: - id: platform display_name: OS values: - - id: ubuntu-16.04 - display_name: "Ubuntu 16.04" - run_on: ubuntu1604-test + - id: ubuntu-20.04 + display_name: "Ubuntu 20.04" + run_on: ubuntu2004-test variables: - PYTHON3_BINARY: "/opt/python/3.8/bin/python3" + PYTHON3_BINARY: "/opt/python/3.9/bin/python3" PYTHON_BIN_DIR: "bin" To encourage re-use of `platform` entries across driver projects, it is diff --git a/pyproject.toml b/pyproject.toml index 7983391..e383f41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dynamic = ["version"] description = "Command-line utility for testing Drivers against MongoDB Atlas " readme = "README.rst" license = {file="LICENSE"} -requires-python = ">=3.8" +requires-python = ">=3.9" authors = [ { name = "Prashant Mital", email = "mongodb-user@googlegroups.com" }, ] @@ -23,8 +23,7 @@ classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Software Development :: Testing", ] From 04b29b75648dc9f2c5a2143b20342ad30ac80a92 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 13 Oct 2024 06:49:18 -0500 Subject: [PATCH 02/10] fix import --- integrations/python/pymongo/workload-executor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integrations/python/pymongo/workload-executor.py b/integrations/python/pymongo/workload-executor.py index 7fe5f73..67cda7d 100644 --- a/integrations/python/pymongo/workload-executor.py +++ b/integrations/python/pymongo/workload-executor.py @@ -13,7 +13,8 @@ # pymongo (this works because pymongo is installed with pip flag -e) test_path = os.path.dirname(os.path.dirname(inspect.getfile(pymongo))) sys.path.insert(0, test_path) -from test.unified_format import UnifiedSpecTestMixinV1, interrupt_loop # noqa: E402 +from test.unified_format import UnifiedSpecTestMixinV1 # noqa: E402 +from test.unified_format_shared import interrupt_loop # noqa: E402 WIN32 = sys.platform in ("win32", "cygwin") From 6ec4707426a167bb8f056fd22709c783f4e22fe9 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 13 Oct 2024 15:03:16 -0500 Subject: [PATCH 03/10] debug --- integrations/python/pymongo/install-driver.sh | 3 +-- integrations/python/pymongo/workload-executor.py | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/integrations/python/pymongo/install-driver.sh b/integrations/python/pymongo/install-driver.sh index bdc65ab..c885db8 100755 --- a/integrations/python/pymongo/install-driver.sh +++ b/integrations/python/pymongo/install-driver.sh @@ -3,5 +3,4 @@ set -o xtrace "$PYTHON_BINARY" --version "$PYTHON_BINARY" -m virtualenv "$PYMONGO_VIRTUALENV_NAME" -"$PYMONGO_VIRTUALENV_NAME/$PYTHON_BIN_DIR/pip" install -e mongo-python-driver/[srv] -"$PYMONGO_VIRTUALENV_NAME/$PYTHON_BIN_DIR/pip" install certifi # TODO: remove this once BUILD-10841 is done. +"$PYMONGO_VIRTUALENV_NAME/$PYTHON_BIN_DIR/pip" install -e mongo-python-driver diff --git a/integrations/python/pymongo/workload-executor.py b/integrations/python/pymongo/workload-executor.py index 67cda7d..022e86b 100644 --- a/integrations/python/pymongo/workload-executor.py +++ b/integrations/python/pymongo/workload-executor.py @@ -31,6 +31,7 @@ def interrupt_handler(signum, frame): def workload_runner(mongodb_uri, test_workload): + print(f"Running workload: {test_workload!r}") runner = UnifiedSpecTestMixinV1() runner.TEST_SPEC = test_workload UnifiedSpecTestMixinV1.TEST_SPEC = test_workload @@ -39,6 +40,7 @@ def workload_runner(mongodb_uri, test_workload): # localhost:27017 on evergreen, so we have to patch it to use the client # specified in the workload runner runner.client = pymongo.MongoClient(mongodb_uri) + print("Running scenario") try: assert len(test_workload["tests"]) == 1 runner.run_scenario(test_workload["tests"][0], uri=mongodb_uri) From 3cc6056345bba6b50522c9f11a656b7e278a4a28 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 13 Oct 2024 15:50:48 -0500 Subject: [PATCH 04/10] debug --- integrations/python/pymongo/workload-executor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integrations/python/pymongo/workload-executor.py b/integrations/python/pymongo/workload-executor.py index 022e86b..beaa5d5 100644 --- a/integrations/python/pymongo/workload-executor.py +++ b/integrations/python/pymongo/workload-executor.py @@ -75,6 +75,7 @@ def workload_runner(mongodb_uri, test_workload): if __name__ == "__main__": + print("Starting up workload executor") connection_string, driver_workload = sys.argv[1], sys.argv[2] try: workload_spec = json.loads(driver_workload) From ec94142a857dae424acf4a37015dba3c2e4c4a5a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 14 Oct 2024 06:26:13 -0500 Subject: [PATCH 05/10] fix --- integrations/python/pymongo/workload-executor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integrations/python/pymongo/workload-executor.py b/integrations/python/pymongo/workload-executor.py index beaa5d5..892ab4a 100644 --- a/integrations/python/pymongo/workload-executor.py +++ b/integrations/python/pymongo/workload-executor.py @@ -13,8 +13,7 @@ # pymongo (this works because pymongo is installed with pip flag -e) test_path = os.path.dirname(os.path.dirname(inspect.getfile(pymongo))) sys.path.insert(0, test_path) -from test.unified_format import UnifiedSpecTestMixinV1 # noqa: E402 -from test.unified_format_shared import interrupt_loop # noqa: E402 +from test.unified_format import UnifiedSpecTestMixinV1, interrupt_loop # noqa: E402 WIN32 = sys.platform in ("win32", "cygwin") From 5e91816d6419364aba2ee47eadb8c517794e09f2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 14 Oct 2024 06:27:35 -0500 Subject: [PATCH 06/10] undo changes --- integrations/python/pymongo/workload-executor.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/integrations/python/pymongo/workload-executor.py b/integrations/python/pymongo/workload-executor.py index 892ab4a..7fe5f73 100644 --- a/integrations/python/pymongo/workload-executor.py +++ b/integrations/python/pymongo/workload-executor.py @@ -30,7 +30,6 @@ def interrupt_handler(signum, frame): def workload_runner(mongodb_uri, test_workload): - print(f"Running workload: {test_workload!r}") runner = UnifiedSpecTestMixinV1() runner.TEST_SPEC = test_workload UnifiedSpecTestMixinV1.TEST_SPEC = test_workload @@ -39,7 +38,6 @@ def workload_runner(mongodb_uri, test_workload): # localhost:27017 on evergreen, so we have to patch it to use the client # specified in the workload runner runner.client = pymongo.MongoClient(mongodb_uri) - print("Running scenario") try: assert len(test_workload["tests"]) == 1 runner.run_scenario(test_workload["tests"][0], uri=mongodb_uri) @@ -74,7 +72,6 @@ def workload_runner(mongodb_uri, test_workload): if __name__ == "__main__": - print("Starting up workload executor") connection_string, driver_workload = sys.argv[1], sys.argv[2] try: workload_spec = json.loads(driver_workload) From 52e41fd2974a703f0b3b2b3ef07b359fc8fac8e2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 14 Oct 2024 06:31:47 -0500 Subject: [PATCH 07/10] test branch --- .evergreen/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index e65c160..8d3e68b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -659,8 +659,8 @@ axes: display_name: "PyMongo (master)" variables: DRIVER_DIRNAME: "python/pymongo" - DRIVER_REPOSITORY: "https://github.com/mongodb/mongo-python-driver" - DRIVER_REVISION: "master" + DRIVER_REPOSITORY: "https://github.com/blink1073/mongo-python-driver" + DRIVER_REVISION: "PYTHON-4862" PYMONGO_VIRTUALENV_NAME: "pymongotestvenv" ASTROLABE_EXECUTOR_STARTUP_TIME: 5 - id: ruby-master From 33348f7f9119a9f08c3eaa00ff8c66e1a0626f82 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 14 Oct 2024 06:47:06 -0500 Subject: [PATCH 08/10] use setup and teardown --- integrations/python/pymongo/workload-executor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integrations/python/pymongo/workload-executor.py b/integrations/python/pymongo/workload-executor.py index 7fe5f73..ced297b 100644 --- a/integrations/python/pymongo/workload-executor.py +++ b/integrations/python/pymongo/workload-executor.py @@ -33,6 +33,7 @@ def workload_runner(mongodb_uri, test_workload): runner = UnifiedSpecTestMixinV1() runner.TEST_SPEC = test_workload UnifiedSpecTestMixinV1.TEST_SPEC = test_workload + UnifiedSpecTestMixinV1.setUpClass() runner.setUp() # this is necessary because there isn't a mongo instance on # localhost:27017 on evergreen, so we have to patch it to use the client @@ -45,6 +46,9 @@ def workload_runner(mongodb_uri, test_workload): runner.entity_map["errors"] = [ {"error": str(exc), "time": time.time(), "type": type(exc).__name__} ] + finally: + runner.tearDown() + UnifiedSpecTestMixinV1.tearDownClass() entity_map = defaultdict(list, runner.entity_map._entities) for entity_type in ["successes", "iterations"]: if entity_type not in entity_map: From 080486b0bb29374bca2cd8ad8b9fbef3ed563b71 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 14 Oct 2024 07:33:04 -0500 Subject: [PATCH 09/10] restore branch --- .evergreen/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 8d3e68b..e65c160 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -659,8 +659,8 @@ axes: display_name: "PyMongo (master)" variables: DRIVER_DIRNAME: "python/pymongo" - DRIVER_REPOSITORY: "https://github.com/blink1073/mongo-python-driver" - DRIVER_REVISION: "PYTHON-4862" + DRIVER_REPOSITORY: "https://github.com/mongodb/mongo-python-driver" + DRIVER_REVISION: "master" PYMONGO_VIRTUALENV_NAME: "pymongotestvenv" ASTROLABE_EXECUTOR_STARTUP_TIME: 5 - id: ruby-master From ee80583cc715279a9a6e5886545e03d8528ad174 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 14 Oct 2024 08:55:49 -0500 Subject: [PATCH 10/10] fix tests --- integrations/python/pymongo/workload-executor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integrations/python/pymongo/workload-executor.py b/integrations/python/pymongo/workload-executor.py index ced297b..e22a16f 100644 --- a/integrations/python/pymongo/workload-executor.py +++ b/integrations/python/pymongo/workload-executor.py @@ -31,9 +31,9 @@ def interrupt_handler(signum, frame): def workload_runner(mongodb_uri, test_workload): runner = UnifiedSpecTestMixinV1() + # Note: we cannot use setUpClass or tearDownClass since we're overriding the client for the tests. runner.TEST_SPEC = test_workload UnifiedSpecTestMixinV1.TEST_SPEC = test_workload - UnifiedSpecTestMixinV1.setUpClass() runner.setUp() # this is necessary because there isn't a mongo instance on # localhost:27017 on evergreen, so we have to patch it to use the client @@ -48,7 +48,6 @@ def workload_runner(mongodb_uri, test_workload): ] finally: runner.tearDown() - UnifiedSpecTestMixinV1.tearDownClass() entity_map = defaultdict(list, runner.entity_map._entities) for entity_type in ["successes", "iterations"]: if entity_type not in entity_map: