[WIP] Interoperability updates for py3.8, py3.9 and py3.10

.github/workflows/test.yaml

@@ -38,6 +38,9 @@ jobs:
           - name: 'Test (pants runs: pytest)'
             python-version-short: '3.9'
             python-version: '3.9.14'
+          - name: 'Test (pants runs: pytest)'
+            python-version-short: '3.10'
+            python-version: '3.10.12'
 
     services:
       mongo:

CHANGELOG.rst

@@ -14,6 +14,9 @@ Changed
 ~~~~~~~
 * Bumped `jsonschema` 2.6.0 -> 3.2.0 now that python3.6 is not supported. #6118
 
+* Support for Python3.8, Python3.9 and Python3.10.  Update underlying modules and
+  remove modules that weren't compatible with newer versions of Python.  #6157
+
 Added
 ~~~~~
 * Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)

Makefile

@@ -55,7 +55,8 @@ REQUIREMENTS := test-requirements.txt requirements.txt
 
 # Pin common pip version here across all the targets
 # Note! Periodic maintenance pip upgrades are required to be up-to-date with the latest pip security fixes and updates
-PIP_VERSION ?= 20.3.3
+# 202403: Use pip 24.0 which is the highest supported version across Python 3.8 - 3.10
+PIP_VERSION ?= 24.0
 SETUPTOOLS_VERSION ?= 51.3.3
 PIP_OPTIONS := $(ST2_PIP_OPTIONS)
 
@@ -73,6 +74,9 @@ endif
 # The minus in front of st2.st2common.bootstrap filters out logging statements from that module.
 # See https://nose.readthedocs.io/en/latest/usage.html#cmdoption-logging-filter
 NOSE_OPTS := --rednose --immediate --with-parallel --parallel-strategy=FILE --nocapture --logging-filter=-st2.st2common.bootstrap
+# https://github.com/pytest-dev/pytest-xdist/issues/71
+#PYTEST_OPTS := -n auto --tx 2*popen//execmodel=eventlet
+PYTEST_OPTS := ""
 
 ifndef NOSE_TIME
 	NOSE_TIME := yes
@@ -486,7 +490,7 @@ flake8: requirements .flake8
 	. $(VIRTUALENV_DIR)/bin/activate; flake8 --config ./lint-configs/python/.flake8 pylint_plugins/
 
 # Make task which verifies st2client README will parse pypi checks
-. PHONY: .st2client-pypi-check
+.PHONY: .st2client-pypi-check
 .st2client-pypi-check:
 	@echo
 	@echo "==================== st2client pypi check ===================="
@@ -735,8 +739,8 @@ check-dependency-conflicts:
 	@echo
 	# Verify there are no conflicting dependencies
 	cat st2*/requirements.txt contrib/runners/*/requirements.txt | sort -u > req.txt && \
-	$(VIRTUALENV_DIR)/bin/pip-compile req.txt || exit 1; \
-	if [[ -e req.txt ]]; then rm req.txt; fi
+	$(VIRTUALENV_DIR)/bin/pip-compile --strip-extras --output-file req.out req.txt || exit 1; \
+	rm -f req.txt req.out
 
 .PHONY: virtualenv
 	# Note: We always want to update virtualenv/bin/activate file to make sure
@@ -823,8 +827,7 @@ unit-tests: requirements .unit-tests
 		echo "Running tests in" $$component; \
 		echo "-----------------------------------------------------------"; \
 		. $(VIRTUALENV_DIR)/bin/activate; \
-		    nosetests $(NOSE_OPTS) -s -v \
-		    $$component/tests/unit || exit 1; \
+		    pytest --capture=no --verbose $$component/tests/unit || exit 1; \
 		echo "-----------------------------------------------------------"; \
 		echo "Done running tests in" $$component; \
 		echo "==========================================================="; \
@@ -846,8 +849,7 @@ endif
 		echo "-----------------------------------------------------------"; \
 		. $(VIRTUALENV_DIR)/bin/activate; \
 		    COVERAGE_FILE=.coverage.unit.$$(echo $$component | tr '/' '.') \
-		    nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \
-		    $(NOSE_COVERAGE_PACKAGES) \
+		    pytest --capture=no --verbose $(PYTEST_OPTS) --cov=$$component --cov-branch \
 		    $$component/tests/unit || exit 1; \
 		echo "-----------------------------------------------------------"; \
 		echo "Done running tests in" $$component; \
@@ -902,7 +904,7 @@ itests: requirements .itests
 		echo "Running integration tests in" $$component; \
 		echo "-----------------------------------------------------------"; \
 		. $(VIRTUALENV_DIR)/bin/activate; \
-		    nosetests $(NOSE_OPTS) -s -v \
+		    pytest --capture=no --verbose $(PYTEST_OPTS) \
 		    $$component/tests/integration || exit 1; \
 		echo "-----------------------------------------------------------"; \
 		echo "Done running integration tests in" $$component; \
@@ -925,8 +927,7 @@ endif
 		echo "-----------------------------------------------------------"; \
 		. $(VIRTUALENV_DIR)/bin/activate; \
 		    COVERAGE_FILE=.coverage.integration.$$(echo $$component | tr '/' '.') \
-		    nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \
-		    $(NOSE_COVERAGE_PACKAGES) \
+		    pytest --capture=no --verbose $(PYTEST_OPTS) --cov=$$component --cov-branch \
 		    $$component/tests/integration || exit 1; \
 		echo "-----------------------------------------------------------"; \
 		echo "Done integration running tests in" $$component; \
@@ -942,8 +943,7 @@ endif
 		echo "==========================================================="; \
 		. $(VIRTUALENV_DIR)/bin/activate; \
 		    COVERAGE_FILE=.coverage.integration.$$(echo $$component | tr '/' '.') \
-			nosetests $(NOSE_OPTS) -s -v \
-			$(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) $$component/tests/integration || exit 1; \
+			pytest --capture=no --verbose $(PYTEST_OPTS) --cov=$$component --cov-branch $$component/tests/integration || exit 1; \
 	done
 	# NOTE: If you also want to run orquesta tests which seem to have a bunch of race conditions, use
 	# ci-integration-full target
@@ -1037,16 +1037,15 @@ orquesta-itests: requirements .orquesta-itests
 	@echo "==================== Orquesta integration tests ===================="
 	@echo "The tests assume st2 is running on 127.0.0.1."
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v st2tests/integration/orquesta || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; pytest --capture=no --verbose $(PYTEST_OPTS) st2tests/integration/orquesta || exit 1;
 
 .PHONY: .orquesta-itests-coverage-html
 .orquesta-itests-coverage-html:
 	@echo
 	@echo "==================== Orquesta integration tests with coverage (HTML reports) ===================="
 	@echo "The tests assume st2 is running on 127.0.0.1."
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v --with-coverage \
-        --cover-inclusive --cover-html st2tests/integration/orquesta || exit 1;
+	. $(VIRTUALENV_DIR)/bin/activate; pytest --capture=no --verbose $(PYTEST_OPTS) --cov=orquesta --cov-branch  st2tests/integration/orquesta || exit 1;
 
 .PHONY: packs-tests
 packs-tests: requirements .packs-tests
@@ -1075,7 +1074,7 @@ runners-tests: requirements .runners-tests
 		echo "==========================================================="; \
 		echo "Running tests in" $$component; \
 		echo "==========================================================="; \
-		. $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v $$component/tests/unit || exit 1; \
+		. $(VIRTUALENV_DIR)/bin/activate; pytest --capture=no --verbose $(PYTEST_OPTS) $$component/tests/unit || exit 1; \
 	done
 
 .PHONY: runners-itests
@@ -1091,7 +1090,7 @@ runners-itests: requirements .runners-itests
 		echo "==========================================================="; \
 		echo "Running integration tests in" $$component; \
 		echo "==========================================================="; \
-		. $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v $$component/tests/integration || exit 1; \
+		. $(VIRTUALENV_DIR)/bin/activate; pytest --capture=no --verbose $(PYTEST_OPTS) $$component/tests/integration || exit 1; \
 	done
 
 .PHONY: .runners-itests-coverage-html
@@ -1104,8 +1103,8 @@ runners-itests: requirements .runners-itests
 		echo "==========================================================="; \
 		echo "Running integration tests in" $$component; \
 		echo "==========================================================="; \
-		. $(VIRTUALENV_DIR)/bin/activate; nosetests $(NOSE_OPTS) -s -v --with-coverage \
-			--cover-inclusive --cover-html $$component/tests/integration || exit 1; \
+		. $(VIRTUALENV_DIR)/bin/activate; pytest --capture=no --verbose $(PYTEST_OPTS) --cov=$$component --cov-report=html \
+			$$component/tests/integration || exit 1; \
 	done
 
 .PHONY: cli
@@ -1147,7 +1146,7 @@ ci-checks: .generated-files-check .shellcheck .black-check .pre-commit-checks .f
 	@echo
 	@echo "==================== rst-check ===================="
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; rstcheck --report warning CHANGELOG.rst
+	. $(VIRTUALENV_DIR)/bin/activate; rstcheck --report-level WARNING CHANGELOG.rst
 
 .PHONY: .generated-files-check
 .generated-files-check:

contrib/examples/actions/ubuntu_pkg_info/ubuntu_pkg_info.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright 2020 The StackStorm Authors.
+# Copyright 2020-2024 The StackStorm Authors.
 # Copyright 2019 Extreme Networks, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,7 +19,7 @@
 import subprocess
 
 import six
-import lib.datatransformer as transformer
+import lib.datatransformer as transformer  # pylint:disable=import-error,no-name-in-module
 
 
 def main(args):

contrib/packs/actions/pack_mgmt/get_pack_dependencies.py

@@ -1,4 +1,4 @@
-# Copyright 2020 The StackStorm Authors.
+# Copyright 2020-2024 The StackStorm Authors.
 # Copyright 2019 Extreme Networks, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -125,9 +125,9 @@ def get_pack_version(pack=None):
         pack_metadata = get_pack_metadata(pack_dir=pack_path)
         result = pack_metadata.get("version", None)
     except Exception:
-        result = None
-    finally:
-        return result
+        return None
+
+    return result
 
 
 def get_dependency_list(pack=None):
@@ -138,6 +138,6 @@ def get_dependency_list(pack=None):
         result = pack_metadata.get("dependencies", None)
     except Exception:
         print("Could not open pack.yaml at location %s" % pack_path)
-        result = None
-    finally:
-        return result
+        return None
+
+    return result

contrib/packs/tests/test_action_download.py

@@ -260,7 +260,7 @@ def mock_acquire(self, timeout=None):
                 fp.write("")
 
             expected_msg = "Timeout waiting to acquire lock for"
-            self.assertRaisesRegexp(
+            self.assertRaisesRegex(
                 LockTimeout,
                 expected_msg,
                 action.run,
@@ -328,7 +328,7 @@ def test_run_pack_download_invalid_version(self):
             "is not a valid version, hash, tag or branch.*?"
             "Available versions are: 1.0.0, 2.0.0."
         )
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             ValueError,
             expected_msg,
             action.run,
@@ -351,7 +351,7 @@ def test_download_pack_stackstorm_version_identifier_check(self):
             'Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but '
             'current version is "2.2.0"'
         )
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             ValueError,
             expected_msg,
             action.run,
@@ -364,7 +364,7 @@ def test_download_pack_stackstorm_version_identifier_check(self):
             'Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but '
             'current version is "2.3.0"'
         )
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             ValueError,
             expected_msg,
             action.run,
@@ -377,7 +377,7 @@ def test_download_pack_stackstorm_version_identifier_check(self):
             'Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but '
             'current version is "1.5.9"'
         )
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             ValueError,
             expected_msg,
             action.run,
@@ -390,7 +390,7 @@ def test_download_pack_stackstorm_version_identifier_check(self):
             'Pack "test3" requires StackStorm ">=1.6.0, <2.2.0", but '
             'current version is "1.5.0"'
         )
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             ValueError,
             expected_msg,
             action.run,
@@ -470,7 +470,7 @@ def test_download_pack_python_version_check(self):
                 r'Pack "test3" requires Python 2.x, but current Python version is '
                 '"3.5.2"'
             )
-            self.assertRaisesRegexp(
+            self.assertRaisesRegex(
                 ValueError,
                 expected_msg,
                 action.run,
@@ -497,7 +497,7 @@ def test_download_pack_python_version_check(self):
                 r'Pack "test3" requires Python 3.x, but current Python version is '
                 '"2.7.2"'
             )
-            self.assertRaisesRegexp(
+            self.assertRaisesRegex(
                 ValueError,
                 expected_msg,
                 action.run,
@@ -657,7 +657,7 @@ def test_run_pack_download_local_directory(self):
 
         # 1. Local directory doesn't exist
         expected_msg = r'Local pack directory ".*" doesn\'t exist'
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             ValueError,
             expected_msg,
             action.run,

contrib/runners/action_chain_runner/tests/unit/test_actionchain.py

@@ -364,7 +364,7 @@ def test_chain_runner_bad_default(self, request):
         expected_msg = (
             'Unable to find node with name "bad_default" referenced in "default".'
         )
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             runnerexceptions.ActionRunnerPreRunError, expected_msg, chain_runner.pre_run
         )
 
@@ -434,7 +434,7 @@ def test_chain_runner_broken_on_success_path_static_task_name(self, request):
             'Unable to find node with name "c5" referenced in "on-success" '
             'in task "c2"'
         )
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             runnerexceptions.ActionRunnerPreRunError, expected_msg, chain_runner.pre_run
         )
 
@@ -453,7 +453,7 @@ def test_chain_runner_broken_on_failure_path_static_task_name(self, request):
             'Unable to find node with name "c6" referenced in "on-failure" '
             'in task "c2"'
         )
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             runnerexceptions.ActionRunnerPreRunError, expected_msg, chain_runner.pre_run
         )
 
@@ -896,7 +896,7 @@ def test_chain_task_passes_invalid_parameter_type_to_action(self, mock_request):
             r'Failed to cast value "stringnotanarray" \(type: str\) for parameter '
             r'"arrtype" of type "array"'
         )
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             ValueError,
             expected_msg,
             chain_runner.run,
@@ -944,7 +944,7 @@ def test_exception_is_thrown_if_both_params_and_parameters_attributes_are_provid
             'Either "params" or "parameters" attribute needs to be provided, but '
             "not both"
         )
-        self.assertRaisesRegexp(
+        self.assertRaisesRegex(
             runnerexceptions.ActionRunnerPreRunError, expected_msg, chain_runner.pre_run
         )
 

contrib/runners/action_chain_runner/tests/unit/test_actionchain_params_rendering.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 from __future__ import absolute_import
-import unittest2
+import unittest
 
 import mock
 
@@ -24,7 +24,7 @@
 from st2common.models.system.actionchain import Node
 
 
-class ActionChainRunnerResolveParamsTests(unittest2.TestCase):
+class ActionChainRunnerResolveParamsTests(unittest.TestCase):
     def test_render_params_action_context(self):
         runner = acr.get_runner()
         chain_context = {

contrib/runners/announcement_runner/tests/unit/test_announcementrunner.py

@@ -64,7 +64,7 @@ def test_announcement_no_experimental(self, dispatch):
         runner.liveaction = mock.Mock(context={})
 
         expected_msg = "Experimental flag is missing for action some.thing"
-        self.assertRaisesRegexp(Exception, expected_msg, runner.pre_run)
+        self.assertRaisesRegex(Exception, expected_msg, runner.pre_run)
 
     @mock.patch("st2common.models.api.trace.TraceContext.__new__")
     def test_announcement_with_trace(self, context, dispatch):