fix(core): python 3.11 compatiblity and fix tests(#3386)

Author: Panaetius

poetry.lock

@@ -36,6 +36,8 @@ classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Development Status :: 4 - Beta",
 ]
 homepage = "https://github.com/swissdatasciencecenter/renku-python"
@@ -73,7 +75,7 @@ importlib-resources = ">=5.12.0,<5.13.0"
 inject = "<4.4.0,>=4.3.0"
 jinja2 = { version = ">=2.11.3,<3.1.3" }
 networkx = "<2.7,>=2.6.0"
-numpy = ">=1.20.0,<1.22.0"
+numpy = ">=1.24.0,<1.25.0"
 packaging = "<24.0,>=23.0"
 pathspec = "<1.0.0,>=0.8.0"
 patool = "==1.12"
@@ -255,6 +257,7 @@ markers = [
     "serial: mark a test that can not be run in parallel",
     "service: mark a test as service test.",
     "shelled: mark a test as a shelled test.",
+    "remote_repo: used to specify which remote to use in test fixtures."
 ]
 filterwarnings = [
     "ignore:<class 'pytest_black.BlackItem'> is not using a cooperative constructor:pytest.PytestDeprecationWarning",

pyproject.toml

@@ -530,7 +530,7 @@ def from_project(cls, project: Optional["Project"]) -> "TemplateMetadata":
         # NOTE: Always set __renku_version__ to the value read from the Dockerfile (if available) since setting/updating
         # the template doesn't change project's metadata version and shouldn't update the Renku version either
         renku_version = metadata.get("__renku_version__")
-        metadata["__renku_version__"] = str(read_renku_version_from_dockerfile()) or renku_version or __version__
+        metadata["__renku_version__"] = str(read_renku_version_from_dockerfile() or renku_version or __version__)
 
         return cls(metadata=metadata, immutable_files=immutable_files)
 

renku/domain_model/template.py

@@ -870,87 +870,77 @@ def serialize(self, object: persistent.Persistent):
 
         return data
 
-    def _serialize_helper(self, object):
+    def _serialize_helper(self, obj):
         # TODO: Raise an error if an unsupported object is being serialized
-        if object is None:
+        if obj is None:
             return None
-        elif isinstance(object, (int, float, str, bool)):
-            return object
-        elif isinstance(object, list):
-            return [self._serialize_helper(value) for value in object]
-        elif isinstance(object, set):
+        elif isinstance(obj, (int, float, str, bool)):
+            return obj
+        elif isinstance(obj, list):
+            return [self._serialize_helper(value) for value in obj]
+        elif isinstance(obj, set):
             return {
                 "@renku_data_type": SET_TYPE,
-                "@renku_data_value": [self._serialize_helper(value) for value in object],
+                "@renku_data_value": [self._serialize_helper(value) for value in obj],
             }
-        elif isinstance(object, frozenset):
+        elif isinstance(obj, frozenset):
             return {
                 "@renku_data_type": FROZEN_SET_TYPE,
-                "@renku_data_value": [self._serialize_helper(value) for value in object],
+                "@renku_data_value": [self._serialize_helper(value) for value in obj],
             }
-        elif isinstance(object, dict):
+        elif isinstance(obj, dict):
             result = dict()
-            items = sorted(object.items(), key=lambda x: x[0])
+            items = sorted(obj.items(), key=lambda x: x[0])
             for key, value in items:
                 result[key] = self._serialize_helper(value)
             return result
-        elif isinstance(object, Index):
+        elif isinstance(obj, Index):
             # NOTE: Index objects are not stored as references and are included in their parent object (i.e. root)
-            state = object.__getstate__()
+            state = obj.__getstate__()
             state = self._serialize_helper(state)
-            return {"@renku_data_type": get_type_name(object), "@renku_oid": object._p_oid, **state}
-        elif isinstance(object, (OOTreeSet, Length, OOSet)):
-            state = object.__getstate__()
+            return {"@renku_data_type": get_type_name(obj), "@renku_oid": obj._p_oid, **state}
+        elif isinstance(obj, (OOTreeSet, Length, OOSet)):
+            state = obj.__getstate__()
             state = self._serialize_helper(state)
-            return {"@renku_data_type": get_type_name(object), "@renku_data_value": state}
-        elif isinstance(object, persistent.Persistent):
-            if not object._p_oid:
-                object._p_oid = Database.generate_oid(object)
-            if object._p_state not in [GHOST, UPTODATE] or (object._p_state == UPTODATE and object._p_serial == NEW):
-                self._database.register(object)
-            return {"@renku_data_type": get_type_name(object), "@renku_oid": object._p_oid, "@renku_reference": True}
-        elif isinstance(object, datetime.datetime):
-            value = object.isoformat()
-        elif isinstance(object, tuple):
-            value = tuple(self._serialize_helper(value) for value in object)
-        elif isinstance(object, (InterfaceClass)):
+            return {"@renku_data_type": get_type_name(obj), "@renku_data_value": state}
+        elif isinstance(obj, persistent.Persistent):
+            if not obj._p_oid:
+                obj._p_oid = Database.generate_oid(obj)
+            if obj._p_state not in [GHOST, UPTODATE] or (obj._p_state == UPTODATE and obj._p_serial == NEW):
+                self._database.register(obj)
+            return {"@renku_data_type": get_type_name(obj), "@renku_oid": obj._p_oid, "@renku_reference": True}
+        elif isinstance(obj, datetime.datetime):
+            value = obj.isoformat()
+        elif isinstance(obj, tuple):
+            value = tuple(self._serialize_helper(value) for value in obj)
+        elif isinstance(obj, (InterfaceClass)):
             # NOTE: Zope interfaces are weird, they're a class with type InterfaceClass, but need to be deserialized
             # as the class (without instantiation)
-            return {"@renku_data_type": TYPE_TYPE, "@renku_data_value": f"{object.__module__}.{object.__name__}"}
-        elif isinstance(object, type):
+            return {"@renku_data_type": TYPE_TYPE, "@renku_data_value": f"{obj.__module__}.{obj.__name__}"}
+        elif isinstance(obj, type):
             # NOTE: We're storing a type, not an instance
-            return {"@renku_data_type": TYPE_TYPE, "@renku_data_value": get_type_name(object)}
-        elif isinstance(object, (FunctionType, BuiltinFunctionType)):
-            name = object.__name__
-            module = getattr(object, "__module__", None)
+            return {"@renku_data_type": TYPE_TYPE, "@renku_data_value": get_type_name(obj)}
+        elif isinstance(obj, (FunctionType, BuiltinFunctionType)):
+            name = obj.__name__
+            module = getattr(obj, "__module__", None)
             return {"@renku_data_type": FUNCTION_TYPE, "@renku_data_value": f"{module}.{name}"}
-        elif hasattr(object, "__getstate__"):
-            if id(object) in self._serialization_cache:
-                # NOTE: We already serialized this -> circular/repeat reference.
-                return {"@renku_data_type": REFERENCE_TYPE, "@renku_data_value": self._serialization_cache[id(object)]}
-
-            # NOTE: The reference used for circular reference is just the position in the serialization cache,
-            # as the order is deterministic. So the order in which objects are encountered is their id for referencing.
-            self._serialization_cache[id(object)] = len(self._serialization_cache)
-
-            value = object.__getstate__().copy()
-            value = {k: v for k, v in value.items() if not k.startswith("_v_")}
-            value = self._serialize_helper(value)
-            assert not isinstance(value, dict) or "id" in value, f"Invalid object state: {value} for {object}"
         else:
-            if id(object) in self._serialization_cache:
+            if id(obj) in self._serialization_cache:
                 # NOTE: We already serialized this -> circular/repeat reference
-                return {"@renku_data_type": REFERENCE_TYPE, "@renku_data_value": self._serialization_cache[id(object)]}
+                return {"@renku_data_type": REFERENCE_TYPE, "@renku_data_value": self._serialization_cache[id(obj)]}
 
             # NOTE: The reference used for circular reference is just the position in the serialization cache,
             # as the order is deterministic So the order in which objects are encoutered is their id for referencing.
-            self._serialization_cache[id(object)] = len(self._serialization_cache)
-
-            value = object.__dict__.copy()
+            self._serialization_cache[id(obj)] = len(self._serialization_cache)
+            if hasattr(obj, "__getstate__"):
+                # NOTE: On Python 3.11+ this just returns __dict__ if __getstate__ isn't implemented.
+                value = obj.__getstate__().copy()
+            else:
+                value = obj.__dict__.copy()
             value = {k: v for k, v in value.items() if not k.startswith("_v_")}
             value = self._serialize_helper(value)
 
-        return {"@renku_data_type": get_type_name(object), "@renku_data_value": value}
+        return {"@renku_data_type": get_type_name(obj), "@renku_data_value": value}
 
 
 class ObjectReader:

renku/infrastructure/database.py

@@ -24,6 +24,9 @@
 IT_REMOTE_REPO_URL = os.getenv(
     "IT_REMOTE_REPOSITORY", "https://gitlab.dev.renku.ch/renku-python-integration-tests/core-integration-test"
 )
+IT_REMOTE_OLD_REPO_URL = os.getenv(
+    "IT_REMOTE_REPOSITORY", "https://gitlab.dev.renku.ch/renku-python-integration-tests/core-integration-test-old"
+)
 IT_PROTECTED_REMOTE_REPO_URL = os.getenv(
     "IT_PROTECTED_REMOTE_REPO", "https://gitlab.dev.renku.ch/renku-python-integration-tests/core-it-protected.git"
 )

tests/fixtures/config.py

@@ -91,7 +91,14 @@ def integration_repo(headers, project_id, url_components) -> Generator[Repositor
 
 @pytest.fixture()
 def integration_lifecycle(
-    svc_client, mock_redis, identity_headers, it_remote_repo_url, it_protected_repo_url, it_workflow_repo_url, request
+    svc_client,
+    mock_redis,
+    identity_headers,
+    it_remote_repo_url,
+    it_protected_repo_url,
+    it_workflow_repo_url,
+    it_remote_old_repo_url,
+    request,
 ):
     """Setup and teardown steps for integration tests."""
     from renku.domain_model.git import GitURL
@@ -104,6 +111,8 @@ def integration_lifecycle(
         remote_repo = it_protected_repo_url
     elif marker.args[0] == "workflow":
         remote_repo = it_workflow_repo_url
+    elif marker.args[0] == "old":
+        remote_repo = it_remote_old_repo_url
     else:
         raise ValueError(f"Couldn't get remote repo for marker {marker.args[0]}")
 

tests/service/fixtures/service_integration.py

@@ -56,6 +56,14 @@ def it_remote_repo_url():
     return IT_REMOTE_REPO_URL
 
 
+@pytest.fixture(scope="module")
+def it_remote_old_repo_url():
+    """Returns a remote path to integration test repository."""
+    from tests.fixtures.config import IT_REMOTE_OLD_REPO_URL
+
+    return IT_REMOTE_OLD_REPO_URL
+
+
 @pytest.fixture(scope="module")
 def it_remote_public_renku_repo_url():
     """Returns a remote path to a public integration test repository."""
@@ -93,6 +101,29 @@ def it_remote_repo_url_temp_branch(it_remote_repo_url):
             repo.branches.remove(branch)
 
 
+@pytest.fixture(scope="function")
+def it_remote_old_repo_url_temp_branch(it_remote_old_repo_url):
+    """Returns a remote path to integration test repository."""
+
+    with tempfile.TemporaryDirectory() as tempdir:
+        # NOTE: create temporary branch and push it
+        git_url = urlparse(it_remote_old_repo_url)
+
+        url = "oauth2:{}@{}".format(os.getenv("IT_OAUTH_GIT_TOKEN"), git_url.netloc)
+        git_url = git_url._replace(netloc=url).geturl()
+        repo = Repository.clone_from(url=git_url, path=tempdir)
+        origin = repo.remotes["origin"]
+        branch_name = uuid.uuid4().hex
+        branch = repo.branches.add(branch_name)
+        repo.push(origin, branch, set_upstream=True)
+        try:
+            yield it_remote_old_repo_url, branch_name
+        finally:
+            # NOTE: delete remote branch
+            repo.push(origin, branch, delete=True)
+            repo.branches.remove(branch)
+
+
 @pytest.fixture(scope="module")
 def it_non_renku_repo_url():
     """Returns a remote path to integration test repository."""

tests/service/fixtures/service_projects.py

@@ -17,7 +17,6 @@
 """Test config jobs."""
 import pytest
 
-from renku.core.errors import MigrationRequired
 from tests.utils import retry_failed
 
 
@@ -67,5 +66,6 @@ def test_delay_config_set_failure(svc_client_cache, it_remote_repo_url, view_use
 
     from renku.ui.service.jobs.delayed_ctrl import delayed_ctrl_job
 
-    with pytest.raises(MigrationRequired):
-        delayed_ctrl_job(context, view_user_data, job.job_id, renku_module, renku_ctrl)
+    updated_job = delayed_ctrl_job(context, view_user_data, job.job_id, renku_module, renku_ctrl)
+
+    assert updated_job

tests/service/jobs/test_config.py

@@ -23,7 +23,7 @@
 import pytest
 from werkzeug.utils import secure_filename
 
-from renku.core.errors import DatasetExistsError, MigrationRequired, ParameterError
+from renku.core.errors import DatasetExistsError, DatasetNotFound, ParameterError
 from renku.infrastructure.repository import Repository
 from renku.ui.service.jobs.cleanup import cache_project_cleanup
 from renku.ui.service.jobs.datasets import dataset_add_remote_file, dataset_import
@@ -362,8 +362,7 @@ def test_delay_add_file_job_failure(svc_client_cache, it_remote_repo_url_temp_br
 
     from renku.ui.service.jobs.delayed_ctrl import delayed_ctrl_job
 
-    with pytest.raises(MigrationRequired):
-        delayed_ctrl_job(context, view_user_data, job.job_id, renku_module, renku_ctrl)
+    delayed_ctrl_job(context, view_user_data, job.job_id, renku_module, renku_ctrl)
 
 
 @pytest.mark.parametrize("doi", ["10.5281/zenodo.3761586"])
@@ -471,15 +470,14 @@ def test_delay_create_dataset_failure(svc_client_cache, it_remote_repo_url_temp_
 
     from renku.ui.service.jobs.delayed_ctrl import delayed_ctrl_job
 
-    with pytest.raises(MigrationRequired):
-        delayed_ctrl_job(context, view_user_data, job.job_id, renku_module, renku_ctrl)
+    delayed_ctrl_job(context, view_user_data, job.job_id, renku_module, renku_ctrl)
 
 
 @pytest.mark.service
 @pytest.mark.integration
 @retry_failed
 def test_delay_remove_dataset_job(svc_client_cache, it_remote_repo_url_temp_branch, view_user_data):
-    """Create a dataset was removed successfully."""
+    """Check a dataset was removed successfully."""
     from renku.ui.service.jobs.delayed_ctrl import delayed_ctrl_job
     from renku.ui.service.serializers.datasets import DatasetRemoveRequest
 
@@ -512,7 +510,7 @@ def test_delay_remove_dataset_job(svc_client_cache, it_remote_repo_url_temp_bran
 @pytest.mark.integration
 @retry_failed
 def test_delay_remove_dataset_job_failure(svc_client_cache, it_remote_repo_url_temp_branch, view_user_data):
-    """Create a dataset was removed successfully."""
+    """Check removing missing dataset fails."""
     from renku.ui.service.jobs.delayed_ctrl import delayed_ctrl_job
     from renku.ui.service.serializers.datasets import DatasetRemoveRequest
 
@@ -535,7 +533,7 @@ def test_delay_remove_dataset_job_failure(svc_client_cache, it_remote_repo_url_t
         user, job_data={"ctrl_context": {**context, "renku_module": renku_module, "renku_ctrl": renku_ctrl}}
     )
 
-    with pytest.raises(MigrationRequired):
+    with pytest.raises(DatasetNotFound):
         delayed_ctrl_job(context, view_user_data, delete_job.job_id, renku_module, renku_ctrl)
 
 
@@ -608,8 +606,7 @@ def test_delay_edit_dataset_job_failure(svc_client_cache, it_remote_repo_url_tem
 
     from renku.ui.service.jobs.delayed_ctrl import delayed_ctrl_job
 
-    with pytest.raises(MigrationRequired):
-        delayed_ctrl_job(context, view_user_data, job.job_id, renku_module, renku_ctrl)
+    delayed_ctrl_job(context, view_user_data, job.job_id, renku_module, renku_ctrl)
 
 
 @pytest.mark.service
@@ -675,8 +672,7 @@ def test_delay_unlink_dataset_job_failure(svc_client_cache, it_remote_repo_url_t
 
     from renku.ui.service.jobs.delayed_ctrl import delayed_ctrl_job
 
-    with pytest.raises(MigrationRequired):
-        delayed_ctrl_job(context, view_user_data, job.job_id, renku_module, renku_ctrl)
+    delayed_ctrl_job(context, view_user_data, job.job_id, renku_module, renku_ctrl)
 
 
 @pytest.mark.service

tests/service/jobs/test_datasets.py

@@ -25,10 +25,10 @@
 @pytest.mark.service
 @pytest.mark.integration
 @retry_failed
-def test_delay_migration_job(svc_client_cache, it_remote_repo_url_temp_branch, view_user_data):
+def test_delay_migration_job(svc_client_cache, it_remote_old_repo_url_temp_branch, view_user_data):
     """Verify delayed project migration."""
 
-    it_remote_repo_url, branch = it_remote_repo_url_temp_branch
+    it_remote_repo_url, branch = it_remote_old_repo_url_temp_branch
 
     context = ProjectMigrateRequest().load({"git_url": it_remote_repo_url, "ref": branch, "skip_docker_update": True})