From c1c8637c7a210c494e0c177f841f7ba5dccb5a1e Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Tue, 4 Feb 2025 10:48:03 +0000 Subject: [PATCH 1/9] Pass custom json payload to be handled by metadata storers --- geonode/resource/manager.py | 6 ++++-- geonode/resource/metadata_storer.py | 15 +++++++++++++++ geonode/resource/tests.py | 19 ++++++++++++++++++- geonode/settings.py | 3 ++- 4 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 geonode/resource/metadata_storer.py diff --git a/geonode/resource/manager.py b/geonode/resource/manager.py index c196feddb67..3e1be077fca 100644 --- a/geonode/resource/manager.py +++ b/geonode/resource/manager.py @@ -313,7 +313,9 @@ def delete(self, uuid: str, /, instance: ResourceBase = None) -> int: ResourceBase.objects.filter(uuid=uuid).delete() return 0 - def create(self, uuid: str, /, resource_type: typing.Optional[object] = None, defaults: dict = {}) -> ResourceBase: + def create( + self, uuid: str, /, resource_type: typing.Optional[object] = None, defaults: dict = {}, *args, **kwargs + ) -> ResourceBase: if resource_type.objects.filter(uuid=uuid).exists(): return resource_type.objects.filter(uuid=uuid).get() uuid = uuid or str(uuid4()) @@ -341,7 +343,7 @@ def create(self, uuid: str, /, resource_type: typing.Optional[object] = None, de uuid, resource_type=resource_type, defaults=resource_dict ) _resource.save() - resourcebase_post_save(_resource.get_real_instance()) + resourcebase_post_save(_resource.get_real_instance(), **kwargs) _resource.set_processing_state(enumerations.STATE_PROCESSED) except Exception as e: logger.exception(e) diff --git a/geonode/resource/metadata_storer.py b/geonode/resource/metadata_storer.py new file mode 100644 index 00000000000..d62f3837f4d --- /dev/null +++ b/geonode/resource/metadata_storer.py @@ -0,0 +1,15 @@ +import logging +from geonode.resource.manager import update_resource + +logger = logging.getLogger(__name__) + + +def store_metadata(instance, custom=None): + if not custom: + return instance + try: + return update_resource(instance, vals=custom) + except Exception as e: + logger.exception(e) + logger.error(f"Failed to update instance with custom payload: {custom}") + return instance diff --git a/geonode/resource/tests.py b/geonode/resource/tests.py index 3d92e9de193..4ec9ceb962e 100644 --- a/geonode/resource/tests.py +++ b/geonode/resource/tests.py @@ -24,13 +24,14 @@ from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist +from django.test import override_settings from geonode.groups.models import GroupProfile from geonode.base.populate_test_data import create_models from geonode.resource.utils import resourcebase_post_save from geonode.tests.base import GeoNodeBaseTestSupport from geonode.resource.manager import ResourceManager -from geonode.base.models import LinkedResource, ResourceBase +from geonode.base.models import License, LinkedResource, ResourceBase from geonode.layers.models import Dataset from geonode.services.models import Service from geonode.documents.models import Document @@ -110,6 +111,22 @@ def test_create(self): res = self.rm.create(new_uuid, resource_type=Dataset, defaults=dataset_defaults) self.assertEqual(res, Dataset.objects.get(uuid=new_uuid)) + @override_settings(METADATA_STORERS=["geonode.resource.metadata_storer.store_metadata"]) + def test_create_passing_custom_to_post_save(self): + license = License.objects.all().first() + public_group, _public_created = GroupProfile.objects.get_or_create( + slug="public_group", title="public_group", access="public" + ) + dataset = self.rm.create( + None, + resource_type=Dataset, + defaults=dict(owner=self.user, title="test"), + custom=dict(group=public_group.pk, license=license), + ) + self.assertIsNotNone(dataset.license) + self.assertIsNotNone(dataset.group) + self.assertEqual(public_group.pk, dataset.group.pk) + def test_update(self): dt = create_single_dataset("test_update_dataset") vals = {"name": "new_name_test_update_dataset"} diff --git a/geonode/settings.py b/geonode/settings.py index 71e45491684..69e69c54063 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -2226,7 +2226,8 @@ def get_geonode_catalogue_service(): List of modules that implement custom metadata storers that will be called when the metadata of a resource is saved """ METADATA_STORERS = [ - # 'geonode.resource.regions_storer.spatial_predicate_region_assignor', + # "geonode.resource.regions_storer.spatial_predicate_region_assignor", + # "geonode.resource.metadata_storer.store_metadata", ] From c0fa2f3c352a1521eebb9b45b659f388703366df Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Fri, 21 Feb 2025 17:52:36 +0000 Subject: [PATCH 2/9] Fix test to save Group instead of GroupProfile --- geonode/resource/tests.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/geonode/resource/tests.py b/geonode/resource/tests.py index 4ec9ceb962e..6be44485853 100644 --- a/geonode/resource/tests.py +++ b/geonode/resource/tests.py @@ -31,7 +31,7 @@ from geonode.resource.utils import resourcebase_post_save from geonode.tests.base import GeoNodeBaseTestSupport from geonode.resource.manager import ResourceManager -from geonode.base.models import License, LinkedResource, ResourceBase +from geonode.base.models import License, LinkedResource, ResourceBase, Group from geonode.layers.models import Dataset from geonode.services.models import Service from geonode.documents.models import Document @@ -114,18 +114,16 @@ def test_create(self): @override_settings(METADATA_STORERS=["geonode.resource.metadata_storer.store_metadata"]) def test_create_passing_custom_to_post_save(self): license = License.objects.all().first() - public_group, _public_created = GroupProfile.objects.get_or_create( - slug="public_group", title="public_group", access="public" - ) + group = Group.objects.all().first() dataset = self.rm.create( None, resource_type=Dataset, defaults=dict(owner=self.user, title="test"), - custom=dict(group=public_group.pk, license=license), + custom=dict(group=group.pk, license=license), ) self.assertIsNotNone(dataset.license) self.assertIsNotNone(dataset.group) - self.assertEqual(public_group.pk, dataset.group.pk) + self.assertEqual(group.pk, dataset.group.pk) def test_update(self): dt = create_single_dataset("test_update_dataset") From 8661c7c1de1631d832889146e07e881f0be96e40 Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Tue, 4 Mar 2025 14:47:31 +0100 Subject: [PATCH 3/9] Also pass the args to post_save --- geonode/resource/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geonode/resource/manager.py b/geonode/resource/manager.py index 3e1be077fca..ef837716a1d 100644 --- a/geonode/resource/manager.py +++ b/geonode/resource/manager.py @@ -343,7 +343,7 @@ def create( uuid, resource_type=resource_type, defaults=resource_dict ) _resource.save() - resourcebase_post_save(_resource.get_real_instance(), **kwargs) + resourcebase_post_save(_resource.get_real_instance(), *args, **kwargs) _resource.set_processing_state(enumerations.STATE_PROCESSED) except Exception as e: logger.exception(e) From d9e51bc9fef7f3a7937b7d64bb8a66046d2e7139 Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Tue, 11 Mar 2025 10:52:47 +0100 Subject: [PATCH 4/9] Move test to isolated test class to avoid side effects during tests --- geonode/base/tests.py | 24 ++++++++++++++++++++++++ geonode/resource/tests.py | 17 +---------------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/geonode/base/tests.py b/geonode/base/tests.py index 31681475624..e1516ea4416 100644 --- a/geonode/base/tests.py +++ b/geonode/base/tests.py @@ -57,6 +57,8 @@ HierarchicalKeyword, ResourceBase, MenuPlaceholder, + License, + Group, Menu, MenuItem, Configuration, @@ -1293,6 +1295,28 @@ def test_regions_are_assigned_if_handler_is_used(self): self.assertEqual("Global", dataset.regions.first().name) +class TestMetadataStorer(GeoNodeBaseTestSupport): + + @override_settings(METADATA_STORERS=["geonode.resource.metadata_storer.store_metadata"]) + def test_create_passing_custom_to_post_save(self): + from geonode.resource.manager import resource_manager + User = get_user_model() + user = User.objects.create(username="test", email="test@test.com") + license = License.objects.all().first() + group = Group.objects.all().first() + dataset = resource_manager.create( + str(uuid4()), + resource_type=Dataset, + defaults=dict(owner=user, title="test"), + custom=dict(group=group.pk, license=license), + ) + self.assertIsNotNone(dataset.license) + self.assertIsNotNone(dataset.group) + self.assertEqual(group.pk, dataset.group.pk) + + resource_manager.delete(dataset) + + class LinkedResourcesTest(GeoNodeBaseTestSupport): def test_autocomplete_linked_resource(self): d = [] diff --git a/geonode/resource/tests.py b/geonode/resource/tests.py index 6be44485853..3d92e9de193 100644 --- a/geonode/resource/tests.py +++ b/geonode/resource/tests.py @@ -24,14 +24,13 @@ from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist -from django.test import override_settings from geonode.groups.models import GroupProfile from geonode.base.populate_test_data import create_models from geonode.resource.utils import resourcebase_post_save from geonode.tests.base import GeoNodeBaseTestSupport from geonode.resource.manager import ResourceManager -from geonode.base.models import License, LinkedResource, ResourceBase, Group +from geonode.base.models import LinkedResource, ResourceBase from geonode.layers.models import Dataset from geonode.services.models import Service from geonode.documents.models import Document @@ -111,20 +110,6 @@ def test_create(self): res = self.rm.create(new_uuid, resource_type=Dataset, defaults=dataset_defaults) self.assertEqual(res, Dataset.objects.get(uuid=new_uuid)) - @override_settings(METADATA_STORERS=["geonode.resource.metadata_storer.store_metadata"]) - def test_create_passing_custom_to_post_save(self): - license = License.objects.all().first() - group = Group.objects.all().first() - dataset = self.rm.create( - None, - resource_type=Dataset, - defaults=dict(owner=self.user, title="test"), - custom=dict(group=group.pk, license=license), - ) - self.assertIsNotNone(dataset.license) - self.assertIsNotNone(dataset.group) - self.assertEqual(group.pk, dataset.group.pk) - def test_update(self): dt = create_single_dataset("test_update_dataset") vals = {"name": "new_name_test_update_dataset"} From 7cf0bfd26a6570aab50fe143659b530a4d1d2637 Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Tue, 11 Mar 2025 11:50:53 +0100 Subject: [PATCH 5/9] Clean globally cached items after test --- geonode/base/tests.py | 3 --- geonode/tests/base.py | 8 ++++++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/geonode/base/tests.py b/geonode/base/tests.py index e1516ea4416..53e7f0ecfa4 100644 --- a/geonode/base/tests.py +++ b/geonode/base/tests.py @@ -1299,7 +1299,6 @@ class TestMetadataStorer(GeoNodeBaseTestSupport): @override_settings(METADATA_STORERS=["geonode.resource.metadata_storer.store_metadata"]) def test_create_passing_custom_to_post_save(self): - from geonode.resource.manager import resource_manager User = get_user_model() user = User.objects.create(username="test", email="test@test.com") license = License.objects.all().first() @@ -1313,8 +1312,6 @@ def test_create_passing_custom_to_post_save(self): self.assertIsNotNone(dataset.license) self.assertIsNotNone(dataset.group) self.assertEqual(group.pk, dataset.group.pk) - - resource_manager.delete(dataset) class LinkedResourcesTest(GeoNodeBaseTestSupport): diff --git a/geonode/tests/base.py b/geonode/tests/base.py index ac8587f4307..c42c4d4e9be 100644 --- a/geonode/tests/base.py +++ b/geonode/tests/base.py @@ -66,6 +66,14 @@ def get_type(cls): def setUp(self): super().setUp() faulthandler.enable() + + def tearDown(self): + # cleanup globally cached items + globally_cached = ["storer_modules", "user_deletion_modules", "geoapp_subtypes"] + for cached in globally_cached: + if cached in globals(): + del globals()[cached] + return super().tearDown() class GeoNodeLiveTestSupport(GeoNodeBaseTestSupport, LiveServerTestCase): From f003331ad2ae3fbacd076fc2963969bcb049f9ac Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Tue, 11 Mar 2025 13:14:26 +0100 Subject: [PATCH 6/9] Fix formatting --- geonode/tests/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geonode/tests/base.py b/geonode/tests/base.py index c42c4d4e9be..f748a512a31 100644 --- a/geonode/tests/base.py +++ b/geonode/tests/base.py @@ -66,7 +66,7 @@ def get_type(cls): def setUp(self): super().setUp() faulthandler.enable() - + def tearDown(self): # cleanup globally cached items globally_cached = ["storer_modules", "user_deletion_modules", "geoapp_subtypes"] From 92dd792181887939c383dd375d7382db9642c25e Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Thu, 13 Mar 2025 10:29:20 +0100 Subject: [PATCH 7/9] Provide patched method to get non-cached storer stettings --- geonode/base/tests.py | 57 ++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/geonode/base/tests.py b/geonode/base/tests.py index 53e7f0ecfa4..901ea28a7c7 100644 --- a/geonode/base/tests.py +++ b/geonode/base/tests.py @@ -27,6 +27,7 @@ from unittest.mock import patch, Mock from guardian.shortcuts import assign_perm +from django.utils.module_loading import import_string from django.db.utils import IntegrityError, OperationalError from django.core.exceptions import ObjectDoesNotExist from django.conf import settings @@ -1246,6 +1247,13 @@ def test_keyword_raise_db_error(self, add_root_mocked): ) +def _cache_less_call_storers(instance, custom={}): + storer_module_path = settings.METADATA_STORERS if hasattr(settings, "METADATA_STORERS") else [] + storers = [import_string(storer_path) for storer_path in storer_module_path] + for storer in storers: + storer(instance, custom) + return instance + class TestRegions(GeoNodeBaseTestSupport): def setUp(self): self.dataset_inside_region = GEOSGeometry( @@ -1283,35 +1291,38 @@ def test_region_assignment_for_extent(self): region.is_assignable_to_geom(self.dataset_outside_region), "Extent outside a region should be assigned" ) - @override_settings(METADATA_STORERS=["geonode.resource.regions_storer.spatial_predicate_region_assignor"]) + @patch("geonode.resource.utils.call_storers", _cache_less_call_storers) def test_regions_are_assigned_if_handler_is_used(self): - dataset = resource_manager.create( - None, - resource_type=Dataset, - defaults=dict(owner=get_user_model().objects.first(), title="test_region_dataset", is_approved=True), - ) - self.assertTrue(dataset.regions.exists()) - self.assertEqual(1, dataset.regions.count()) - self.assertEqual("Global", dataset.regions.first().name) + with override_settings(METADATA_STORERS=["geonode.resource.regions_storer.spatial_predicate_region_assignor"]): + dataset = resource_manager.create( + None, + resource_type=Dataset, + defaults=dict(owner=get_user_model().objects.first(), title="test_region_dataset", is_approved=True), + ) + self.assertTrue(dataset.regions.exists()) + self.assertEqual(1, dataset.regions.count()) + self.assertEqual("Global", dataset.regions.first().name) class TestMetadataStorer(GeoNodeBaseTestSupport): - @override_settings(METADATA_STORERS=["geonode.resource.metadata_storer.store_metadata"]) + @patch("geonode.resource.utils.call_storers", _cache_less_call_storers) def test_create_passing_custom_to_post_save(self): - User = get_user_model() - user = User.objects.create(username="test", email="test@test.com") - license = License.objects.all().first() - group = Group.objects.all().first() - dataset = resource_manager.create( - str(uuid4()), - resource_type=Dataset, - defaults=dict(owner=user, title="test"), - custom=dict(group=group.pk, license=license), - ) - self.assertIsNotNone(dataset.license) - self.assertIsNotNone(dataset.group) - self.assertEqual(group.pk, dataset.group.pk) + + with override_settings(METADATA_STORERS=["geonode.resource.metadata_storer.store_metadata"]): + User = get_user_model() + user = User.objects.create(username="test", email="test@test.com") + license = License.objects.all().first() + group = Group.objects.all().first() + dataset = resource_manager.create( + str(uuid4()), + resource_type=Dataset, + defaults=dict(owner=user, title="test"), + custom=dict(group=group.pk, license=license), + ) + self.assertIsNotNone(dataset.license) + self.assertIsNotNone(dataset.group) + self.assertEqual(group.pk, dataset.group.pk) class LinkedResourcesTest(GeoNodeBaseTestSupport): From 651c5627e3a08faba3ab70cb93142a0753514a02 Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Thu, 13 Mar 2025 10:30:03 +0100 Subject: [PATCH 8/9] Run black formatting --- geonode/base/tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/geonode/base/tests.py b/geonode/base/tests.py index 901ea28a7c7..f419e2da02a 100644 --- a/geonode/base/tests.py +++ b/geonode/base/tests.py @@ -1254,6 +1254,7 @@ def _cache_less_call_storers(instance, custom={}): storer(instance, custom) return instance + class TestRegions(GeoNodeBaseTestSupport): def setUp(self): self.dataset_inside_region = GEOSGeometry( @@ -1305,10 +1306,10 @@ def test_regions_are_assigned_if_handler_is_used(self): class TestMetadataStorer(GeoNodeBaseTestSupport): - + @patch("geonode.resource.utils.call_storers", _cache_less_call_storers) def test_create_passing_custom_to_post_save(self): - + with override_settings(METADATA_STORERS=["geonode.resource.metadata_storer.store_metadata"]): User = get_user_model() user = User.objects.create(username="test", email="test@test.com") From 94514ec96da64feb58a07ed45f8d2803b93287e6 Mon Sep 17 00:00:00 2001 From: Henning Bredel Date: Thu, 13 Mar 2025 10:32:58 +0100 Subject: [PATCH 9/9] Reset tearDown method Tried to reset the global cache which did not work --- geonode/tests/base.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/geonode/tests/base.py b/geonode/tests/base.py index f748a512a31..ac8587f4307 100644 --- a/geonode/tests/base.py +++ b/geonode/tests/base.py @@ -67,14 +67,6 @@ def setUp(self): super().setUp() faulthandler.enable() - def tearDown(self): - # cleanup globally cached items - globally_cached = ["storer_modules", "user_deletion_modules", "geoapp_subtypes"] - for cached in globally_cached: - if cached in globals(): - del globals()[cached] - return super().tearDown() - class GeoNodeLiveTestSupport(GeoNodeBaseTestSupport, LiveServerTestCase): integration = True