Pass custom json payload to be handled by metadata storers

Author: ridoo

geonode/resource/manager.py

@@ -258,7 +258,7 @@ def delete(self, uuid: str, /, instance: ResourceBase = None) -> int:
                 ResourceBase.objects.filter(uuid=uuid).delete()
         return 0
 
-    def create(self, uuid: str, /, resource_type: typing.Optional[object] = None, defaults: dict = {}) -> ResourceBase:
+    def create(self, uuid: str, /, resource_type: typing.Optional[object] = None, defaults: dict = {}, *args, **kwargs) -> ResourceBase:
         if resource_type.objects.filter(uuid=uuid).exists():
             return resource_type.objects.filter(uuid=uuid).get()
         uuid = uuid or str(uuid4())
@@ -286,7 +286,7 @@ def create(self, uuid: str, /, resource_type: typing.Optional[object] = None, de
                         uuid, resource_type=resource_type, defaults=resource_dict
                     )
                 _resource.save()
-                resourcebase_post_save(_resource.get_real_instance())
+                resourcebase_post_save(_resource.get_real_instance(), **kwargs)
                 _resource.set_processing_state(enumerations.STATE_PROCESSED)
             except Exception as e:
                 logger.exception(e)

geonode/settings.py

@@ -2220,7 +2220,8 @@ def get_geonode_catalogue_service():
 List of modules that implement custom metadata storers that will be called when the metadata of a resource is saved
 """
 METADATA_STORERS = [
-    # 'geonode.resource.regions_storer.spatial_predicate_region_assignor',
+    # "geonode.resource.regions_storer.spatial_predicate_region_assignor",
+    # "geonode.upload.handlers.common.metadata_storer.store_metadata",
 ]
 
 

geonode/upload/api/serializer.py

@@ -35,6 +35,7 @@ class Meta:
             "store_spatial_files",
             "skip_existing_layers",
             "action",
+            "custom",
         )
 
     base_file = serializers.FileField()
@@ -43,6 +44,7 @@ class Meta:
     store_spatial_files = serializers.BooleanField(required=False, default=True)
     skip_existing_layers = serializers.BooleanField(required=False, default=False)
     action = serializers.CharField(required=True)
+    custom = serializers.JSONField(required=False, default={})
 
 
 class OverwriteImporterSerializer(ImporterSerializer):

geonode/upload/celery_tasks.py

@@ -352,20 +352,23 @@ def create_geonode_resource(
 
         handler = import_string(handler_module_path)()
         _overwrite = _exec.input_params.get("overwrite_existing_layer")
+        _custom = _exec.input_params.get("custom")
 
         if _overwrite:
             resource = handler.overwrite_geonode_resource(
                 layer_name=layer_name,
                 alternate=alternate,
                 execution_id=execution_id,
                 asset=_asset,
+                custom=_custom,
             )
         else:
             resource = handler.create_geonode_resource(
                 layer_name=layer_name,
                 alternate=alternate,
                 execution_id=execution_id,
                 asset=_asset,
+                custom=_custom,
             )
 
         # assign geonode resource to ExectionRequest

geonode/upload/handlers/README.md

@@ -159,7 +159,7 @@ class BaseVectorFileHandler(BaseHandler):
         return
 
     def overwrite_geonode_resource(
-        self, layer_name: str, alternate: str, execution_id: str, resource_type: Dataset = Dataset, asset=None
+        self, layer_name: str, alternate: str, execution_id: str, resource_type: Dataset = Dataset, asset=None, custom={}
     ):
         """
         Base function to override the resource into geonode. Each handler can specify

geonode/upload/handlers/base.py

@@ -148,7 +148,16 @@ def extract_params_from_data(_data):
         Remove from the _data the params that needs to save into the executionRequest object
         all the other are returned
         """
-        return []
+        return {
+            "resource_pk": _data.pop("resource_pk", None),
+            "dataset_title": _data.pop("dataset_title", None),
+            "original_zip_name": _data.pop("original_zip_name", None),
+            "skip_existing_layers": _data.pop("skip_existing_layers", "False"),
+            "overwrite_existing_layer": _data.pop("overwrite_existing_layer", False),
+            "store_spatial_file": _data.pop("store_spatial_files", "True"),
+            "action": _data.pop("action", "upload"),
+            "custom": _data.pop("custom", {}),
+        }, _data
 
     @staticmethod
     def perform_last_step(execution_id):
@@ -253,7 +262,9 @@ def publish_resources(resources: List[str], catalog, store, workspace):
         """
         return NotImplementedError
 
-    def create_geonode_resource(self, layer_name, alternate, execution_id, resource_type: Dataset = Dataset):
+    def create_geonode_resource(
+        self, layer_name, alternate, execution_id, resource_type: Dataset = Dataset, custom={}
+    ):
         """
         Base function to create the resource into geonode. Each handler can specify
         and handle the resource in a different way

geonode/upload/handlers/common/metadata.py

@@ -52,14 +52,7 @@ def extract_params_from_data(_data, action=None):
         Remove from the _data the params that needs to save into the executionRequest object
         all the other are returned
         """
-        return {
-            "dataset_title": _data.pop("dataset_title", None),
-            "skip_existing_layers": _data.pop("skip_existing_layers", "False"),
-            "overwrite_existing_layer": _data.pop("overwrite_existing_layer", False),
-            "resource_pk": _data.pop("resource_pk", None),
-            "store_spatial_file": _data.pop("store_spatial_files", "True"),
-            "action": _data.pop("action"),
-        }, _data
+        return BaseHandler.extract_params_from_data(_data)
 
     @staticmethod
     def perform_last_step(execution_id):

geonode/upload/handlers/common/metadata_storer.py

@@ -0,0 +1,15 @@
+import logging
+from geonode.resource.manager import update_resource
+
+logger = logging.getLogger(__name__)
+
+
+def store_metadata(instance, custom=None):
+    if not custom:
+        return instance
+    try:
+        return update_resource(instance, vals=custom)
+    except Exception as e:
+        logger.exception(e)
+        logger.error(f"Failed to update instance with custom payload: {custom}")
+    return instance

geonode/upload/handlers/common/raster.py

@@ -117,13 +117,7 @@ def extract_params_from_data(_data, action=None):
             title = json.loads(_data.get("defaults"))
             return {"title": title.pop("title"), "store_spatial_file": True}, _data
 
-        return {
-            "skip_existing_layers": _data.pop("skip_existing_layers", "False"),
-            "overwrite_existing_layer": _data.pop("overwrite_existing_layer", False),
-            "resource_pk": _data.pop("resource_pk", None),
-            "store_spatial_file": _data.pop("store_spatial_files", "True"),
-            "action": _data.pop("action", "upload"),
-        }, _data
+        return BaseHandler.extract_params_from_data(_data)
 
     @staticmethod
     def publish_resources(resources: List[str], catalog, store, workspace):
@@ -321,6 +315,7 @@ def create_geonode_resource(
         execution_id: str,
         resource_type: Dataset = Dataset,
         asset=None,
+        custom: object = {},
     ):
         """
         Base function to create the resource into geonode. Each handler can specify
@@ -357,6 +352,7 @@ def create_geonode_resource(
                 owner=_exec.user,
                 asset=asset,
             ),
+            custom=custom,
         )
 
         saved_dataset.refresh_from_db()
@@ -378,6 +374,7 @@ def overwrite_geonode_resource(
         execution_id: str,
         resource_type: Dataset = Dataset,
         asset=None,
+        custom={},
     ):
 
         _exec = self._get_execution_request_object(execution_id)
@@ -391,7 +388,7 @@ def overwrite_geonode_resource(
         if dataset.exists() and _overwrite:
             dataset = dataset.first()
 
-            dataset = resource_manager.update(dataset.uuid, instance=dataset)
+            dataset = resource_manager.update(dataset.uuid, instance=dataset, vals=custom)
 
             self.handle_xml_file(dataset, _exec)
             self.handle_sld_file(dataset, _exec)
@@ -403,7 +400,7 @@ def overwrite_geonode_resource(
             logger.warning(
                 f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created"
             )
-            return self.create_geonode_resource(layer_name, alternate, execution_id, resource_type, asset)
+            return self.create_geonode_resource(layer_name, alternate, execution_id, resource_type, asset, custom)
         elif not dataset.exists() and not _overwrite:
             logger.warning("The resource does not exists, please use 'create_geonode_resource' to create one")
         return

geonode/upload/handlers/common/remote.py

@@ -104,13 +104,15 @@ def extract_params_from_data(_data, action=None):
             title = json.loads(_data.get("defaults"))
             return {"title": title.pop("title"), "store_spatial_file": True}, _data
 
-        return {
-            "action": _data.pop("action", "upload"),
-            "title": _data.pop("title", None),
-            "url": _data.pop("url", None),
-            "type": _data.pop("type", None),
-            "overwrite_existing_layer": _data.pop("overwrite_existing_layer", False),
-        }, _data
+        return (
+            BaseHandler.extract_params_from_data(_data)[0]
+            | {
+                "title": _data.pop("title", None),
+                "url": _data.pop("url", None),
+                "type": _data.pop("type", None),
+            },
+            _data,
+        )
 
     def pre_validation(self, files, execution_id, **kwargs):
         """
@@ -199,6 +201,7 @@ def create_geonode_resource(
         execution_id: str,
         resource_type: ResourceBase = ResourceBase,
         asset=None,
+        custom: object = {},
     ):
         """
         Creating geonode base resource
@@ -213,6 +216,7 @@ def create_geonode_resource(
             None,
             resource_type=resource_type,
             defaults=self.generate_resource_payload(layer_name, alternate, asset, _exec, None, **params),
+            custom=custom,
         )
         resource_manager.set_thumbnail(None, instance=resource)
 
@@ -268,6 +272,7 @@ def overwrite_geonode_resource(
         execution_id: str,
         resource_type: Dataset = ResourceBase,
         asset=None,
+        custom={},
     ):
         _exec = self._get_execution_request_object(execution_id)
         resource = resource_type.objects.filter(alternate__icontains=alternate, owner=_exec.user)
@@ -278,15 +283,15 @@ def overwrite_geonode_resource(
         if resource.exists() and _overwrite:
             resource = resource.first()
 
-            resource = resource_manager.update(resource.uuid, instance=resource)
+            resource = resource_manager.update(resource.uuid, instance=resource, vals=custom)
             resource_manager.set_thumbnail(resource.uuid, instance=resource, overwrite=True)
             resource.refresh_from_db()
             return resource
         elif not resource.exists() and _overwrite:
             logger.warning(
                 f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created"
             )
-            return self.create_geonode_resource(layer_name, alternate, execution_id, resource_type, asset)
+            return self.create_geonode_resource(layer_name, alternate, execution_id, resource_type, asset, custom)
         elif not resource.exists() and not _overwrite:
             logger.warning("The resource does not exists, please use 'create_geonode_resource' to create one")
         return