align 1.1.x with master

Dockerfile

@@ -1,7 +1,7 @@
 FROM geonode/geonode-base:latest-ubuntu-22.04
 RUN rm -rf /usr/src/geonode
 RUN git clone https://github.com/GeoNode/geonode.git /usr/src/geonode
-RUN cd /usr/src/geonode && git fetch --all && git checkout master && cd -
+RUN cd /usr/src/geonode && git fetch --all && git checkout 4.4.x && cd -
 RUN mkdir -p /usr/src/importer
 
 RUN cd ..

README.md

@@ -1,6 +1,9 @@
 ![PyPI - Downloads](https://img.shields.io/pypi/dm/geonode-importer) ![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/GeoNode/geonode-importer/runtests.yml) ![GitHub top language](https://img.shields.io/github/languages/top/GeoNode/geonode-importer)
 
+# NOTE:
+GeoNode 5 (master branch) includes the importer in its core. This repository and the `geonode-importer` package will be maintained for GeoNode <= 4.4.x
 
+For more information https://github.com/GeoNode/geonode/issues/12368.
 
 # geonode-importer
 

importer/api/serializer.py

@@ -16,6 +16,7 @@ class Meta:
             "overwrite_existing_layer",
             "skip_existing_layers",
             "source",
+            "custom",
         )
 
     base_file = serializers.FileField()
@@ -25,3 +26,4 @@ class Meta:
     overwrite_existing_layer = serializers.BooleanField(required=False, default=False)
     skip_existing_layers = serializers.BooleanField(required=False, default=False)
     source = serializers.CharField(required=False, default="upload")
+    custom = serializers.JSONField(required=False, default={})

importer/api/views.py

@@ -17,7 +17,7 @@
 #
 #########################################################################
 import logging
-from urllib.parse import urljoin
+from urllib.parse import urljoin, urlsplit
 from django.conf import settings
 from django.urls import reverse
 from pathlib import Path
@@ -48,6 +48,7 @@
 from rest_framework.response import Response
 from geonode.assets.handlers import asset_handler_registry
 from geonode.assets.local import LocalAssetHandler
+from geonode.proxy.utils import proxy_urls_registry
 
 logger = logging.getLogger(__name__)
 
@@ -136,6 +137,7 @@ def create(self, request, *args, **kwargs):
             try:
                 # cloning data into a local folder
                 extracted_params, _data = handler.extract_params_from_data(_data)
+                extracted_params.update({"custom": _data.pop("custom", {})})
                 if _file:
                     storage_manager, asset, files = self._handle_asset(
                         request, asset_dir, storage_manager, _data, handler
@@ -145,6 +147,10 @@ def create(self, request, *args, **kwargs):
 
                 action = ExecutionRequestAction.IMPORT.value
 
+                if "url" in extracted_params:
+                    # we should register the hosts for the proxy
+                    proxy_urls_registry.register_host(urlsplit(extracted_params["url"]).hostname)
+
                 input_params = {
                     **{"files": files, "handler_module_path": str(handler)},
                     **extracted_params,

importer/celery_tasks.py

@@ -342,20 +342,23 @@ def create_geonode_resource(
 
         handler = import_string(handler_module_path)()
         _overwrite = _exec.input_params.get("overwrite_existing_layer")
+        _custom = _exec.input_params.get("custom")
 
         if _overwrite:
             resource = handler.overwrite_geonode_resource(
                 layer_name=layer_name,
                 alternate=alternate,
                 execution_id=execution_id,
                 asset=_asset,
+                custom=_custom,
             )
         else:
             resource = handler.create_geonode_resource(
                 layer_name=layer_name,
                 alternate=alternate,
                 execution_id=execution_id,
                 asset=_asset,
+                custom=_custom,
             )
 
         if _overwrite:
@@ -367,6 +370,9 @@ def create_geonode_resource(
                 handler_module_path, resource, _exec, **kwargs
             )
 
+        # assign geonode resource to ExectionRequest
+        orchestrator.update_execution_request_obj(_exec, {"geonode_resource": resource})
+
         # at the end recall the import_orchestrator for the next step
         import_orchestrator.apply_async(
             (

importer/handlers/base.py

@@ -236,7 +236,7 @@ def publish_resources(resources: List[str], catalog, store, workspace):
         return NotImplementedError
 
     def create_geonode_resource(
-        self, layer_name, alternate, execution_id, resource_type: Dataset = Dataset
+        self, layer_name, alternate, execution_id, resource_type: Dataset = Dataset, custom={}
     ):
         """
         Base function to create the resource into geonode. Each handler can specify

importer/handlers/common/metadata.py

@@ -78,6 +78,8 @@ def import_resource(self, files: dict, execution_id: str, **kwargs):
         self.handle_metadata_resource(_exec, dataset, original_handler)
 
         dataset.refresh_from_db()
+        # assign the resource to the execution_obj
+        orchestrator.update_execution_request_obj(_exec, {"geonode_resource": dataset})
 
         orchestrator.evaluate_execution_progress(
             execution_id, handler_module_path=str(self)

importer/handlers/common/raster.py

@@ -104,7 +104,7 @@ def extract_params_from_data(_data, action=None):
         """
         if action == exa.COPY.value:
             title = json.loads(_data.get("defaults"))
-            return {"title": title.pop("title")}, _data
+            return {"title": title.pop("title"), "store_spatial_file": True}, _data
 
         return {
             "skip_existing_layers": _data.pop("skip_existing_layers", "False"),
@@ -271,9 +271,7 @@ def import_resource(self, files: dict, execution_id: str, **kwargs) -> str:
                 overwrite_existing_layer=should_be_overwritten,
             ):
                 workspace = DataPublisher(None).workspace
-                user_datasets = Dataset.objects.filter(
-                    owner=_exec.user, alternate=f"{workspace.name}:{layer_name}"
-                )
+                user_datasets = Dataset.objects.filter(alternate=f"{workspace.name}:{layer_name}")
 
                 dataset_exists = user_datasets.exists()
 
@@ -312,6 +310,7 @@ def create_geonode_resource(
         execution_id: str,
         resource_type: Dataset = Dataset,
         asset=None,
+        custom={},
     ):
         """
         Base function to create the resource into geonode. Each handler can specify
@@ -348,6 +347,7 @@ def create_geonode_resource(
                 owner=_exec.user,
                 asset=asset,
             ),
+            custom=custom,
         )
 
         saved_dataset.refresh_from_db()
@@ -369,6 +369,7 @@ def overwrite_geonode_resource(
         execution_id: str,
         resource_type: Dataset = Dataset,
         asset=None,
+        custom={},        
     ):
 
         _exec = self._get_execution_request_object(execution_id)
@@ -397,7 +398,7 @@ def overwrite_geonode_resource(
                 f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created"
             )
             return self.create_geonode_resource(
-                layer_name, alternate, execution_id, resource_type, asset
+                layer_name, alternate, execution_id, resource_type, asset, custom
             )
         elif not dataset.exists() and not _overwrite:
             logger.warning(
@@ -503,17 +504,17 @@ def _import_resource_rollback(self, exec_id, istance_name=None, *args, **kwargs)
         """
         pass
 
-    def _publish_resource_rollback(self, exec_id, istance_name=None, *args, **kwargs):
+    def _publish_resource_rollback(self, exec_id, instance_name=None, *args, **kwargs):
         """
         We delete the resource from geoserver
         """
         logger.info(
-            f"Rollback publishing step in progress for execid: {exec_id} resource published was: {istance_name}"
+            f"Rollback publishing step in progress for execid: {exec_id} resource published was: {instance_name}"
         )
         exec_object = orchestrator.get_execution_object(exec_id)
         handler_module_path = exec_object.input_params.get("handler_module_path")
         publisher = DataPublisher(handler_module_path=handler_module_path)
-        publisher.delete_resource(istance_name)
+        publisher.delete_resource(instance_name)
 
 
 @importer_app.task(

importer/handlers/common/remote.py

@@ -72,7 +72,8 @@ def is_valid_url(url):
             r.raise_for_status()
         except requests.exceptions.Timeout:
             raise ImportException("Timed out")
-        except Exception:
+        except Exception as e:
+            logger.exception(e)
             raise ImportException("The provided URL is not reachable")
         return True
 
@@ -84,7 +85,7 @@ def extract_params_from_data(_data, action=None):
         """
         if action == exa.COPY.value:
             title = json.loads(_data.get("defaults"))
-            return {"title": title.pop("title")}, _data
+            return {"title": title.pop("title"), "store_spatial_file": True}, _data
 
         return {
             "source": _data.pop("source", "upload"),
@@ -179,6 +180,7 @@ def create_geonode_resource(
         execution_id: str,
         resource_type: ResourceBase = ResourceBase,
         asset=None,
+        custom={},
     ):
         """
         Creating geonode base resource
@@ -195,6 +197,7 @@ def create_geonode_resource(
             defaults=self.generate_resource_payload(
                 layer_name, alternate, asset, _exec, None, **params
             ),
+            custom=custom,
         )
         resource_manager.set_thumbnail(None, instance=resource)
 
@@ -252,6 +255,7 @@ def overwrite_geonode_resource(
         execution_id: str,
         resource_type: Dataset = ResourceBase,
         asset=None,
+        custom={},
     ):
         _exec = self._get_execution_request_object(execution_id)
         resource = resource_type.objects.filter(alternate__icontains=alternate, owner=_exec.user)
@@ -275,10 +279,10 @@ def overwrite_geonode_resource(
                 f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created"
             )
             return self.create_geonode_resource(
-                layer_name, alternate, execution_id, resource_type, asset
+                layer_name, alternate, execution_id, resource_type, asset, custom=custom,
             )
         elif not resource.exists() and not _overwrite:
             logger.warning(
                 "The resource does not exists, please use 'create_geonode_resource' to create one"
             )
-        return
+        return

importer/handlers/common/vector.py

@@ -114,7 +114,7 @@ def extract_params_from_data(_data, action=None):
         """
         if action == exa.COPY.value:
             title = json.loads(_data.get("defaults"))
-            return {"title": title.pop("title")}, _data
+            return {"title": title.pop("title"), "store_spatial_file": True}, _data
 
         return {
             "skip_existing_layers": _data.pop("skip_existing_layers", "False"),
@@ -220,7 +220,7 @@ def perform_last_step(execution_id):
         that the execution is completed
         """
         _exec = BaseHandler.perform_last_step(execution_id=execution_id)
-        if _exec and not _exec.input_params.get("store_spatial_file", False):
+        if _exec and not _exec.input_params.get("store_spatial_file", True):
             resources = ResourceHandlerInfo.objects.filter(execution_request=_exec)
             # getting all assets list
             assets = filter(None, [get_default_asset(x.resource) for x in resources])
@@ -568,6 +568,7 @@ def create_geonode_resource(
         execution_id: str,
         resource_type: Dataset = Dataset,
         asset=None,
+        custom={},
     ):
         """
         Base function to create the resource into geonode. Each handler can specify
@@ -597,6 +598,7 @@ def create_geonode_resource(
             defaults=self.generate_resource_payload(
                 layer_name, alternate, asset, _exec, workspace
             ),
+            custom=custom,
         )
 
         saved_dataset.refresh_from_db()
@@ -607,7 +609,7 @@ def create_geonode_resource(
         resource_manager.set_thumbnail(None, instance=saved_dataset)
 
         ResourceBase.objects.filter(alternate=alternate).update(dirty_state=False)
-
+        
         saved_dataset.refresh_from_db()
         return saved_dataset
 
@@ -631,6 +633,7 @@ def overwrite_geonode_resource(
         execution_id: str,
         resource_type: Dataset = Dataset,
         asset=None,
+        custom={},
     ):
         _exec = self._get_execution_request_object(execution_id)
 
@@ -662,7 +665,7 @@ def overwrite_geonode_resource(
                 f"The dataset required {alternate} does not exists, but an overwrite is required, the resource will be created"
             )
             return self.create_geonode_resource(
-                layer_name, alternate, execution_id, resource_type, asset
+                layer_name, alternate, execution_id, resource_type, asset, custom=custom,
             )
         elif not dataset.exists() and not _overwrite:
             logger.warning(
@@ -805,13 +808,13 @@ def _import_resource_rollback(self, exec_id, instance_name=None, *args, **kwargs
                     "Dynamic model does not exists, removing ogr2ogr table in progress"
                 )
                 if instance_name is None:
-                    logger.info("No table created, skipping...")
+                    logger.warning("No table created, skipping...")
                     return
                 db_name = os.getenv("DEFAULT_BACKEND_DATASTORE", "datastore")
                 with connections[db_name].cursor() as cursor:
                     cursor.execute(f"DROP TABLE {instance_name}")
             except Exception as e:
-                logger.info(e)
+                logger.warning(e)
                 pass
 
     def _publish_resource_rollback(self, exec_id, instance_name=None, *args, **kwargs):

importer/handlers/remote/tiles3d.py

@@ -61,9 +61,10 @@ def create_geonode_resource(
         execution_id: str,
         resource_type: Dataset = ResourceBase,
         asset=None,
+        custom={},
     ):
         resource = super().create_geonode_resource(
-            layer_name, alternate, execution_id, resource_type, asset
+            layer_name, alternate, execution_id, resource_type, asset, custom=custom
         )
         _exec = orchestrator.get_execution_object(exec_id=execution_id)
         try:

importer/handlers/remote/wms.py

@@ -115,13 +115,14 @@ def create_geonode_resource(
         execution_id: str,
         resource_type: Dataset = ...,
         asset=None,
+        custom={},
     ):
         """
         Use the default RemoteResourceHandler to create the geonode resource
         after that, we assign the bbox and re-generate the thumbnail
         """
         resource = super().create_geonode_resource(
-            layer_name, alternate, execution_id, Dataset, asset
+            layer_name, alternate, execution_id, Dataset, asset, custom=custom,
         )
         _exec = orchestrator.get_execution_object(execution_id)
         remote_bbox = _exec.input_params.get("bbox")
@@ -151,7 +152,8 @@ def generate_resource_payload(
             .encode("utf-8", "ignore")
             .decode("utf-8")
             .replace(".", "")
-            .replace("/", ""),
+            .replace("/", "")
+            .replace(":", ""),
             ptype="gxp_wmscsource",
             ows_url=_exec.input_params.get("ows_url"),
         )

importer/handlers/shapefile/handler.py

@@ -85,7 +85,7 @@ def extract_params_from_data(_data, action=None):
         """
         if action == exa.COPY.value:
             title = json.loads(_data.get("defaults"))
-            return {"title": title.pop("title")}, _data
+            return {"title": title.pop("title"), "store_spatial_file": True}, _data
 
         additional_params = {
             "skip_existing_layers": _data.pop("skip_existing_layers", "False"),
@@ -164,7 +164,7 @@ def create_ogr2ogr_command(files, original_name, ovverwrite_layer, alternate):
         ):
             additional_options.append("-nlt PROMOTE_TO_MULTI")
         if encoding:
-            additional_options.append(f"-lco ENCODING={encoding}")
+            additional_options.append(f"--config SHAPE_ENCODING {encoding}")
 
         return (
             f"{base_command } -lco precision=no -lco GEOMETRY_NAME={BaseVectorFileHandler().default_geometry_column_name} "

importer/handlers/shapefile/tests.py

@@ -121,7 +121,7 @@ def test_should_create_ogr2ogr_command_with_encoding_from_cst(self):
             actual = self.handler.create_ogr2ogr_command(shp_with_cst, "a", False, "a")
 
             _file.assert_called_once_with(cst_file, "r")
-            self.assertIn("ENCODING=UTF-8", actual)
+            self.assertIn("--config SHAPE_ENCODING UTF-8", actual)
 
     @patch("importer.handlers.common.vector.Popen")
     def test_import_with_ogr2ogr_without_errors_should_call_the_right_command(