[Fixes #13479] cannot upload geojson/3dtiles from zip file (#13480)

* [Fixes #13479] cannot upload geojson/3dtiles from zip file

Author: mattiagiupponi

geonode/upload/handlers/base.py

@@ -79,6 +79,10 @@ def get_task_list(cls, action) -> tuple:
             raise Exception("The requested action is not implemented yet")
         return cls.TASKS.get(action)
 
+    @property
+    def have_table(self):
+        return True
+
     @property
     def default_geometry_column_name(self):
         return "geometry"
@@ -212,6 +216,12 @@ def pre_processing(self, files, execution_id, **kwargs):
             )
             # updating the execution id params
             orchestrator.update_execution_request_obj(_exec_obj, {"input_params": _data})
+            # removing zip file
+            if "zip_file" in files and os.path.exists(files["zip_file"]):
+                os.remove(files["zip_file"])
+            if "kmz_file" in files and os.path.exists(files["kmz_file"]):
+                os.remove(files["kmz_file"])
+
         return _data, execution_id
 
     def fixup_name(self, name):

geonode/upload/handlers/common/remote.py

@@ -69,6 +69,10 @@ def has_serializer(data) -> bool:
             return RemoteResourceSerializer
         return False
 
+    @property
+    def have_table(self):
+        return False
+
     @staticmethod
     def can_handle(_data) -> bool:
         """

geonode/upload/handlers/common/vector.py

@@ -101,6 +101,10 @@ class BaseVectorFileHandler(BaseHandler):
         ira.UPSERT.value: ("start_import", "geonode.upload.upsert_data", "geonode.upload.refresh_geonode_resource"),
     }
 
+    @property
+    def have_table(self):
+        return True
+
     @property
     def default_geometry_column_name(self):
         return "geometry"
@@ -179,6 +183,7 @@ def extract_params_from_data(_data, action=None):
             "resource_pk": _data.pop("resource_pk", None),
             "store_spatial_file": _data.pop("store_spatial_files", "True"),
             "action": _data.pop("action", "upload"),
+            "upsert_key": _data.pop("upsert_key", None),
         }, _data
 
     @staticmethod
@@ -687,7 +692,7 @@ def create_geonode_resource(
         saved_dataset.refresh_from_db()
 
         # if dynamic model is enabled, we can save up with is the primary key of the table
-        if settings.IMPORTER_ENABLE_DYN_MODELS:
+        if settings.IMPORTER_ENABLE_DYN_MODELS and self.have_table:
             from django.db import connections
 
             column = None
@@ -1182,7 +1187,7 @@ def _process_feature(
 
     def extract_upsert_key(self, exec_obj, dynamic_model_instance):
         # first we check if the upsert key is passed by the call
-        key = exec_obj.input_params.get("upsert_key")
+        key = exec_obj.input_params.get("upsert_key", "ogc_fid")
         if not key:
             # if the upsert key is not passed, we use the primary key as upsert key
             # the primary key is defined in the Fields of the dynamic model

geonode/upload/handlers/geojson/handler.py

@@ -19,6 +19,7 @@
 import json
 import logging
 import os
+import zipfile
 from geonode.upload.utils import UploadLimitValidator
 from geonode.upload.handlers.common.vector import BaseVectorFileHandler
 from osgeo import ogr
@@ -74,6 +75,11 @@ def can_handle(_data) -> bool:
                 if isinstance(base, str):
                     with open(base, "r") as f:
                         _file = json.loads(f.read())
+                elif "zip_file" in _data:
+                    # if we have a zipfile we need to read the file content before proceed
+                    with zipfile.ZipFile(_data["zip_file"], "r") as z:
+                        with z.open(_file.name) as inner_file:
+                            _file = json.loads(inner_file.read().decode("utf-8"))
                 else:
                     _file = json.loads(base.read())
 

geonode/upload/handlers/remote/tiles3d.py

@@ -53,6 +53,10 @@ def can_handle(_data) -> bool:
             return True
         return False
 
+    @property
+    def have_table(self):
+        return False
+
     @staticmethod
     def is_valid_url(url, **kwargs):
         BaseRemoteResourceHandler.is_valid_url(url)

geonode/upload/handlers/tiles3d/handler.py

@@ -21,6 +21,8 @@
 import os
 from pathlib import Path
 import math
+import zipfile
+from geonode.assets.models import LocalAsset
 from geonode.layers.models import Dataset
 from geonode.resource.enumerator import ExecutionRequestAction as exa
 from geonode.upload.utils import UploadLimitValidator
@@ -84,7 +86,7 @@ def can_handle(_data) -> bool:
             if not base:
                 return False
             ext = base.split(".")[-1] if isinstance(base, str) else base.name.split(".")[-1]
-            if ext in ["json"] and Tiles3DFileHandler.is_3dtiles_json(base):
+            if ext in ["json"] and Tiles3DFileHandler.is_3dtiles_json(base, **_data):
                 return True
         except Exception:
             return False
@@ -123,9 +125,15 @@ def is_valid(files, user, **kwargs):
         return True
 
     @staticmethod
-    def is_3dtiles_json(_file):
-        with open(_file, "r") as _readed_file:
-            _file = json.loads(_readed_file.read())
+    def is_3dtiles_json(_file, **kwargs):
+        if "zip_file" in kwargs:
+            # if we have a zipfile we need to read the file content before proceed
+            with zipfile.ZipFile(kwargs["zip_file"], "r") as z:
+                with z.open(_file.name) as inner_file:
+                    _file = json.loads(inner_file.read().decode("utf-8"))
+        else:
+            with open(_file, "r") as _readed_file:
+                _file = json.loads(_readed_file.read())
             # required key described in the specification of 3dtiles
             # https://docs.ogc.org/cs/22-025r4/22-025r4.html#toc92
         is_valid = all(key in _file.keys() for key in ("asset", "geometricError", "root"))
@@ -225,25 +233,28 @@ def create_geonode_resource(
         asset=None,
     ):
         # we want just the tileset.json as location of the asset
-        asset.location = [path for path in asset.location if path.endswith(".json")]
-        asset.save()
+        # asset.location = [path for path in asset.location if path.endswith(".json")]
+        # asset.save()
+        exec_obj = orchestrator.get_execution_object(execution_id)
 
         resource = super().create_geonode_resource(layer_name, alternate, execution_id, ResourceBase, asset)
+        asset = self.create_asset_and_link(resource, files=exec_obj.input_params["files"], action=exec_obj.action)
 
-        # fixing-up bbox for the 3dtile object
-        js_file = None
-        with open(asset.location[0]) as _file:
-            js_file = json.loads(_file.read())
+        if isinstance(asset, LocalAsset):
+            # fixing-up bbox for the 3dtile object
+            js_file = None
+            with open(asset.location[0]) as _file:
+                js_file = json.loads(_file.read())
 
-        if not js_file:
-            return resource
+            if not js_file:
+                return resource
 
-        if self._has_region(js_file):
-            resource = self.set_bbox_from_region(js_file, resource=resource)
-        elif self._has_sphere(js_file):
-            resource = self.set_bbox_from_boundingVolume_sphere(js_file, resource=resource)
-        else:
-            resource = self.set_bbox_from_boundingVolume(js_file, resource=resource)
+            if self._has_region(js_file):
+                resource = self.set_bbox_from_region(js_file, resource=resource)
+            elif self._has_sphere(js_file):
+                resource = self.set_bbox_from_boundingVolume_sphere(js_file, resource=resource)
+            else:
+                resource = self.set_bbox_from_boundingVolume(js_file, resource=resource)
 
         return resource
 

geonode/upload/handlers/tiles3d/tests.py

@@ -301,8 +301,6 @@ def test_set_bbox_from_bounding_volume_wit_transform(self):
         self.assertEqual(resource.bbox_y0, 40.03610390613993)
         self.assertEqual(resource.bbox_y1, 40.04895731654794)
 
-        os.remove("/tmp/tileset.json")
-
     def test_set_bbox_from_bounding_volume_without_transform(self):
         # https://github.com/geosolutions-it/MapStore2/blob/master/web/client/api/__tests__/ThreeDTiles-test.js#L147-L180
         tilesetjson_file = {
@@ -347,8 +345,6 @@ def test_set_bbox_from_bounding_volume_without_transform(self):
         self.assertEqual(resource.bbox_y0, 86.81078622278615)
         self.assertEqual(resource.bbox_y1, 86.81124587650872)
 
-        os.remove("/tmp/tileset.json")
-
     def test_set_bbox_from_bounding_volume_sphere_with_transform(self):
         # https://github.com/geosolutions-it/MapStore2/blob/master/web/client/api/__tests__/ThreeDTiles-test.js#L102-L146
         tilesetjson_file = {
@@ -396,8 +392,6 @@ def test_set_bbox_from_bounding_volume_sphere_with_transform(self):
         self.assertAlmostEqual(resource.bbox_y0, 40.042485645323616)
         self.assertAlmostEqual(resource.bbox_y1, 40.042575577526556)
 
-        os.remove("/tmp/tileset.json")
-
     def test_set_bbox_from_bounding_volume_sphere_without_transform(self):
         # https://github.com/geosolutions-it/MapStore2/blob/master/web/client/api/__tests__/ThreeDTiles-test.js#L53C4-L79C8
         tilesetjson_file = {
@@ -425,8 +419,6 @@ def test_set_bbox_from_bounding_volume_sphere_without_transform(self):
         self.assertEqual(resource.bbox_y0, 86.81097108354597)
         self.assertEqual(resource.bbox_y1, 86.8110610157489)
 
-        os.remove("/tmp/tileset.json")
-
     def test_set_bbox_from_bounding_volume_sphere_with_center_zero_without_transform(self):
         # https://github.com/geosolutions-it/MapStore2/blob/master/web/client/api/__tests__/ThreeDTiles-test.js#L53C4-L79C8
         # This test should not extract bbox from boundingVolume sphere with center 0, 0, 0
@@ -450,8 +442,6 @@ def test_set_bbox_from_bounding_volume_sphere_with_center_zero_without_transform
         )
         self.assertTrue(resource.bbox == self.default_bbox)
 
-        os.remove("/tmp/tileset.json")
-
     def _generate_execid_asset(self):
         exec_id = orchestrator.create_execution_request(
             user=self.owner,

geonode/upload/tests/end2end/test_env2end_upsert.py

@@ -80,7 +80,10 @@ def setUp(self) -> None:
         self.admin.is_staff = True
         self.admin.save()
         for el in Dataset.objects.all():
-            el.delete()
+            try:
+                el.delete()
+            except:  # noqa
+                continue
 
     def tearDown(self) -> None:
         super().tearDown()
@@ -193,7 +196,7 @@ def test_import_shapefile_upsert(self):
         payload = {_filename: open(_file, "rb") for _filename, _file in self.upsert_geojson.items()}
         payload["resource_pk"] = prev_dataset.pk
         payload["action"] = "upsert"
-        payload["upsert_key"] = "ogc_fid"
+        payload["upsert_key"] = "id"
 
         # time to upsert the data
         self.client.force_login(self.admin)

geonode/upload/tests/fixture/upsert/original.json

@@ -12,6 +12,7 @@
       },
       "properties": {
         "ogc_fid": 999,
+        "id": 999,
         "st_area_shape": 192656942.39615256,
         "st_length_shape": 117499.50756699985,
         "ra_cira": 9,

geonode/upload/tests/fixture/upsert/upsert.json

@@ -1,66 +1,78 @@
 {
-    "type": "FeatureCollection",
-    "features": [
+  "type": "FeatureCollection",
+  "features":
+  [
       {
-        "type": "Feature",
-        "geometry": {
-          "type": "Point",
-          "coordinates": [
-            -47.20138110814478,
-            -13.8281037379215
-          ]
-        },
-        "properties": {
-          "ogc_fid": 999,
-          "st_area_shape": 192656942.39615256,
-          "st_length_shape": 117499.50756699985,
-          "ra_cira": 9,
-          "ra_nome": "THIS IS A CREATE",
-          "ra_codigo": "THIS IS A CREATE",
-          "ra_path": "http://localhost:8080",
-          "ra_areakm2": 192.65694240000002
-        }
+          "type": "Feature",
+          "geometry":
+          {
+              "type": "Point",
+              "coordinates":
+              [
+                  -47.20138110814478,
+                  -13.8281037379215
+              ]
+          },
+          "properties":
+          {
+              "ogc_fid": 999,
+              "id": 999,
+              "st_area_shape": 192656942.39615256,
+              "st_length_shape": 117499.50756699985,
+              "ra_cira": 9,
+              "ra_nome": "THIS IS A CREATE",
+              "ra_codigo": "THIS IS A CREATE",
+              "ra_path": "http://localhost:8080",
+              "ra_areakm2": 192.65694240000002
+          }
       },
       {
-        "type": "Feature",
-        "geometry": {
-          "type": "Point",
-          "coordinates": [
-            -47.20138110814478,
-            -13.8281037379215
-          ]
-        },
-        "properties": {
-          "ogc_fid": 123,
-          "st_area_shape": 192656942.39615256,
-          "st_length_shape": 117499.50756699985,
-          "ra_cira": 9,
-          "ra_nome": "THIS IS A CREATE",
-          "ra_codigo": "THIS IS A CREATE",
-          "ra_path": "http://localhost:8080",
-          "ra_areakm2": 192.65694240000002
-        }
+          "type": "Feature",
+          "geometry":
+          {
+              "type": "Point",
+              "coordinates":
+              [
+                  -47.20138110814478,
+                  -13.8281037379215
+              ]
+          },
+          "properties":
+          {
+              "ogc_fid": 123,
+              "id": 123,
+              "st_area_shape": 192656942.39615256,
+              "st_length_shape": 117499.50756699985,
+              "ra_cira": 9,
+              "ra_nome": "THIS IS A CREATE",
+              "ra_codigo": "THIS IS A CREATE",
+              "ra_path": "http://localhost:8080",
+              "ra_areakm2": 192.65694240000002
+          }
       },
       {
-        "type": "Feature",
-        "geometry": {
-          "type": "Point",
-          "coordinates": [
-            -47.20138110814478,
-            -13.8281037379215
-          ]
-        },
-        "properties": {
-          "ogc_fid": 1,
-          "st_area_shape": 192656942.39615256,
-          "st_length_shape": 117499.50756699985,
-          "ra_cira": 9,
-          "ra_nome": "THIS IS A CREATE",
-          "ra_codigo": "THIS IS A CREATE",
-          "ra_path": "http://localhost:8080",
-          "ra_areakm2": 192.65694240000002
-        }
+          "type": "Feature",
+          "geometry":
+          {
+              "type": "Point",
+              "coordinates":
+              [
+                  -47.20138110814478,
+                  -13.8281037379215
+              ]
+          },
+          "properties":
+          {
+              "ogc_fid": 1,
+              "id": 1,
+              "st_area_shape": 192656942.39615256,
+              "st_length_shape": 117499.50756699985,
+              "ra_cira": 9,
+              "ra_nome": "THIS IS A CREATE",
+              "ra_codigo": "THIS IS A CREATE",
+              "ra_path": "http://localhost:8080",
+              "ra_areakm2": 192.65694240000002
+          }
       }
-    ]
-  }
-  
+  ]
+}