[Fixes #12657] Align supported_file_extension_config with the new client configuration (#12673)

* [Fixes #12657] Rename ACTIONS into TASKS

* [Fixes #12657] Refactor supported types

* [Fixes #12657] Refactor supported types

* [Fixes #12657] Refactor supported type, fix data retriever and refactor handlers configuration

* [Fixes #12657] Refactor supported type, fix data retriever and refactor handlers configuration

* [Fixes #12657] Refactor supported type, fix data retriever and refactor handlers configuration

* [Fixes #12657] Refactor supported type, fix data retriever and refactor handlers configuration

* [Fixes #12657] Refactor supported type, fix data retriever and refactor handlers configuration

* [Fixes #12657] Refactor supported type, fix data retriever and refactor handlers configuration

Author: mattiagiupponi

geonode/geoserver/management/commands/importlayers.py

@@ -149,6 +149,7 @@ def execute(self):
                             params[name] = os.path.basename(value.name)
 
                     params["non_interactive"] = 'true'
+                    params["action"] = 'upload'
                     response = client.post(
                         urljoin(self.host, "/api/v2/uploads/upload/"),
                         auth=HTTPBasicAuth(self.username, self.password),

geonode/resource/api/tests.py

@@ -157,9 +157,7 @@ def test_endpoint_should_raise_error_if_pk_is_not_passed(self):
 
     def test_endpoint_should_return_the_source(self):
         # creating dummy execution request
-        obj = ExecutionRequest.objects.create(
-            user=self.superuser, func_name="import_new_resource", action="import", source="upload_workflow"
-        )
+        obj = ExecutionRequest.objects.create(user=self.superuser, func_name="import_new_resource", action="upload")
         self.client.force_login(self.superuser)
 
         _url = f"{reverse('executionrequest-list')}/{obj.exec_id}"

geonode/resource/enumerator.py

@@ -22,6 +22,7 @@
 
 class ExecutionRequestAction(enum.Enum):
     IMPORT = _("import")
+    UPLOAD = _("upload")
     CREATE = _("create")
     COPY = _("copy")
     DELETE = _("delete")

geonode/resource/migrations/0009_remove_executionrequest_source_and_more.py

@@ -0,0 +1,37 @@
+# Generated by Django 4.2.9 on 2024-10-18 10:41
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("resource", "0008_executionrequest_source"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="executionrequest",
+            name="source",
+        ),
+        migrations.AlterField(
+            model_name="executionrequest",
+            name="action",
+            field=models.CharField(
+                choices=[
+                    ("import", "import"),
+                    ("upload", "upload"),
+                    ("create", "create"),
+                    ("copy", "copy"),
+                    ("delete", "delete"),
+                    ("permissions", "permissions"),
+                    ("update", "update"),
+                    ("ingest", "ingest"),
+                    ("unknown", "unknown"),
+                ],
+                default="unknown",
+                max_length=50,
+                null=True,
+            ),
+        ),
+    ]

geonode/resource/models.py

@@ -60,5 +60,3 @@ class ExecutionRequest(models.Model):
     action = models.CharField(
         max_length=50, choices=ACTION_CHOICES, default=ExecutionRequestAction.UNKNOWN.value, null=True
     )
-
-    source = models.CharField(max_length=250, null=True, default=None)

geonode/security/tests.py

@@ -752,9 +752,7 @@ def test_dataset_permissions(self):
         bobby = get_user_model().objects.get(username="bobby")
 
         self.client.force_login(get_user_model().objects.get(username="admin"))
-        payload = {
-            "base_file": open(f"{project_dir}/tests/fixture/valid.geojson", "rb"),
-        }
+        payload = {"base_file": open(f"{project_dir}/tests/fixture/valid.geojson", "rb"), "action": "upload"}
         response = self.client.post(reverse("importer_upload"), data=payload)
         layer = ResourceHandlerInfo.objects.filter(execution_request=response.json()["execution_id"]).first().resource
         if layer is None:

geonode/settings.py

@@ -2240,56 +2240,6 @@ def get_geonode_catalogue_service():
     "document_upload",
 )
 
-SUPPORTED_DATASET_FILE_TYPES = [
-    {
-        "id": "shp",
-        "label": "ESRI Shapefile",
-        "format": "vector",
-        "ext": ["shp"],
-        "requires": ["shp", "prj", "dbf", "shx"],
-        "optional": ["xml", "sld"],
-    },
-    {
-        "id": "tiff",
-        "label": "GeoTIFF",
-        "format": "raster",
-        "ext": ["tiff", "tif", "geotiff", "geotif"],
-        "mimeType": ["image/tiff"],
-        "optional": ["xml", "sld"],
-    },
-    {
-        "id": "csv",
-        "label": "Comma Separated Value (CSV)",
-        "format": "vector",
-        "ext": ["csv"],
-        "mimeType": ["text/csv"],
-        "optional": ["xml", "sld"],
-    },
-    {
-        "id": "zip",
-        "label": "Zip Archive",
-        "format": "archive",
-        "ext": ["zip"],
-        "mimeType": ["application/zip"],
-        "optional": ["xml", "sld"],
-    },
-    {
-        "id": "xml",
-        "label": "XML Metadata File",
-        "format": "metadata",
-        "ext": ["xml"],
-        "mimeType": ["application/json"],
-        "needsFiles": ["shp", "prj", "dbf", "shx", "csv", "tiff", "zip", "sld"],
-    },
-    {
-        "id": "sld",
-        "label": "Styled Layer Descriptor (SLD)",
-        "format": "metadata",
-        "ext": ["sld"],
-        "mimeType": ["application/json"],
-        "needsFiles": ["shp", "prj", "dbf", "shx", "csv", "tiff", "zip", "xml"],
-    },
-]
 INSTALLED_APPS += (
     "dynamic_models",
     # "importer",

geonode/storage/tests.py

@@ -573,23 +573,6 @@ def test_zip_file_should_correctly_index_file_extensions(self):
         # extensions found more than once get indexed
         self.assertIsNotNone(_files.get("csv_file_1"))
 
-    @override_settings(
-        SUPPORTED_DATASET_FILE_TYPES=[
-            {"id": "kmz", "label": "kmz", "format": "vector", "ext": ["kmz"]},
-            {"id": "kml", "label": "kml", "format": "vector", "ext": ["kml"]},
-        ]
-    )
-    def test_zip_file_should_correctly_recognize_main_extension_with_kmz(self):
-        # reinitiate the storage manager with the zip file
-        storage_manager = self.sut(
-            remote_files={"base_file": os.path.join(f"{self.project_root}", "tests/data/Italy.kmz")}
-        )
-        storage_manager.clone_remote_files()
-
-        self.assertIsNotNone(storage_manager.data_retriever.temporary_folder)
-        _files = storage_manager.get_retrieved_paths()
-        self.assertTrue("doc.kml" in _files.get("base_file"), msg=f"files available: {_files}")
-
     def test_zip_file_should_correctly_recognize_main_extension_with_shp(self):
         # zipping files
         storage_manager = self.sut(remote_files=self.local_files_paths)

geonode/tests/test_utils.py

@@ -18,7 +18,6 @@
 #########################################################################
 import copy
 from unittest import TestCase
-from django.test import override_settings
 
 from unittest.mock import patch
 from datetime import datetime, timedelta
@@ -32,8 +31,7 @@
 from geonode.geoserver.helpers import set_attributes
 from geonode.tests.base import GeoNodeBaseTestSupport
 from geonode.br.management.commands.utils.utils import ignore_time
-from geonode.utils import copy_tree, get_supported_datasets_file_types, bbox_to_wkt
-from geonode import settings
+from geonode.utils import copy_tree, bbox_to_wkt
 
 
 class TestCopyTree(GeoNodeBaseTestSupport):
@@ -205,39 +203,6 @@ def setUp(self):
             },
         ]
 
-    @override_settings(
-        ADDITIONAL_DATASET_FILE_TYPES=[
-            {"id": "dummy_type", "label": "Dummy Type", "format": "dummy", "ext": ["dummy"]},
-        ]
-    )
-    def test_should_append_additional_type_if_config_is_provided(self):
-        prev_count = len(settings.SUPPORTED_DATASET_FILE_TYPES)
-        supported_types = get_supported_datasets_file_types()
-        supported_keys = [t.get("id") for t in supported_types]
-        self.assertIn("dummy_type", supported_keys)
-        self.assertEqual(len(supported_keys), prev_count + 1)
-
-    @override_settings(
-        ADDITIONAL_DATASET_FILE_TYPES=[
-            {
-                "id": "shp",
-                "label": "Replaced type",
-                "format": "vector",
-                "ext": ["shp"],
-                "requires": ["shp", "prj", "dbf", "shx"],
-                "optional": ["xml", "sld"],
-            },
-        ]
-    )
-    def test_should_replace_the_type_id_if_already_exists(self):
-        prev_count = len(settings.SUPPORTED_DATASET_FILE_TYPES)
-        supported_types = get_supported_datasets_file_types()
-        supported_keys = [t.get("id") for t in supported_types]
-        self.assertIn("shp", supported_keys)
-        self.assertEqual(len(supported_keys), prev_count)
-        shp_type = [t for t in supported_types if t["id"] == "shp"][0]
-        self.assertEqual(shp_type["label"], "Replaced type")
-
 
 class TestRegionsCrossingDateLine(TestCase):
     def setUp(self):

geonode/upload/api/serializer.py

@@ -34,15 +34,15 @@ class Meta:
             "sld_file",
             "store_spatial_files",
             "skip_existing_layers",
-            "source",
+            "action",
         )
 
     base_file = serializers.FileField()
     xml_file = serializers.FileField(required=False)
     sld_file = serializers.FileField(required=False)
     store_spatial_files = serializers.BooleanField(required=False, default=True)
     skip_existing_layers = serializers.BooleanField(required=False, default=False)
-    source = serializers.CharField(required=False, default="upload")
+    action = serializers.CharField(required=True)
 
 
 class OverwriteImporterSerializer(ImporterSerializer):