Handle embargo_date in occurrence uploader (#5094)

Author: dimasciput

bims/download/collection_record.py

@@ -7,7 +7,7 @@
 import shutil
 
 from bims.models.download_request import DownloadRequest
-from bims.scripts.collection_csv_keys import PARK_OR_MPA_NAME
+from bims.scripts.collection_csv_keys import PARK_OR_MPA_NAME, END_EMBARGO_DATE
 
 logger = logging.getLogger(__name__)
 
@@ -37,7 +37,8 @@ def queryset_iterator(qs, batch_size=500, gc_collect=True):
     'sub_species': 'SubSpecies',
     'cites_listing': 'CITES listing',
     'park_or_mpa_name': PARK_OR_MPA_NAME,
-    'authors': 'Author(s)'
+    'authors': 'Author(s)',
+    'end_embargo_date': END_EMBARGO_DATE,
 }
 
 

bims/scripts/collection_csv_keys.py

@@ -100,3 +100,4 @@
 CERTAINTY_OF_IDENTIFICATION = 'Certainty'
 DATE_ACCURACY = 'Date Accuracy'
 DATA_TYPE = 'Data type'
+END_EMBARGO_DATE = 'End Embargo Date'

bims/scripts/occurrences_upload.py

@@ -7,6 +7,7 @@
 
 from bims.scripts.collection_csv_keys import *  # noqa
 from datetime import datetime
+from dateutil.parser import parse as parse_datetime
 
 from django.contrib.gis.geos import Point
 from django.db.models import Q
@@ -642,6 +643,20 @@ def process_data(self, row):
             if not sampling_date:
                 return
 
+            # -- End embargo date
+            end_embargo_date = None
+            end_embargo_date_str = DataCSVUpload.row_value(row, END_EMBARGO_DATE)
+            if end_embargo_date_str:
+                try:
+                    end_embargo_date = parse_datetime(
+                        timestr=end_embargo_date_str, dayfirst=True)
+                except Exception:
+                    self.handle_error(
+                        row=row,
+                        message=f"Incorrect end embargo date format: {end_embargo_date_str}"
+                    )
+                    return
+
             # -- Processing Taxonomy
             taxonomy = self.taxonomy(row)
             if not taxonomy:
@@ -685,6 +700,12 @@ def process_data(self, row):
                 collector=collectors[0],
             )
 
+            # -- Apply end embargo date to the survey
+            if end_embargo_date and self.survey:
+                Survey.objects.filter(id=self.survey.id).update(
+                    end_embargo_date=end_embargo_date
+                )
+
             # -- Optional data - Present
             if PRESENT in row:
                 optional_data["present"] = bool(DataCSVUpload.row_value(row, PRESENT))
@@ -794,6 +815,10 @@ def process_data(self, row):
                 wetland_indicator_status = None
             optional_data["wetland_indicator_status"] = wetland_indicator_status
 
+            # -- End embargo date on the occurrence record
+            if end_embargo_date:
+                optional_data["end_embargo_date"] = end_embargo_date
+
             # -- Processing chemical records
             self.chemical_records(row, location_site, sampling_date)
 

bims/serializers/bio_collection_serializer.py

@@ -161,6 +161,7 @@ class BioCollectionOneRowSerializer(
     cites_listing = serializers.SerializerMethodField()
     data_type = serializers.SerializerMethodField()
     dataset = serializers.SerializerMethodField()
+    end_embargo_date = serializers.SerializerMethodField()
 
     @staticmethod
     def _has_value(v) -> bool:
@@ -703,6 +704,11 @@ def get_data_type(self, obj: BiologicalCollectionRecord):
             return obj.data_type.capitalize()
         return 'Public'
 
+    def get_end_embargo_date(self, obj: BiologicalCollectionRecord):
+        if obj.end_embargo_date:
+            return obj.end_embargo_date.isoformat()
+        return ''
+
     class Meta:
         model = BiologicalCollectionRecord
         fields = [
@@ -769,7 +775,8 @@ class Meta:
             'dataset',
             'dataset_key',
             'cites_listing',
-            'data_type'
+            'data_type',
+            'end_embargo_date',
         ]
 
     def _get_geocontext_parks_group(self):