Merge pull request #9 from WHOIGit/r2r_event

add r2r_event to event log

Author: joefutrelle

api/core/migrations/0010_remove_event_unique_cruise_message_id_and_more.py

@@ -0,0 +1,41 @@
+# Generated by Django 5.2.3 on 2025-06-17 17:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0009_hplc_alter_cruise_start_time'),
+    ]
+
+    operations = [
+        migrations.RemoveConstraint(
+            model_name='event',
+            name='unique_cruise_message_id',
+        ),
+        migrations.AddField(
+            model_name='event',
+            name='r2r_event',
+            field=models.CharField(blank=True, max_length=30, null=True),
+        ),
+        migrations.AddField(
+            model_name='historicalevent',
+            name='r2r_event',
+            field=models.CharField(blank=True, max_length=30, null=True),
+        ),
+        migrations.AlterField(
+            model_name='event',
+            name='message_id',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='historicalevent',
+            name='message_id',
+            field=models.IntegerField(null=True),
+        ),
+        migrations.AddConstraint(
+            model_name='event',
+            constraint=models.UniqueConstraint(fields=('cruise', 'r2r_event'), name='unique_cruise_r2r_event'),
+        ),
+    ]

api/core/models.py

@@ -204,7 +204,8 @@ def __str__(self):
 
 class Event(models.Model):
     cruise = models.ForeignKey(Cruise, on_delete=models.CASCADE, related_name='events')
-    message_id = models.IntegerField()
+    r2r_event = models.CharField(max_length=30, null=True, blank=True)
+    message_id = models.IntegerField(null=True)
     instrument = models.CharField(max_length=100)
     action = models.CharField(max_length=32)
     station = models.CharField(max_length=100)
@@ -216,7 +217,7 @@ class Event(models.Model):
 
     class Meta:
         constraints = [
-            UniqueConstraint(fields=['cruise', 'message_id'], name='unique_cruise_message_id')
+            UniqueConstraint(fields=['cruise', 'r2r_event'], name='unique_cruise_r2r_event')
         ]
 
     def __str__(self):

api/ctd/management/commands/importniskin.py

@@ -165,7 +165,10 @@ def handle(self, *args, **options):
                 cruise = Cruise.objects.get(name__iexact=cruise_name)
 
                 directory = f'/vast/raw/{cruise_name}/ctd/'
-                btl_files = sorted(glob.glob(os.path.join(directory, '*.btl')))
+                btl_files = sorted(
+                    f for f in glob.glob(os.path.join(directory, '*.btl'))
+                    if not f.endswith('_original.btl')
+                )
                 if cruise_name.lower() == "en627":
                     added_dir = os.path.join(directory, "cast_1_files_used_for_corrected_cast_2")
                     btl_files += sorted(glob.glob(os.path.join(added_dir, '*.btl')))

api/events/management/commands/importevent.py

@@ -16,30 +16,39 @@
 from datetime import datetime
 
 DATETIME = 'dateTime8601'
-MESSAGE_ID = 'Message ID'  
+MESSAGE_ID = 'Message ID'
+R2R_EVENT = 'R2R_Event'
 FILE_SUFFIX = '_elog.csv'
 
 INSTRUMENT_MAPPING = {
     "Attune": "Attune Flow Cytometer",
+    "Attune Flow Cytomoeter": "Attune Flow Cytometer",
     "Bongo": "Bongo Net",
     "Cytomeoter": "Cytometer",
     "EK80": "EK80 broadband",
     "IFCB_continuous": "IFCB continuous",
     "IFCB Continuous": "IFCB continuous",
     "IFCB 109": "IFCB continuous",
+    "IFCB": "IFCB continuous",
     "Incubation": "Incubation Grazing",
     "Incubation O2": "Incubation Respiration O2",
+    "Issacs Kidd Midwater Trawl": "Isaacs-Kidd Midwater Trawl",
+    "Midwater Trawl": "RMT10 Midwater Trawl (Tucker-style)",
+    "RMT8 Midwater Trawl": "RMT10 Midwater Trawl (Tucker-style)",
     "RingNet": "Ring Net",
     "RingNetIFCB Continuous": "IFCB continuous",
     "SSW": "Underway Science seawater diaphragm pump",
+    "Stingray": "Sting Ray",
     "SUNA V2": "SUNAV2",
     "Thermosalinograph SBE45": "Thermosalinographs on underway impeller ",
     "Transmissometer 10": "Transmissometer 10cm",
     "trans10": "Transmissometer 10cm",    
     "trans25": "Transmissometer 25cm",        
     "Underway diaphram pump": "Underway Science seawater diaphragm pump",
     "Underway Impeller": "Underway Science seawater impeller",
+    "Underway Science seawater impeller pump": "Underway Science seawater impeller",
     "Underway Science Seawater Diaphragm Pump": "Underway Science seawater diaphragm pump",
+    "Valeport Modus SVS": "Valeport SVS"
 }
 
 
@@ -72,23 +81,6 @@ def store_csv_file(self, cruise_name, csv_data):
                 print(e, flush=True)
                 raise
 
-    def apply_corrections(self, path):
-        corr = pd.read_excel(path)
-        corr[DATETIME] = pd.to_datetime(corr[DATETIME], utc=True)
-        corr.pop('Instrument')
-        corr.pop('Action')
-        return corr
-
-    def apply_additions(self, addns_path):
-        addns = pd.read_excel(addns_path)
-        addns[DATETIME] = pd.to_datetime(addns[DATETIME], utc=True, format="ISO8601")
-        # add placeholder columns
-        addns.insert(4, MESSAGE_ID, np.nan)
-        addns.insert(4, 'Longitude', np.nan)
-        addns.insert(4, 'Latitude', np.nan)
-        addns.insert(4, 'Cast', np.nan)
-        return addns
-
     def handle(self, *args, **options):
         cruise_name = options['cruise_name']
 
@@ -105,14 +97,13 @@ def handle(self, *args, **options):
                directory = f'/vast/corrected/{cruise_name}/elog/'
                file_pattern = os.path.join(directory, '*_elog.csv')
                matching_file = glob.glob(file_pattern)
-               if matching_file:
+               if matching_file and cruise_name.lower() not in ['en608', 'en617', 'en627']:  # serve the original elogs for these cruises
                    file_path = matching_file[0]
                    df = pd.read_csv(file_path, parse_dates=[DATETIME], dtype={'Station': str, 'Cast': str})
                    try:
                        df[DATETIME] = pd.to_datetime(df[DATETIME]).dt.tz_convert('UTC')
                    except:
                        df[DATETIME] = pd.to_datetime(df[DATETIME]).dt.tz_localize('UTC')   # en617
-                   df[MESSAGE_ID] = range(1, len(df) + 1)   # assign message ids
                else:
                    directory = f'/vast/raw/{cruise_name}/elog/'
                    file_pattern = os.path.join(directory, 'R2R_ELOG*FINAL*')  # do not read corrections or additions files in elog dir
@@ -136,10 +127,13 @@ def handle(self, *args, **options):
                        raw_instrument = row['Instrument']
                        instrument = INSTRUMENT_MAPPING.get(raw_instrument, raw_instrument)
 
+                       message_id = int(row[MESSAGE_ID]) if pd.notna(row[MESSAGE_ID]) else None  # Nan is a float
+                       
                        event, created = Event.objects.update_or_create(
                                cruise=cruise,
-                               message_id=row[MESSAGE_ID],
+                               r2r_event=row[R2R_EVENT],
                                defaults={
+                                   "message_id": message_id,
                                    "instrument":instrument,
                                    "action":row['Action'],
                                    "station":row['Station'],
@@ -151,6 +145,7 @@ def handle(self, *args, **options):
                                )
 
                        csv_data.append({
+                           R2R_EVENT: event.r2r_event,
                            MESSAGE_ID: event.message_id,
                            DATETIME: event.datetime,
                            "Instrument": event.instrument,

api/nut/management/commands/importnut.py

@@ -154,7 +154,10 @@ def read_nut_data(self, cruise, merged):
 
         # set date, lat, lon, depth to NaN when there is no bottle file for the cast
         btl_dir = f'/vast/raw/{cruise}/ctd/'
-        for file in sorted(glob.glob(os.path.join(btl_dir, '*.asc'))):
+        for file in sorted(
+            f for f in glob.glob(os.path.join(btl_dir, '*.asc'))
+            if not f.endswith('_original.asc')
+        ):
             if cruise == 'en627':
                 file = file.replace("_u", "")
             btl_file = file[:-3] + 'btl'