Merge pull request #346 from ARamsay17/dev

Small PR

Author: oceancolorcoder

Source/ProcessInstrumentUncertainties.py

@@ -336,14 +336,10 @@ def ClassBased(self, node: HDFRoot, uncGrp: HDFGroup, stats: dict[str, np.array]
         with warnings.catch_warnings():
             warnings.filterwarnings("ignore", message="invalid value encountered in divide")
             # convert to relative in order to avoid a complex unit conversion process in ProcessL2.
-            ES_unc = es_unc / es
-            LI_unc = li_unc / li
-            LT_unc = lt_unc / lt
-            if any(ES_unc < 0) or any(LI_unc < 0) or any(LT_unc < 0):
-                print('############ WARNING ###########')
-                msg = 'Negatives encountered in uncertainties likely deriving from (ir)radiance normalization and bad QC.'
-                print(msg)
-                Utilities.writeLogFile(msg)
+
+            ES_unc = es_unc / np.abs(es)
+            LI_unc = li_unc / np.abs(li)
+            LT_unc = lt_unc / np.abs(lt)
 
 
         # interpolation step - bringing uncertainties to common wavebands from radiometric calibration wavebands.

Source/ProcessL1aTriOS.py

@@ -46,21 +46,49 @@ def processL1a(fp, outFilePath):
 
 
                 ## Test filename for station/cast
-                match1 = re.search(r'\d{8}_\d{6}', file.split('/')[-1])
-                match2 = re.search(r'\d{4}S', file.split('/')[-1])
-                match3 = re.search(r'\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}', file.split('/')[-1])
-                if match1 is not None:
-                    a_name = match1.group()
-                elif match2 is not None:
-                    a_name = match2.group()
-                elif match3 is not None:
-                    a_name = match3.group()
-                else:
+
+                def parse_filename(data):
+                    dates = []
+                    for pattern in [
+                        r'\d{8}.\d{6}', 
+                        r'\d{4}.\d{2}.\d{2}.\d{2}.\d{2}.\d{2}',
+                        r'\d{8}.\d{2}.\d{2}.\d{2}',
+                        r'\d{4}.\d{2}.\d{2}.\d{6}',
+                        r'\d{4}S', 
+                    ]:
+                        match = re.search(pattern, data)
+                        if match is not None:
+                            dates.append(match.group(0))
+                    
+                    if len(dates) == 0:
+                        raise IndexError
+                    
+                    return dates[0]
+                
+                try:
+                    a_name = parse_filename(file.split('/')[-1])
+                except IndexError:
                     print("  ERROR: no identifier recognized in TRIOS L0 file name" )
                     print("  L0 filename should have a cast to identify triplet instrument")
                     print("  ending in 4 digits before S.mlb ")
                     return None,None
 
+                # match1 = re.search(r'\d{8}_\d{6}', file.split('/')[-1])
+                # match2 = re.search(r'\d{4}S', file.split('/')[-1])
+                # match3 = re.search(r'\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}', file.split('/')[-1])
+                # # string except for serial number will be the same for a triplet
+                # if match1 is not None:
+                #     a_name = match1.group()
+                # elif match2 is not None:
+                #     a_name = match2.group()
+                # elif match3 is not None:
+                #     a_name = match3.group()
+                # else:
+                #     print("  ERROR: no identifier recognized in TRIOS L0 file name" )
+                #     print("  L0 filename should have a cast to identify triplet instrument")
+                #     print("  ending in 4 digits before S.mlb ")
+                #     return None,None
+
                 # acq_time.append(a_cast)
                 acq_name.append(a_name)
 
@@ -114,8 +142,8 @@ def processL1a(fp, outFilePath):
 
                 try:
                     new_name = file.split('/')[-1].split('.mlb')[0].split(f'SAM_{name}_RAW_SPECTRUM_')[1]
-                    if match2 is not None:
-                        new_name = new_name+'_'+str(start)
+                    if re.search(r'\d{4}S', file.split('/')[-1]) is not None:
+                        new_name = new_name+'_'+str(start)  # I'm not sure what match2 was supposed to find in ACRI's code - AR
                 except IndexError as err:
                     print(err)
                     msg = "possibly an error in naming of Raw files"

Source/ProcessL1b.py

@@ -113,8 +113,33 @@ def read_unc_coefficient_class(root, inpath, radcal_dir):
 
 
         # Read sensor-specific radiometric calibration
+        import re
+        
+        cal_dates = []
+        for s in ['ES', 'LI', 'LT']:
+            save_delta = None
+            if ConfigFile.settings["SensorType"].lower() == "trios":
+                s_tag = root.getGroup(f"{s}").attributes['IDDevice'][-4:]
+            else:
+                s_tag = root.getGroup(f"{s}_LIGHT").attributes['FrameTag'][-4:]
+
+            for f in glob.glob(os.path.join(radcal_dir, f'*{s_tag}_RADCAL*')):
+                cal_date = datetime.strptime(re.search(r'\d{14}', f.split('/')[-1]).group(), "%Y%m%d%H%M%S")
+                meas_date = root.getGroup("ANCILLARY_METADATA").getDataset("DATETIME").data[0]
+                t_delta = cal_date - meas_date.replace(tzinfo=None)
+                
+                if save_delta is None and t_delta.total_seconds() < 0:
+                    save_delta = abs(t_delta.total_seconds())
+                
+                if t_delta.total_seconds() < 0 and abs(t_delta.total_seconds()) <= save_delta:
+                    save_cal_date = cal_date
+                    save_delta = abs(t_delta.total_seconds())
+            
+            cal_dates.append(save_cal_date)
+
         for f in glob.glob(os.path.join(radcal_dir, r'*RADCAL*')):
-            Utilities.read_char(f, gp)
+            if datetime.strptime(re.search(r'\d{14}', f.split('/')[-1]).group(), "%Y%m%d%H%M%S") in cal_dates:
+                Utilities.read_char(f, gp)
 
         # Unc dataset renaming
         Utilities.RenameUncertainties_Class(root)