Merge branch 'dev' into dev

Author: oceancolorcoder

.github/actions/SetupEnvironment/action.yml

@@ -29,7 +29,7 @@ runs:
       shell: bash -el {0}
       run: |
         mamba env update -n hypercp -f environment.yml
-        mamba install --channel=conda-forge pyinstaller==6.6
+        mamba install --channel=conda-forge pyinstaller
 
     - name: 📸 Capture Environment
       if: runner.os != 'Windows'
@@ -43,7 +43,7 @@ runs:
       shell: pwsh
       run: |
         mamba env update -n hypercp -f environment.yml
-        mamba install --channel=conda-forge pyinstaller==6.6
+        mamba install --channel=conda-forge pyinstaller
 
     - name: 📸 Capture Environment [Windows]
       if: runner.os == 'Windows'

Main.py

@@ -614,10 +614,16 @@ def __init__(
         super().__init__()
 
         # Confirm that core data files are in place. Download if necessary.
-        fpfZhang = os.path.join(CODE_HOME, "Data", "Zhang_rho_db.mat")
+        fpfZhang = os.path.join(CODE_HOME, "Data", "Zhang_rho_db_expanded.mat")
         if not os.path.exists(fpfZhang):
             Utilities.downloadZhangDB(fpfZhang, force=True)
 
+        # Confirm that core data files are in place. Download if necessary.
+        # fpfZhangLUT = os.path.join(CODE_HOME, "Data", "Zhang_rho_LUT.nc")
+        fpfZhangLUT = os.path.join(CODE_HOME, "Data", "Z17_LUT_v2.nc")
+        if not os.path.exists(fpfZhangLUT):
+            Utilities.downloadZhangLUT(fpfZhangLUT, force=True)
+
         # Create a default main config to be filled with cmd argument
         # to avoid reading the one generated with the GUI
         MainConfig.createDefaultConfig("cmdline_main.config", VERSION)

Source/ProcessInstrumentUncertainties.py

@@ -336,13 +336,16 @@ def ClassBased(self, node: HDFRoot, uncGrp: HDFGroup, stats: dict[str, np.array]
         with warnings.catch_warnings():
             warnings.filterwarnings("ignore", message="invalid value encountered in divide")
             # convert to relative in order to avoid a complex unit conversion process in ProcessL2.
+
             ES_unc = es_unc / np.abs(es)
             LI_unc = li_unc / np.abs(li)
             LT_unc = lt_unc / np.abs(lt)
 
+
         # interpolation step - bringing uncertainties to common wavebands from radiometric calibration wavebands.
         data_wvl = np.asarray(list(stats['ES']['std_Signal_Interpolated'].keys()),
                               dtype=float)
+    
         es_Unc = self.interp_common_wvls(ES_unc,
                                          np.array(uncGrp.getDataset(rad_cal_str).columns[cal_col_str],
                                                      dtype=float)[ind_rad_wvl],
@@ -361,7 +364,7 @@ def ClassBased(self, node: HDFRoot, uncGrp: HDFGroup, stats: dict[str, np.array]
                                          data_wvl,
                                          return_as_dict=True
                                          )
-
+        
         # return uncertainties to ProcessL2 as dictionary - will update xUnc dict with new uncs propagated to L1B
         return dict(
             esUnc=es_Unc,
@@ -2269,4 +2272,4 @@ def update_cal_rad(PANEL, S12_sl_corr, LAMP, int_time_t0, t1):
         ind_nocal = ind_nan | ind_zero
         updated_radcal_gain[
             ind_nocal == True] = 1  # set 1 instead of 0 to perform calibration (otherwise division per 0)
-        return updated_radcal_gain
+        return updated_radcal_gain

Source/ProcessL1aqc.py

@@ -502,11 +502,12 @@ def processL1aqc(node, calibrationMap, ancillaryData=None):
             start = -1
             stop = None
             index = None
+            # Threshold for an Es datetime distance from a Suntracker datetime:
+            tThreshold = datetime.timedelta(seconds=30)
+
             for index, esTimeI in enumerate(esDateTime):
-                # Threshold for an Es datetime distance from a Suntracker datetime:
-                tThreshold = datetime.timedelta(seconds=30)
 
-                tDiff = [x - esTimeI for x in sunTrackerDateTime]
+                tDiff = [abs(x - esTimeI) for x in sunTrackerDateTime]
                 if min(tDiff) > tThreshold:
                     i += 1
                     if start == -1:
@@ -516,7 +517,7 @@ def processL1aqc(node, calibrationMap, ancillaryData=None):
                     if start != -1:
                         startstop = [esDateTime[start],esDateTime[stop]]
                         msg = f'   Flag data from {startstop[0]} to {startstop[1]}'
-                        # print(msg)
+                        print(msg)
                         Utilities.writeLogFile(msg)
                         badTimes.append(startstop)
                         start = -1

Source/ProcessL2.py

@@ -1311,11 +1311,11 @@ def ensemblesReflectance(node, sasGroup, refGroup, ancGroup, uncGroup,
             # no need to retain SZA
             # Copy datasets to dictionary
             diffuseData.datasetToColumns()
-            diffuseColumns = diffuseData.columns
+            # diffuseColumns = diffuseData.columns
             directData.datasetToColumns()
-            directColumns = directData.columns
+            # directColumns = directData.columns
             sixSesData.datasetToColumns()
-            sixSesColumns = sixSesData.columns
+            # sixSesColumns = sixSesData.columns
 
             # diffuseSlice = ProcessL2.columnToSlice(diffuseColumns,start, end)
             # directSlice = ProcessL2.columnToSlice(directColumns,start, end)
@@ -2328,7 +2328,6 @@ def stationsEnsemblesReflectance(node, root, station=None):
                 EndOfFileFlag = True # In case the whole file is shorter than the selected interval
 
             for i in range(0, esLength):
-                # time = Utilities.timeTag2ToSec(tt2[i])
                 timei = timeStamp[i]
                 if (timei > endTime) or EndOfFileFlag: # end of increment reached
                     if EndOfFileFlag:

Source/Utilities.py

@@ -227,7 +227,7 @@ def dop(year):
             fp = 'Data/hybrid_reference_spectrum_p1nm_resolution_c2020-09-21_with_unc.nc'
             # fp = 'Data/Thuillier_F0.sb'
             # print("SB_support.readSB: " + fp)
-            print("Reading : " + fp)
+            # print("Reading : " + fp)
             if not HDFRoot.readHDF5(fp):
                 msg = "Unable to read TSIS-1 netcdf file."
                 print(msg)

Tests/test_sample_data.py

@@ -46,7 +46,7 @@ def setUp(self):
         self.cfg_filename = os.path.join(root, "Config", "sample_SEABIRD_SOLARTRACKER.cfg")
         self.files = sorted(glob.glob(os.path.join(self.path_to_data, 'RAW', f'*.RAW')))
 
-    def test_pysas(self):
+    def test_seabird_solar_tracker(self):
         from Main import Command
         os.chdir(root)  # Need to switch to root as path in Config files are relative
         for file in self.files:

environment.yml

@@ -8,7 +8,7 @@ dependencies:
   - pyqt=5.15
   - requests=2.28
   - tqdm=4.65
-  - numpy=1.23
+  - numpy=1
   - h5py=3.7
   - pytz=2022.7
   - matplotlib
@@ -20,10 +20,11 @@ dependencies:
   - netcdf4=1.6
   - j6s
   - pytables=3.7
-  - pyspectral=0.12
+  - pyspectral=0.12.5
   - pyqtgraph
   - ocdb-client
   - pip
   - pip:
+    - comet_maths<=1.0.3
     - punpy
     - fpdf2