modifications to read SWIR spectra

Author: clemgoyens

docs/atbd/Hypernets-NPL-Algorithm_Theoretical_Basis_Document-v0.0.docx

@@ -231,10 +231,20 @@ def preprocess_l0(self, datasetl0, datasetl1a):
 
         flagval = 2 ** (self.context.get_config_value("outliers"))
 
+        print(np.where(mask > 0))
+
+        break
+        datasetl0["quality_flag"].values = [
+            flagval + datasetl0["quality_flag"].values[i] if mask[i] == 1 else
+            datasetl0["quality_flag"].values[i] for i in range(len(mask))]
+
+
+
         datasetl0["quality_flag"].values = [
             flagval + datasetl0["quality_flag"].values[i] if mask[i] == 1 else
             datasetl0["quality_flag"].values[i] for i in range(len(mask))]
 
+
         datasetl1a["quality_flag"].values = [
             flagval + datasetl1a["quality_flag"].values[i] if mask[i] == 1 else
             datasetl1a["quality_flag"].values[i] for i in range(len(mask))]

docs/file_formats/Hypernets-Product_Data_Format_Specification-v0.6.docx

@@ -220,6 +220,138 @@ def get_default_fill_value(dtype):
         elif dtype == np.float64:
             return np.float64(9.969209968386869E36)
 
+    @staticmethod
+    def _get_flag_encoding(da):
+        """
+        Returns flag encoding for flag type data array
+        :type da: xarray.DataArray
+        :param da: data array
+        :return: flag meanings
+        :rtype: list
+        :return: flag masks
+        :rtype: list
+        """
+
+        try:
+            flag_meanings = da.attrs["flag_meanings"].split()
+            flag_masks = [int(fm) for fm in da.attrs["flag_masks"].split(",")]
+        except KeyError:
+            raise KeyError(da.name + " not a flag variable")
+
+        return flag_meanings, flag_masks
+
+    @staticmethod
+    def unpack_flags(da):
+        """
+        Breaks down flag data array into dataset of boolean masks for each flag
+        :type da: xarray.DataArray
+        :param da: dataset
+        :return: flag masks
+        :rtype: xarray.Dataset
+        """
+
+        flag_meanings, flag_masks = DatasetUtil._get_flag_encoding(da)
+
+        ds = Dataset()
+        for flag_meaning, flag_mask in zip(flag_meanings, flag_masks):
+            ds[flag_meaning] = DatasetUtil.create_variable(list(da.shape), bool, dim_names=list(da.dims))
+            ds[flag_meaning] = (da & flag_mask).astype(bool)
+
+        return ds
+
+    @staticmethod
+    def set_flag(da, flag_name, error_if_set=False):
+        """
+        Sets named flag for elements in data array
+        :type da: xarray.DataArray
+        :param da: dataset
+        :type flag_name: str
+        :param flag_name: name of flag to set
+        :type error_if_set: bool
+        :param error_if_set: raises error if chosen flag is already set for any element
+        """
+
+        set_flags = DatasetUtil.unpack_flags(da)[flag_name]
+
+        if np.any(set_flags == True) and error_if_set:
+            raise ValueError("Flag " + flag_name + " already set for variable " + da.name)
+
+        # Find flag mask
+        flag_meanings, flag_masks = DatasetUtil._get_flag_encoding(da)
+        flag_bit = flag_meanings.index(flag_name)
+        flag_mask = flag_masks[flag_bit]
+
+        return da | flag_mask
+
+    @staticmethod
+    def unset_flag(da, flag_name, error_if_unset=False):
+        """
+        Unsets named flag for specified index of dataset variable
+        :type da: xarray.DataArray
+        :param da: data array
+        :type flag_name: str
+        :param flag_name: name of flag to unset
+        :type error_if_unset: bool
+        :param error_if_unset: raises error if chosen flag is already set at specified index
+        """
+
+        set_flags = DatasetUtil.unpack_flags(da)[flag_name]
+
+        if np.any(set_flags == False) and error_if_unset:
+            raise ValueError("Flag " + flag_name + " already set for variable " + da.name)
+
+        # Find flag mask
+        flag_meanings, flag_masks = DatasetUtil._get_flag_encoding(da)
+        flag_bit = flag_meanings.index(flag_name)
+        flag_mask = flag_masks[flag_bit]
+
+        return da & ~flag_mask
+
+    @staticmethod
+    def get_set_flags(da):
+        """
+        Return list of set flags for single element data array
+        :type da: xarray.DataArray
+        :param da: single element data array
+        :return: set flags
+        :rtype: list
+        """
+
+        if da.shape != ():
+            raise ValueError("Must pass single element data array")
+
+        flag_meanings, flag_masks = DatasetUtil._get_flag_encoding(da)
+
+        set_flags = []
+        for flag_meaning, flag_mask in zip(flag_meanings, flag_masks):
+            if (da & flag_mask):
+                set_flags.append(flag_meaning)
+
+        return set_flags
+
+    @staticmethod
+    def check_flag_set(da, flag_name):
+        """
+        Returns if flag for single element data array
+        :type da: xarray.DataArray
+        :param da: single element data array
+        :type flag_name: str
+        :param flag_name: name of flag to set
+        :return: set flags
+        :rtype: list
+        """
+
+        if da.shape != ():
+            raise ValueError("Must pass single element data array")
+
+        set_flags = DatasetUtil.get_set_flags(da)
+
+        if flag_name in set_flags:
+            return True
+        return False
+
+
+
 
 if __name__ == "__main__":
     pass

hypernets_processor/calibration/calibrate.py

@@ -204,7 +204,7 @@
                                                           "long_name": "",
                                                           "units": "-"},
                                            "encoding": {'dtype': np.uint32, "scale_factor": 0.01, "offset": 0.0}},
-                    "digital_number": {"dim": [SCAN_DIM],
+                    "digital_number": {"dim": [WL_DIM, SCAN_DIM],
                                        "dtype": np.float32,
                                        "attributes": {"standard_name": "digital_number",
                                                       "long_name": "Digital number, raw data",

hypernets_processor/data_io/dataset_util.py

@@ -16,7 +16,7 @@
 from hypernets_processor.data_io.format.header import HEADER_DEF
 from hypernets_processor.data_io.hypernets_ds_builder import HypernetsDSBuilder
 from hypernets_processor.version import __version__
-
+from hypernets_processor.data_io.dataset_util import DatasetUtil as du
 '''___Authorship___'''
 __author__ = "Clémence Goyens"
 __created__ = "12/2/2020"
@@ -187,10 +187,10 @@ def read_series(self, seq_dir, series, lat, lon, metadata, flag, fileformat):
         f = open(FOLDER_NAME + series[1], "rb")
         # Header definition with length, description and decoding format
 
-        #header = self.read_header(f, HEADER_DEF)
-        #print(header)
+        header = self.read_header(f, HEADER_DEF)
+        print(header)
 
-        pixCount= 2048 #header['Pixel Count']
+        pixCount= header['Pixel Count']
 
         # if bool(header) == False:
         #     print("Data corrupt go to next line")
@@ -251,14 +251,14 @@ def read_series(self, seq_dir, series, lat, lon, metadata, flag, fileformat):
 
             nextLine = True
             while nextLine:
-                # # if no reader comment those lines
-                # header = self.read_header(f, HEADER_DEF)
-                # print(header)
-                # if bool(header) == False:
-                #     print("Data corrupt go to next line")
-                #     break
-                #     continue
-                pixCount= 2048#header['Pixel Count']
+                # if no header comment those lines
+                header = self.read_header(f, HEADER_DEF)
+                print(header)
+                if bool(header) == False:
+                    print("Data corrupt go to next line")
+                    break
+                    continue
+                pixCount= header['Pixel Count']
                 scan = self.read_data(f, pixCount)
                 # should include this back again when crc32 is in the headers!
                 #crc32 = self.read_footer(f, 4)
@@ -300,8 +300,8 @@ def read_series(self, seq_dir, series, lat, lon, metadata, flag, fileformat):
                 ds["solar_azimuth_angle"][scan_number] = get_azimuth(float(lat), float(lon), acquisitionTime)
 
                 ds['quality_flag'][scan_number] = flag
-                ds['integration_time'][scan_number] = 1000#header['integration_time']
-                ds['temperature'][scan_number] = 1000#header['temperature']
+                ds['integration_time'][scan_number] = header['integration_time']
+                ds['temperature'][scan_number] = header['temperature']
 
                 # accelaration:
                 # Reference acceleration data contains 3x 16 bit signed integers with X, Y and Z
@@ -314,13 +314,16 @@ def read_series(self, seq_dir, series, lat, lon, metadata, flag, fileformat):
 
                 a = 19.6
                 b = 2 ** 15
-                # ds['acceleration_x_mean'][scan_number] = header['acceleration_x_mean'] * a / b
-                # ds['acceleration_x_std'][scan_number] = header['acceleration_x_std'] * a / b
-                # ds['acceleration_y_mean'][scan_number] = header['acceleration_y_mean'] * a / b
-                # ds['acceleration_y_std'][scan_number] = header['acceleration_y_std'] * a / b
-                # ds['acceleration_z_mean'][scan_number] = header['acceleration_z_mean'] * a / b
-                # ds['acceleration_z_std'][scan_number] = header['acceleration_z_std'] * a / b
-
+                ds['acceleration_x_mean'][scan_number] = header['acceleration_x_mean'] * a / b
+                ds['acceleration_x_std'][scan_number] = header['acceleration_x_std'] * a / b
+                ds['acceleration_y_mean'][scan_number] = header['acceleration_y_mean'] * a / b
+                ds['acceleration_y_std'][scan_number] = header['acceleration_y_std'] * a / b
+                ds['acceleration_z_mean'][scan_number] = header['acceleration_z_mean'] * a / b
+                ds['acceleration_z_std'][scan_number] = header['acceleration_z_std'] * a / b
+
+                print(header['Pixel Count'])
+                print(pixCount)
+                print(len(scan))
                 ds['digital_number'][0:pixCount, scan_number] = scan
 
                 scan_number += 1
@@ -388,7 +391,7 @@ def read_metadata(self, seq_dir):
             else:
                 print("Latitude is not given, use default")
                 lat = self.context.get_config_value("lat")
-                flag = flag + 2 ** self.context.get_config_value("lat_default")
+                flag = flag + 2 ** self.context.get_config_value("lat_default") #du.set_flag(flag, "lat_default") #
 
             if 'longitude' in (globalattr.keys()):
                 lon = float(globalattr['longitude'])
@@ -397,7 +400,7 @@ def read_metadata(self, seq_dir):
             else:
                 print("Longitude is not given, use default")
                 lon = self.context.get_config_value("lon")
-                flag = flag + 2 ** self.context.get_config_value("lon_default")
+                flag = flag + 2 ** self.context.get_config_value("lon_default") #du.set_flag(flag, "lon_default")  #
 
             # 2. Estimate wavelengths - NEED TO CHANGE HERE!!!!!!
             # ----------------------
@@ -477,14 +480,14 @@ def read_sequence(self, seq_dir, setfile=None):
         else:
             print("No irradiance data for this sequence")
 
-        # if seriesRad:
-        #     L0_RAD = self.read_series(seq_dir, seriesRad, lat, lon, metadata,flag, "L0_RAD")
-        # #         if all([os.path.isfile(os.path.join(seq_dir,"RADIOMETER/",f)) for f in seriesRad]):
-        # #             L0_RAD=read_series(seriesRad,cc, lat, lon, metadata, "L0_RAD")
-        # #         else:
-        # #             print("Radiance files listed but don't exist")
-        # else:
-        #     print("No radiance data for this sequence")
+        if seriesRad:
+            L0_RAD = self.read_series(seq_dir, seriesRad, lat, lon, metadata,flag, "L0_RAD")
+        #         if all([os.path.isfile(os.path.join(seq_dir,"RADIOMETER/",f)) for f in seriesRad]):
+        #             L0_RAD=read_series(seriesRad,cc, lat, lon, metadata, "L0_RAD")
+        #         else:
+        #             print("Radiance files listed but don't exist")
+        else:
+            print("No radiance data for this sequence")
 
         if seriesBlack:
             L0_BLA = self.read_series(seq_dir, seriesBlack, lat, lon, metadata,flag, "L0_BLA")

hypernets_processor/data_io/format/variables.py

@@ -41,10 +41,88 @@ pt_ask=90.00; 180.00
 pt_abs=90.00; 180.00
 pt_ref=90.13; 179.93
 
+[01_002_0090_2_0180]
+01_002_0090_2_0180_192_00_0000_03_0000.spe=20200506T130519
+pt_ask=90.00; 180.00
+pt_abs=90.00; 180.00
+pt_ref=89.78; 179.77
+
+[01_003_0090_2_0180]
+01_003_0090_2_0180.jpg=20200506T130519
+pt_ask=90.00; 180.00
+pt_abs=90.00; 180.00
+pt_ref=89.87; 180.20
+
+[01_004_0090_2_0140]
+01_004_0090_2_0140_128_16_0000_03_0000.spe=20200506T130519
+pt_ask=90.00; 140.00
+pt_abs=90.00; 140.00
+pt_ref=90.09; 139.76
+
+[01_005_0090_2_0180]
+01_005_0090_2_0180_192_00_0000_03_0000.spe=20200506T130527
+pt_ask=90.00; 180.00
+pt_abs=90.00; 180.00
+pt_ref=90.09; 179.96
+
+[01_006_0090_2_0140]
+01_006_0090_2_0140.jpg=20200506T130535
+pt_ask=90.00; 140.00
+pt_abs=90.00; 140.00
+pt_ref=89.89; 140.04
+
+[01_007_0090_2_0040]
+01_007_0090_2_0040_128_16_0000_06_0000.spe=20200506T130543
+pt_ask=90.00; 40.00
+pt_abs=90.00; 40.00
+pt_ref=90.12; 40.24
+
+[01_008_0090_2_0180]
+01_008_0090_2_0180_192_00_0000_03_0000.spe=20200506T130603
+pt_ask=90.00; 180.00
+pt_abs=90.00; 180.00
+pt_ref=89.78; 180.12
+
+[01_009_0090_2_0040]
+01_009_0090_2_0040.jpg=20200506T130631
+pt_ask=90.00; 40.00
+pt_abs=90.00; 40.00
+pt_ref=89.98; 39.92
+
+[01_010_0090_2_0140]
+01_010_0090_2_0140_128_16_0000_03_0000.spe=20200506T130659
+pt_ask=90.00; 140.00
+pt_abs=90.00; 140.00
+pt_ref=90.05; 139.82
+
+[01_011_0090_2_0180]
+01_011_0090_2_0180_192_00_0000_03_0000.spe=20200506T130719
+pt_ask=90.00; 180.00
+pt_abs=90.00; 180.00
+pt_ref=90.05; 179.88
+
+[01_012_0090_2_0140]
+01_012_0090_2_0140.jpg=20200506T130727
+pt_ask=90.00; 140.00
+pt_abs=90.00; 140.00
+pt_ref=89.94; 139.99
+
 [01_013_0090_2_0180]
 01_013_0090_2_0180_128_08_0000_03_0000.spe=20200506T130735
 pt_ask=90.00; 180.00
 pt_abs=90.00; 180.00
 pt_ref=90.19; 180.05
 
+[01_014_0090_2_0180]
+01_014_0090_2_0180_192_00_0000_03_0000.spe=20200506T130743
+pt_ask=90.00; 180.00
+pt_abs=90.00; 180.00
+pt_ref=89.87; 180.10
+
+[01_015_0090_2_0180]
+01_015_0090_2_0180.jpg=20200506T130743
+pt_ask=90.00; 180.00
+pt_abs=90.00; 180.00
+pt_ref=90.19; 179.80
+
 

hypernets_processor/data_io/hypernets_reader.py

@@ -3,8 +3,6 @@ PyxisVersion=PYXIS_V000.50
 Datetime=20200821T154756
 PI=CPE2
 Site_name=Villefranche-sur-mer
-Lat=43.69862 
-Lon=7.30692
 
 [01_001_0090_2_0180]
 pt_ask=90.00;180.00

hypernets_processor/data_io/tests/reader/SEQ20200506T130443/metadata.txt

@@ -14,6 +14,7 @@ verbosity: 0
 #raw_data_directory: None
 #metadata_db_url: None
 
+#SEQ20201016T141413
 sequence_id: SEQ20200821T154756
 Lat: 43.69886
 Lon: 7.30720