Update base class for products with RSLC (rather than SLC) group (#773)

* fix Base class for SLC/RSLC and L2 productTypes

* make Product.cpp general for SLC/RSLC and S-SAR

* fix Base class setProductType() method

* add RRSD to the list of NISAR products

* add ProductFactory() class

* add ProductFactory() class (2)

* make method setProductType() private

* set productType as RSLC regardless of the HDF5 group RSLC/SLC

* update value assignment of productType in Base and SLC classes

* instantiate existing product classes before creating a new ProductFactory object

* simplify _setProduct method of SLC class after changes in ProductFactory

* make Base.py general for LSAR and SSAR frequency bands

* avoid memory allocation and string copying

* add comments to Product.cpp and limit the maximum number of grids/swaths and identification groups to 1

* add function to find unique group paths (excluding repeated occurrences)

* add unitest to ProductFactory class

* fix ProductFactory for GSLC products

* fix ProductFactory for GSLC products (2)

* fix extractWithIterator for val undefined if KeyError exception is raised

* create open_nisar_product and move ProductFactory to GenericProduct

* rename open_nisar_product() to open_product()

* substitute backslashes with parenthesis for line continuation

* substitute get_product_type_from_hdf5File() with get_product_type_from_hdf5_file()

* improve docstrings and function/variable names

* if logger is declared (not None) use logger to raise an exception in h5prep

* send exceptions to pyre journal rather than raising default python exceptions

* declare pyre error channel locally

Author: gshiroma

cxx/isce3/product/Product.cpp

@@ -1,21 +1,140 @@
 #include "Product.h"
 #include <isce3/product/Serialization.h>
 
+/** Find unique group path excluding repeated occurrences
+*/
+std::vector<std::string> _findGroupPath(isce3::io::IGroup& group, 
+                                        const std::string group_name) {
+    
+    auto group_vector = group.find(group_name, ".", "GROUP");
+
+    std::vector<std::string> filtered_group_vector;
+
+    // Filter unique group paths
+    for (const auto& full_group_path: group_vector) {
+        std::size_t group_name_position = full_group_path.find(group_name);
+
+        // Get base group path (remove sub-groups)
+        std::string base_group = full_group_path.substr(
+            0, group_name_position + group_name.length());
+
+        /* If base group is not in filtered_group_vector, 
+        append it to the vector
+        */
+        if (find(filtered_group_vector.begin(), 
+                 filtered_group_vector.end(), 
+                 base_group) ==  filtered_group_vector.end()) {
+            filtered_group_vector.push_back(base_group);
+            }
+    }
+
+    return filtered_group_vector;
+}
+
+
 /** @param[in] file IH5File object for product. */
 isce3::product::Product::
 Product(isce3::io::IH5File & file) {
-    // Get swaths group
-    isce3::io::IGroup imGroup = file.openGroup("/science/LSAR/SLC/swaths");
+
+    std::string base_dir = "/science/";
+
+    isce3::io::IGroup base_group = file.openGroup(base_dir);
+
+    bool flag_has_swaths = false;
+
+    /* 
+    In this section, we look for grids or swaths group within base_group.
+    We start by assigning an empty string to image_group_str in case
+    grids and swaths group are not found. 
+    */
+   std::vector<std::string> key_vector = {"grids", "swaths"};
+    std::string image_group_str = "", metadata_group_str; 
+    for (const auto& key: key_vector) {
+
+        // Look for HDF5 groups that match key (i.e., "grids" or "swaths")
+        auto group_vector = _findGroupPath(base_group, key);
+
+        if (group_vector.size() > 1) {
+            /*
+            NISAR products should have only one grids or swaths group
+            within base_group:
+            */ 
+            std::string error_msg = ("ERROR there should be at most one " +
+                                     key + " group in " +file.getFileName());
+            throw isce3::except::RuntimeError(ISCE_SRCINFO(), error_msg);
+        }
+        else if (group_vector.size() > 0) {
+
+            // Group swaths or grid was found
+
+            /* 
+            Assign the group (grids or swaths) to image_group_str,
+            locate the group metadata by substituting "grids"/"swaths" with
+            "metadata", and save it on metadata_group_str
+            */
+            image_group_str = base_dir + group_vector[0];
+            metadata_group_str = image_group_str;
+            std::size_t key_position = metadata_group_str.rfind(key);
+            metadata_group_str.replace(key_position, key.length(), "metadata");
+            if (key == "swaths") {
+                /* 
+                flag_has_swaths indicates if image_group_str reffers to swaths or
+                grids
+                */
+                flag_has_swaths = true;
+            }
+            break;
+        } 
+    }
+
+    // If did not find HDF5 groups swaths or grids
+    if (image_group_str.size() == 0) {
+        std::string error_msg = ("ERROR swaths and grids groups"
+                                 " not found in " + file.getFileName());
+        throw isce3::except::RuntimeError(ISCE_SRCINFO(), error_msg);
+    }
+
+    // Get swaths/grids group
+    isce3::io::IGroup imGroup = file.openGroup(image_group_str);
+
     // Configure swaths
-    loadFromH5(imGroup, _swaths);
+    if (flag_has_swaths) {
+        loadFromH5(imGroup, _swaths);
+    } 
+    /* 
+    Not implemented yet:
+    else {
+        loadFromH5(imGroup, _grids); 
+    }
+    */
+
     // Get metadata group
-    isce3::io::IGroup metaGroup = file.openGroup("/science/LSAR/SLC/metadata"); 
+    isce3::io::IGroup metaGroup = file.openGroup(metadata_group_str); 
     // Configure metadata
+
     loadFromH5(metaGroup, _metadata);
+    
     // Get look direction
+    auto identification_vector = _findGroupPath(
+        base_group, "identification");
+    if (identification_vector.size() == 0) {
+            std::string error_msg = ("ERROR identification group not found in " +
+                                     file.getFileName());
+            throw isce3::except::RuntimeError(ISCE_SRCINFO(), error_msg);
+    }
+    else if (identification_vector.size() > 1) {
+            std::string error_msg = ("ERROR there should be only one identification"
+                                     " group in " + file.getFileName());
+            throw isce3::except::RuntimeError(ISCE_SRCINFO(), error_msg);
+    }
+    
+    std::string identification_group_str = base_dir + identification_vector[0];
     std::string lookDir;
-    isce3::io::loadFromH5(file, "/science/LSAR/identification/lookDirection", lookDir);
+    isce3::io::loadFromH5(file, identification_group_str + "/lookDirection", 
+                          lookDir);
     lookSide(lookDir);
+
     // Save the filename
     _filename = file.filename();
+
 }

python/packages/pybind_nisar/h5/h5utils.py

@@ -41,12 +41,16 @@ def extractWithIterator(h5grp, key, iterType, logger=None, msg=None):
         val = h5grp[key][()]
         val = [iterType(x) for x in val]
     except KeyError:
+        errmsg = '{0} not found at {1}'.format(key, h5grp.name)
         if logger:
-            logger.log('{0} not found at {1}'.format(key, h5grp.name))
+            logger.log(errmsg)
+        raise KeyError(errmsg)
     except:
         errmsg = 'Something went wrong when trying to parse {0}/{1} \n'.format(h5grp.name, key)
         if msg is not None:
             errmsg += msg
+        if logger:
+            logger.log(errmsg)
         raise ValueError(errmsg)
 
     return val

python/packages/pybind_nisar/products/readers/Base/Base.py

@@ -2,10 +2,51 @@
 
 import h5py
 import os
+import journal
 import pyre
 import pybind_isce3 as isce3
 from ..protocols import ProductReader
 
+
+def get_hdf5_file_root_path(filename: str, root_path: str = None) -> str:
+    '''
+    Return root path from HDF5 file. If a root path is provided as
+    a parameter and it exists, it will have precedence over the 
+    root path from the HDF5 file
+
+    Parameters
+    ----------
+    filename : str
+        HDF5 filename
+    root_path : str
+        Preliminary root path to check (e.g., XSAR, PSAR) before default root 
+        path list
+
+    Returns
+    -------
+    str
+        Root path from HDF5 file
+
+    '''
+
+    error_channel = journal.error('get_hdf5_file_root_path')
+
+    SCIENCE_PATH = '/science/'
+    NISAR_FREQ_BAND_LIST = ['SSAR', 'LSAR']
+    with h5py.File(filename, 'r', libver='latest', swmr=True) as f:
+        if root_path is not None and root_path in f:
+            return root_path
+        science_group = f[SCIENCE_PATH]
+        for freq_band in NISAR_FREQ_BAND_LIST:
+            if freq_band not in science_group:
+                continue
+            return SCIENCE_PATH + freq_band
+
+    error_msg = ("HDF5 could not find NISAR frequency"
+                 f" band group LSAR or SSAR in file: {filename}")
+
+    error_channel.log(error_msg)
+
 class Base(pyre.component,
            family='nisar.productreader.base',
            implements=ProductReader):
@@ -17,13 +58,13 @@ class Base(pyre.component,
     _CFPath = pyre.properties.str(default='/')
     _CFPath.doc = 'Absolute path to scan for CF convention metadata'
 
-    _RootPath = pyre.properties.str(default='/science/LSAR')
+    _RootPath = pyre.properties.str(default=None)
     _RootPath.doc = 'Absolute path to SAR data from L-SAR/S-SAR'
 
     _IdentificationPath = pyre.properties.str(default='identification')
     _IdentificationPath.doc = 'Absolute path ath to unique product identification information'
 
-    _ProductType = pyre.properties.str(default='SLC')
+    _ProductType = pyre.properties.str(default=None)
     _ProductType.doc = 'The type of the product.'
 
     _MetadataPath = pyre.properties.str(default='metadata')
@@ -41,10 +82,19 @@ class Base(pyre.component,
     productValidationType = pyre.properties.str(default='BASE')
     productValidationType.doc = 'Validation tag to compare identification information against to ensure that the right product type is being used.'
 
-    def __init__(self, hdf5file='None', **kwds):
+    def __init__(self, hdf5file=None, **kwds):
         '''
         Constructor.
         '''
+
+        # Set error channel
+        self.error_channel = journal.error('Base')
+
+        # Check hdf5file
+        if hdf5file is None:
+            err_str = f"Please provide an input HDF5 file"
+            self.error_channel.log(err_str)
+
         # Filename
         self.filename = hdf5file
 
@@ -55,8 +105,12 @@ def __init__(self, hdf5file='None', **kwds):
         self.polarizations = {}
 
         self.populateIdentification()
-        #For now, Needs to be an assertion check in the future
-        self.identification.productType = self.productValidationType
+
+        self.identification.productType = self._ProductType
+
+        if self._ProductType is None:
+            return
+
         self.parsePolarizations()
 
     @pyre.export
@@ -141,28 +195,6 @@ def getSlantRange(self, frequency='A'):
 
         return slantRange
 
-
-    def getSlcDataset(self, frequency, polarization):
-        '''
-        Return SLC dataset of given frequency and polarization from hdf5 file
-        '''
-
-        # TODO add checks for (1) file open error (2) path check
-        slcDataset = None
-
-        # open H5 with swmr mode enabled
-        fid = h5py.File(self.filename, 'r', libver='latest', swmr=True)
-
-        # build path the desired dataset
-        dsPath = self.slcPath(frequency, polarization)
-
-        # get dataset
-        slcDataset = fid[dsPath]
-
-        # return dataset
-        return slcDataset
-
-
     def parsePolarizations(self):
         '''
         Parse HDF5 and identify polarization channels available for each frequency.
@@ -195,6 +227,9 @@ def CFPath(self):
 
     @property
     def RootPath(self):
+        if self._RootPath is None:
+            self._RootPath = get_hdf5_file_root_path(
+                self.filename)
         return self._RootPath
 
     @property
@@ -255,12 +290,4 @@ def computeBoundingBox(self, epsg=4326):
         '''
         raise NotImplementedError
 
-    def slcPath(self, frequency, polarization):
-        '''
-        return path to hdf5 dataset of given frequency and polarization
-        '''
-        datasetPath = os.path.join(self.SwathPath, f'frequency{frequency}', polarization)
-        return datasetPath
-
-
 # end of file

python/packages/pybind_nisar/products/readers/Base/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
 
-from .Base import Base
+from .Base import Base, get_hdf5_file_root_path
 
 # end of file