WIP: Workflow rubbersheet (#771)

* Add rubbersheet parameters to insar schema and defaults runconfig

* Rubbersheet runconfig

* Rubbersheet main

* Add rubbersheet as InSAR workflow step

* Correct mispelled out_paths for rubbersheet

* Copying culled dense offsets from RIFG to RUNW

* Add culled offsets among the datasets to geocode

* Turn off rubbersheet when executing insar.py CPU unit test

* Separate median filter kernel from moving average kernel

* Add rubbersheet unit test and its CMake dependency

* Remove horizontal comments for dense offsets unit test runconfig

* Correct mistake in smoothing function

* Simply mask application for range/azmuth dense offsets

* Simplify code for geocoding along track and slant range offsets

* Remove unnecessary parenthesis for rubbersheet step execution

* Simply copy of datasets from RIFG to RUNW

* Have one line function comments in rubbersheet unit test

* Add checks to run rubbersheet only if user has enabled dense offsets

* Modify schemas, defaults and rubbersheet.py to optionally apply mask refinement

* Doc string for functions in rubbersheet

* Expose number of iteration for fill_smoothed and check for residual NaNs

* Remove hybrid outlier filling method (redundancy of solutions)

* Reorganize rubbersheet parameters in schema and expose more filter options

* Organize rubbersheet schema and expose more filter options

* Correct formatting

* Alphabetical order of module imports

* Rename mask3x3 to mask_refine

* Check dense offsets and rubbersheet enabled flag in insar_runconfig.py

* Sum rubbersheet offsets to geo2rdr offsets

* Remove comment on failing rubbersheeting without flag enabled

* Expose interpolation method to the user

* Correct description of VRT file generation

* Simplify string notation

Co-authored-by: vbrancat <[email protected]>

Author: 2 people

python/packages/pybind_nisar/workflows/geocode_insar.py

@@ -98,6 +98,12 @@ def run(cfg, runw_hdf5, output_hdf5):
                     if not geocode_this_dataset:
                         continue
 
+                    if dataset_name in ['alongTrackOffset','slantRangeOffset']:
+                        src_group_path = f"/science/LSAR/RUNW/swaths/frequency{freq}/pixelOffsets/{pol}"
+                        dst_group_path = f"/science/LSAR/GUNW/grids/frequency{freq}/pixelOffsets/{pol}"
+
+                    # prepare input raster
+
                     if (dataset_name == "layoverShadowMask"):
                         # prepare input raster
                         raster_ref = scratch_path / 'rdr2geo' / f'freq{freq}' / 'mask.rdr'
@@ -119,6 +125,7 @@ def run(cfg, runw_hdf5, output_hdf5):
                         radar_grid = radar_grid_multilook
                         dataset_path = f"{dst_group_path}/{dataset_name}"
 
+
                     geocoded_dataset = dst_h5[dataset_path]
 
                     # Construct the output ratster directly from HDF5 dataset
@@ -168,6 +175,7 @@ def run(cfg, runw_hdf5, output_hdf5):
     info_channel.log(f"Successfully ran geocode in {t_all_elapsed:.3f} seconds")
 
 
+
 if __name__ == "__main__":
     """
     run geocode from command line

python/packages/pybind_nisar/workflows/geocode_insar_runconfig.py

@@ -46,7 +46,8 @@ def yaml_check(self):
             self.cfg['processing']['geocode']['datasets'] = {}
 
         # default to True for datasets not found
-        gunw_datasets = ["connectedComponents", "coherenceMagnitude", "unwrappedPhase", "layoverShadowMask"]
+        gunw_datasets = ["connectedComponents", "coherenceMagnitude", "unwrappedPhase",
+                         "alongTrackOffset", "slantRangeOffset", "layoverShadowMask"]
         for gunw_dataset in gunw_datasets:
             if gunw_dataset not in self.cfg['processing']['geocode']:
                 self.cfg['processing']['geocode']['datasets'][gunw_dataset] = True

python/packages/pybind_nisar/workflows/insar.py

@@ -4,7 +4,7 @@
 import journal
 from pybind_nisar.workflows import (crossmul, dense_offsets, geo2rdr,
                                     geocode_insar, h5_prep, rdr2geo,
-                                    resample_slc, unwrap)
+                                    resample_slc, rubbersheet, unwrap)
 from pybind_nisar.workflows.insar_runconfig import InsarRunConfig
 from pybind_nisar.workflows.persistence import Persistence
 from pybind_nisar.workflows.yaml_argparse import YamlArgparse
@@ -15,6 +15,7 @@ def run(cfg: dict, out_paths: dict, run_steps: dict):
     Run INSAR workflow with parameters in cfg dictionary
     '''
     info_channel = journal.info("insar.run")
+    error_channel = journal.error('insar.run')
     info_channel.log("starting INSAR")
 
     t_all = time.time()
@@ -32,6 +33,10 @@ def run(cfg: dict, out_paths: dict, run_steps: dict):
             (cfg['processing']['dense_offsets']['enabled']):
         dense_offsets.run(cfg)
 
+    if run_steps['rubbersheet'] and \
+            cfg['processing']['rubbersheet']['enabled']:
+        rubbersheet.run(cfg, out_paths['RIFG'])
+
     if run_steps['crossmul']:
         crossmul.run(cfg, out_paths['RIFG'])
 

python/packages/pybind_nisar/workflows/insar_runconfig.py

@@ -14,6 +14,14 @@ def yaml_check(self):
         '''
         scratch_path = self.cfg['ProductPathGroup']['ScratchPath']
 
+        # If dense_offsets is disabled and rubbersheet is enabled
+        # throw an exception and do not run the workflow
+        if not self.cfg['processing']['dense_offsets']['enabled'] and \
+                self.cfg['processing']['rubbersheet']['enabled']:
+            err_str = "Dense_offsets must be enabled to run rubbersheet"
+            error_channel.log(err_str)
+            raise RuntimeError(err_str)
+
         # for each submodule check if user path for input data assigned
         # if not assigned, assume it'll be in scratch
         if 'topo_path' not in self.cfg['processing']['geo2rdr']:
@@ -26,6 +34,16 @@ def yaml_check(self):
             self.cfg['processing']['dense_offsets'][
                 'coregistered_slc_path'] = scratch_path
 
+        # When running insar.py dense_offsets_path and geo2rdr_offsets_path
+        # come from previous step through scratch_path
+        if self.cfg['processing']['rubbersheet']['dense_offsets_path'] is None:
+            self.cfg['processing']['rubbersheet'][
+                'dense_offsets_path'] = scratch_path
+
+        if self.cfg['processing']['rubbersheet']['geo2rdr_offsets_path'] is None:
+            self.cfg['processing']['rubbersheet'][
+                'geo2rdr_offsets_path'] = scratch_path
+
         if 'coregistered_slc_path' not in self.cfg['processing']['crossmul']:
             self.cfg['processing']['crossmul'][
                 'coregistered_slc_path'] = scratch_path
@@ -55,7 +73,9 @@ def yaml_check(self):
 
         # default to True for datasets not found
         gunw_datasets = ["connectedComponents", "coherenceMagnitude",
-                         "unwrappedPhase", 'layoverShadowMask']
+                         "unwrappedPhase", "alongTrackOffset", "slantRangeOffset",
+                         'layoverShadowMask']
+
         for gunw_dataset in gunw_datasets:
             if gunw_dataset not in self.cfg['processing']['geocode']:
                 self.cfg['processing']['geocode']['datasets'][

python/packages/pybind_nisar/workflows/persistence.py

@@ -7,7 +7,9 @@ class Persistence():
     basic class that determines InSAR persistence
     '''
     # init InSAR steps in reverse chronological run order
-    insar_steps = ['geocode', 'unwrap', 'crossmul', 'dense_offsets', 'resample', 'geo2rdr', 'rdr2geo', 'h5_prep']
+    insar_steps = ['geocode', 'unwrap', 'crossmul', 'rubbersheet',
+                   'dense_offsets', 'resample', 'geo2rdr', 'rdr2geo', 'h5_prep']
+
     def __init__(self, restart=False):
         # bool flag that determines if insar.run is called
         # prevents calling of insar.run if last run was successful and no restart