Merge branch 'isce-framework:develop' into develop

gshiroma · web-flow · commit dd18a849c17b · 2025-09-17T11:00:26.000-07:00
diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml
@@ -20,6 +20,12 @@ jobs:
       # stage_dem requires AWS permissions to access the S3 bucket containing the DEM data.
       # pybind.unwrap.phass occasionally segfaults on GHA runners.
 
+      # Default values for optional matrix parameters
+      env-file: ${{ matrix.deps.env-file || 'environment.yml' }}
+      ctest-exclude: ${{ matrix.os.ctest_exclude || '-E ".*($ALL_EXCLUDE)"'}}
+      extra-specs: ${{ matrix.deps.extra-specs || '' }}
+      build-type: ${{ matrix.build-type || 'RelWithDebInfo' }}
+
     defaults:
       run:
         shell: bash -l {0}
@@ -29,23 +35,23 @@ jobs:
       matrix:
         os:
           - label: Linux
-            runner: ubuntu-latest
-            ctest_exclude: -E ".*($ALL_EXCLUDE)"
-          - runner: macos-latest
-            label: macOS
+          - label: macOS
+            runner: macos-latest
             ctest_exclude: -E ".*($ALL_EXCLUDE|io\.gdal\.raster|unwrap\.snaphu|background)"
             # io.gdal.raster and unwrap.snaphu tests fail due to known unresolved issues
             # with GDAL memory mapping on macOS.
             # background reader/writer test sometimes fails on macOS due to slow runtimes.
         deps:
           - label: Latest
-            env-file: environment.yml
-            extra-specs: ""
         build-type: [ Release, Debug ]
-
+        include:
+          - deps:
+              label: Minimum Python supported
+              extra-specs: |
+                python=3.9
 
     name: Build And Test - ${{ matrix.os.label }} ${{ matrix.deps.label }} ${{ matrix.build-type }}
-    runs-on: ${{ matrix.os.runner }}
+    runs-on: ${{ matrix.os.runner || 'ubuntu-latest' }}
     steps:
       # Typical github repo checkout step
       - name: Github Repo Checkout
@@ -54,20 +60,20 @@ jobs:
       # Prints the variables for this run to simplify debugging
       - name: Print Run Variables
         run: |
-          echo Runner OS: ${{ matrix.os.runner }}
-          echo Environment: ${{ matrix.deps.env-file }}
-          echo Build Type: ${{ matrix.build-type }}
-          echo ctest exclude command: ${{ matrix.os.ctest_exclude }}
-          echo Depenencies: ${{ matrix.deps.extra-specs }}
+          echo Runner OS: ${{ matrix.os.runner || 'ubuntu-latest' }}
+          echo Environment: ${{ env.env-file }}
+          echo Build Type: ${{ env.build-type }}
+          echo ctest exclude command: ${{ env.ctest_exclude }}
+          echo Dependencies: ${{ env.extra-specs }}
 
       # Set the conda environment up using Mamba and the dependencies for this run
       - name: Setup Environment
         uses: mamba-org/setup-micromamba@v2
         with:
-          environment-file: ${{ matrix.deps.env-file }}
+          environment-file: ${{ env.env-file }}
           environment-name: isce3
           create-args: >-
-            ${{ matrix.deps.extra-specs }}
+            ${{ env.extra-specs }}
 
       # Fix missing conda command on macOS
       - name: macOS conda command fixup
@@ -89,7 +95,7 @@ jobs:
           cmake \
             -B build \
             -G Ninja \
-            -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
+            -DCMAKE_BUILD_TYPE=${{ env.build-type }} \
             -DCMAKE_INSTALL_PREFIX=$ISCE3_PREFIX \
             -DCMAKE_PREFIX_PATH=$CONDA_PREFIX \
             .
@@ -122,4 +128,4 @@ jobs:
         run: |
           cd build
           export GDAL_MEM_ENABLE_OPEN=YES
-          ctest --output-on-failure ${{ matrix.os.ctest_exclude }}
+          ctest --output-on-failure ${{ env.ctest-exclude }}
diff --git a/python/packages/nisar/antenna/pattern.py b/python/packages/nisar/antenna/pattern.py
@@ -400,27 +400,27 @@ def form_pattern(self, tseq, slant_range: Linspace,
             if tx_pol == "L":
                 tx_bmf_pat[tx_pol] = (
                     self.tx_bmf['H'].form_pattern(
-                        t, slant_range, nearest=nearest,
+                        tseq, slant_range, nearest=nearest,
                         channel_adj_factors=self.channel_adj_fact_tx['H']) +
                     1j * self.tx_bmf['V'].form_pattern(
-                        t, slant_range, nearest=nearest,
+                        tseq, slant_range, nearest=nearest,
                         channel_adj_factors=self.channel_adj_fact_tx['V'])
                 ).astype(np.complex64)
 
             elif tx_pol == "R":
                 tx_bmf_pat[tx_pol] = (
                     self.tx_bmf['H'].form_pattern(
-                        t, slant_range, nearest=nearest,
+                        tseq, slant_range, nearest=nearest,
                         channel_adj_factors=self.channel_adj_fact_tx['H']) -
                     1j * self.tx_bmf['V'].form_pattern(
-                        t, slant_range, nearest=nearest,
+                        tseq, slant_range, nearest=nearest,
                         channel_adj_factors=self.channel_adj_fact_tx['V'])
                 ).astype(np.complex64)
 
             else:  # other non-compact pol types
                 adj = self.channel_adj_fact_tx[tx_pol]
                 tx_bmf_pat[tx_pol] = self.tx_bmf[tx_pol].form_pattern(
-                        t, slant_range, nearest=nearest, channel_adj_factors=adj
+                        tseq, slant_range, nearest=nearest, channel_adj_factors=adj
                     ).astype(np.complex64)
 
         # build two-way pattern for all unique TxRx products obtained from all
diff --git a/python/packages/nisar/workflows/focus.py b/python/packages/nisar/workflows/focus.py
@@ -1816,6 +1816,8 @@ def temp(suffix):
                 z[np.isnan(z)] = 0.0
                 if cfg.processing.zero_fill_gaps:
                     fill_gaps(z, swaths[:, pulse:pulse+nblock, :], 0.0)
+                if cfg.processing.nullify_azimuth_mean:
+                    z -= z.mean(axis=0)
                 raw_mm[block_out] = z
 
             raw_clean, rfi_likelihood = process_rfi(cfg, raw_mm, temp)
diff --git a/python/packages/nisar/workflows/ionosphere.py b/python/packages/nisar/workflows/ionosphere.py
@@ -461,6 +461,16 @@ def compute_differential_phase(phase_first,
         The differential phase data is written directly to the specified
         output datasets. This function does not return any data.
     """
+    def _to_complex_if_needed(arr):
+        """
+        Ensure an array is complex. If it's already complex, return as-is.
+        Otherwise interpret values as phase (radians) and convert with exp(j*phi).
+        """
+        if np.iscomplexobj(arr):
+            return arr
+        # Cast to float to avoid integer exponentiation issues
+        return np.exp(1j * arr)
+
     # Check if reading and writing will happen on the same file
     is_same_file_first_output = (phase_first == output_path)
     is_same_file_first_second = (phase_first == phase_second)
@@ -521,13 +531,19 @@ def compute_differential_phase(phase_first,
                         second_data_block = get_raster_block(
                             phase_second_raster,
                             block_param_side)
-                        block_param = block_param_side
+                        chosen_block_param = block_param_side
                     else:
                         second_data_block = get_raster_block(
                             phase_second_raster,
                             block_param_main)
-                        block_param = block_param_main
+                        chosen_block_param = block_param_main
+
+                    # Ensure complex (convert real-valued phase in radians to
+                    # complex phase)
+                    first_data_block = _to_complex_if_needed(first_data_block)
+                    second_data_block = _to_complex_if_needed(second_data_block)
 
+                    # Optional resampling of FIRST to SECOND grid
                     if resampling_flag:
                         first_data_block = decimate_freq_a_array(
                             main_slant,
@@ -539,7 +555,7 @@ def compute_differential_phase(phase_first,
 
                     # Write result block to output
                     write_raster_block(output_data_raster,
-                                       diff_phase, block_param)
+                                       diff_phase, chosen_block_param)
         finally:
             # Close the output file if it was opened separately
             if not is_same_file_first_second:
@@ -826,7 +842,7 @@ def insar_ionosphere_pair(original_cfg, runw_hdf5):
         if iono_method == 'main_diff_ms_band':
             diff_dir = pathlib.Path(orig_scratch_path,
                                     'ionosphere', 'diff_ms')
-            phase_first = original_out_paths['RIFG']
+            phase_first = original_out_paths['RUNW']
             if rerun_insar_pairs > 0:
                 additional_runw = f'{new_scratch}/RUNW.h5'
                 phase_second = pathlib.Path(orig_scratch_path,
@@ -860,14 +876,16 @@ def insar_ionosphere_pair(original_cfg, runw_hdf5):
             pol_list_a = iono_freq_pols['A']
             pol_list_b = iono_freq_pols['B']
             swath_path = RIFGGroupsPaths().SwathsPath
+            runw_swath_path = RUNWGroupsPaths().SwathsPath
+
             first_data_path = []
             for pol_a in pol_list_a:
 
-                dest_freq_path = f"{swath_path}/frequencyA"
+                dest_freq_path = f"{runw_swath_path}/frequencyA"
                 dest_pol_path = f"{dest_freq_path}/interferogram/{pol_a}"
-                rifg_path_freq = f"{dest_pol_path}/wrappedInterferogram"
+                runw_path_freq = f"{dest_pol_path}/unwrappedPhase"
 
-                first_data_path.append(rifg_path_freq)
+                first_data_path.append(runw_path_freq)
             first_slant_path = f"{dest_freq_path}/interferogram/slantRange"
             second_data_path = []
             for pol_b in pol_list_b:
diff --git a/share/nisar/defaults/focus.yaml b/share/nisar/defaults/focus.yaml
@@ -415,6 +415,12 @@ runconfig:
             # cal pulses.
             zero_fill_gaps: True
 
+            # If true, subtract the azimuth mean from the raw data, which is
+            # useful when the caltone is nearly constant in azimuth but not
+            # stationary in range.  Calculating the mean and subtracting it is
+            # done in blocks using rangecomp.block_size
+            nullify_azimuth_mean: True
+
             # Processing stage switches, mostly for debug.
             # Any missing stages assumed True
             is_enabled:
diff --git a/share/nisar/defaults/insar.yaml b/share/nisar/defaults/insar.yaml
@@ -1184,25 +1184,25 @@ runconfig:
                         # (-1 logs the percentages as INFO, but will not trigger a QA failure.)
 
                         # Percent of NaN pixels per total raster area allowed
-                        # Default: 95.0
-                        nan_threshold: 95.0
+                        # Default: -1
+                        nan_threshold: -1
 
                         # Percent of +/- Inf pixels per total raster area allowed
                         # Default: 95.0
                         inf_threshold: 95.0
 
                         # Percent of Fill-valued pixels per total raster area allowed
-                        # Default: 95.0
-                        fill_threshold: 95.0
+                        # Default: -1
+                        fill_threshold: -1
 
                         # Percent of Near-zero pixels per total raster area allowed
                         # Default: -1
                         near_zero_threshold: -1
 
                         # Percent of total invalid pixels per total raster area allowed
                         # 'Total invalid pixels' is the sum of NaN, Inf, fill, & (if `zero_is_invalid` is True) near-zero pixels.
-                        # Default: 95.0
-                        total_invalid_threshold: 95.0
+                        # Default: -1
+                        total_invalid_threshold: -1
 
                         # Absolute tolerance for determining if a raster pixel is 'near zero'.
                         # Default: 1e-06
@@ -1603,25 +1603,25 @@ runconfig:
                         # (-1 logs the percentages as INFO, but will not trigger a QA failure.)
 
                         # Percent of NaN pixels per total raster area allowed
-                        # Default: 99.0
-                        nan_threshold: 99.0
+                        # Default: -1
+                        nan_threshold: -1
 
                         # Percent of +/- Inf pixels per total raster area allowed
                         # Default: 99.0
                         inf_threshold: 99.0
 
                         # Percent of Fill-valued pixels per total raster area allowed
-                        # Default: 99.0
-                        fill_threshold: 99.0
+                        # Default: -1
+                        fill_threshold: -1
 
                         # Percent of Near-zero pixels per total raster area allowed
                         # Default: -1
                         near_zero_threshold: -1
 
                         # Percent of total invalid pixels per total raster area allowed
                         # 'Total invalid pixels' is the sum of NaN, Inf, fill, & (if `zero_is_invalid` is True) near-zero pixels.
-                        # Default: 99.0
-                        total_invalid_threshold: 99.0
+                        # Default: -1
+                        total_invalid_threshold: -1
 
                         # Absolute tolerance for determining if a raster pixel is 'near zero'.
                         # Default: 1e-06
diff --git a/share/nisar/schemas/focus.yaml b/share/nisar/schemas/focus.yaml
@@ -253,6 +253,12 @@ runconfig:
       # If true, fill transmit gaps with zeros to remove loop-back cal pulses.
       zero_fill_gaps: bool(required=False)
 
+      # If true, subtract the azimuth mean from the raw data, which is useful
+      # when the caltone is nearly constant in azimuth but not stationary in
+      # range.  Calculating the mean and subtracting it is done in blocks using
+      # rangecomp.block_size
+      nullify_azimuth_mean: bool(required=False)
+
       # Processing stage switches, mostly for debug.
       # Any missing stages assumed True
       is_enabled:
diff --git a/tools/imagesets/oracle8conda/distrib_nisar/Dockerfile b/tools/imagesets/oracle8conda/distrib_nisar/Dockerfile
@@ -11,7 +11,7 @@ ARG GIT_OAUTH_TOKEN
 RUN cd /opt \
  && git clone https://$GIT_OAUTH_TOKEN@github-fn.jpl.nasa.gov/NISAR-ADT/SoilMoisture \
  && git clone https://github.com/isce-framework/nisarqa.git \
- && cd /opt/nisarqa && git checkout v14.0.1 && rm -rf .git \
+ && cd /opt/nisarqa && git checkout v15.0.1 && rm -rf .git \
  && cd /opt/SoilMoisture && git checkout v0.3.1 && rm -rf .git
 
 FROM $distrib_img
