Pre rel006 (#42)

* package_data/ATL11_output_attrs.csv: removed slope_change parameters

* point.py : save values for reference surf across-track and along-track slopes

* ATL11/get_xover_data.py: added missing LR and dh_fit_dx fields to the crossover fields

* geoloc_biases_from_xovers.py: script for generating crossover bias estimates

* ATL11/rtw_mask.py: minor changes of unclear significance

* ATL11/package_data/xy_biases_rel006_c1_18_plateau__0_800km_3day.csv: bias estimates for rel006 through cycle 18

* ATL11/data.py: skip writing poly_exponent_list attribute for beam pair groups

* c19_rel006_held_granules.csv: Avoid processing the last day of cycle 19

* scripts/geoloc_biases_from_xovers.py: make functions importable

* moved the 2020 held_granules file to an 'ignore' directory

* scripts/geoloc_biases_from_xovers.py: read the hold files

* ATL11/package_data/xy_biases_rel006_c1_19_plateau__0_800km_3day.csv: xy biases calculated using version 2 of the April-15 granules

* scripts/geoloc_biases_from_xovers.py: don't filter if there are not holds

* Updated hold_list code to check crossover data

* removed blacklists directory and files

* ATL11/check_ATL06_hold_list.py : return if hold list is None, and move the hold_list reading code out of check_hold_list

* adding xy_bias to scripts

* add seg_azimuth to xover read

* ATL06_to_ATL11.py: check for successful ATL06 read before calculating geolocation bias

* accounting for empty ATL06 files

---------

Co-authored-by: Ben Smith <bsmith@apl.washington.edu>
Co-authored-by: Ben Smith <besmith@uw.edu>
Co-authored-by: Benjamin P. Jelley <bjelley@discover13.prv.cube>

Author: 4 people

ATL11/check_ATL06_hold_list.py

@@ -14,7 +14,10 @@ def read_hold_files(hold_dir=None):
     if hold_dir is None:
         hold_dir = str(resources.files('ATL11').joinpath("package_data/held_granules"))
     rgt=[]; cycle=[]; subprod=[];
-    for file in glob.glob(os.path.join(hold_dir, '*.csv')):
+    hold_files=glob.glob(os.path.join(hold_dir, '*.csv'))
+    if hold_files is None or len(hold_files)==0:
+        return None
+    for file in hold_files:
         with open(file,'r') as ff:
             for line in ff:
                 try:
@@ -28,15 +31,67 @@ def read_hold_files(hold_dir=None):
     return hold_list
 
 
-def check_ATL06_hold_list(filenames, hold_list=None, hold_dir=None):
-    if hold_list is None:
-        hold_list=read_hold_files(hold_dir=hold_dir)
+def check_ATL06_hold_list(filenames, hold_list=None):
+    """
+    Check if files should be held.
+
+    Parameters
+    ----------
+    filenames : list or string
+        Filenames to check against the hold list
+    hold_list : list of iterables, optional
+        list of cycle, rgt, subproducts to hole. The default is None.
+
+    Returns
+    -------
+    list of booleans
+        Boolean for each file: True=skip, False=use.
+
+    """
+
     if isinstance(filenames, (str)):
         filenames=list(filenames)
 
+    if hold_list is None:
+        return [False]*len(filenames)
+
     r06=re.compile('ATL.._(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)_(\d\d\d\d)(\d\d)(\d\d)_(\d\d\d)_(\d\d).h5')
     bad=[]
     for filename in filenames:
         m=r06.search(filename)
         bad.append((int(m.group(8)), int(m.group(7)), int(m.group(9))) in hold_list)
     return bad
+
+def check_ATL06_data_against_hold_list(D, hold_list=None):
+    """
+    Remove held rgt/cycle combinations from a data object.
+
+    Parameters
+    ----------
+    D : pc.data
+        ATL06 data, must have cycle (or cycle_number) and rgt fields
+    hold_list : list, optinal
+        list of cycle, rgt combinations that will be excluded from
+        analysis. The default is None.
+
+    Returns
+    -------
+    None.
+
+    """
+
+    if hold_list is None:
+        return
+
+    # assume that hold_list is cycle, rgt, subproduct
+    if 'cycle_number' in D.fields:
+        c=D.cycle_number
+    else:
+        c=D.cycle
+    hold_arr=np.c_[hold_list]
+    cr = np.round(c).astype(int)+1j*np.round(D.rgt).astype(int)
+    good = ~np.in1d(cr,  hold_arr[:,0]+1j*hold_arr[:,1])
+    if good.ndim==2:
+        good=np.all(good, axis=2)
+    if not np.all(good):
+        D.index(good)

ATL11/data.py

@@ -334,6 +334,8 @@ def write_to_file(self, fileout, params_11=None):
 
         # write each variable in params_11 as an attribute
         for param, val in  vars(params_11).items():
+            if param=="poly_exponent_list":
+                continue
             if not isinstance(val,(dict,type(None))):
                 try:
                     if param == 'ATL06_xover_field_list':
@@ -573,6 +575,7 @@ def from_ATL06(self, D6, GI_files=None, beam_pair=1, cycles=[1, 12],\
                    ref_pt_numbers=None, ref_pt_x=None, hemisphere=-1,\
                    mission_time_bds=None, max_xover_latitude=90, \
                    atc_shift_table=None, release_bias_dict=None,\
+                   hold_list=None,\
                    verbose=False, DOPLOT=None,DEBUG=None, return_list=True):
         """
         Fit a collection of ATL06 files with ATL11 surface models
@@ -589,6 +592,7 @@ def from_ATL06(self, D6, GI_files=None, beam_pair=1, cycles=[1, 12],\
                 mission_time_bds: starting and ending times for the mission
                 verbose: write fitting info to stdout if true
                 atc_shift_table: dict giving the along-track bias correction as a function of time
+                hold_list: list of cycle, rgt, subproduct combinations to exclude
                 DOPLOT: list of plots to make
                 DEBUG: output debugging info
         """
@@ -713,6 +717,7 @@ def from_ATL06(self, D6, GI_files=None, beam_pair=1, cycles=[1, 12],\
                 D_xover=ATL11.get_xover_data(x0, y0, P11.rgt, GI_files,
                                              D_xover_cache, index_bin_size, params_11,
                                              xy_bin=D6_xyB, release_bias_dict=release_bias_dict,
+                                             hold_list=hold_list,
                                              verbose=verbose)
                 P11.corr_xover_heights(D_xover, atc_shift_table=atc_shift_table)
             # if we have read any data for the current bin, run the crossover calculation

ATL11/get_xover_data.py

@@ -13,6 +13,7 @@
 import h5py
 import re
 from ATL11 import apply_release_bias
+from ATL11.check_ATL06_hold_list import check_ATL06_data_against_hold_list
 
 def get_ATL06_release(D6):
     if D6 is None:
@@ -34,6 +35,7 @@ def get_ATL06_release(D6):
 
 def get_xover_data(x0, y0, rgt, GI_files, xover_cache, index_bin_size, params_11,
                    release_bias_dict=None,
+                   hold_list=None,
                    verbose=False, xy_bin=None):
     """
     Read the data from other tracks.
@@ -53,11 +55,12 @@ def get_xover_data(x0, y0, rgt, GI_files, xover_cache, index_bin_size, params_11
     D_xover=[]
     ATL06_fields = ['delta_time', 'latitude','longitude', 'h_li', 'h_li_sigma',
                     'atl06_quality_summary', 'segment_id', 'sigma_geo_h',
-                    'x_atc', 'y_atc',
+                    'x_atc', 'y_atc', 'dh_fit_dx',
                     'sigma_geo_at','sigma_geo_xt', 'sigma_geo_r',
-                    'ref_azimuth', 'ref_coelv',
+                    'ref_azimuth', 'ref_coelv', 'seg_azimuth',
                     'tide_ocean', 'dac',
-                    'rgt', 'cycle_number', 'BP',  'spot',
+                    'rgt', 'cycle_number',
+                    'BP',  'spot', 'LR',
                     'source_file_num']
 
     for x0_ctr in x0_ctrs:
@@ -106,6 +109,8 @@ def get_xover_data(x0, y0, rgt, GI_files, xover_cache, index_bin_size, params_11
                     D_xover.append(xover_cache[this_key]['D'][np.arange(i0, i1+1, dtype=int)])
     if len(D_xover) > 0:
         D_xover=pc.data().from_list(D_xover)
+        if hold_list is not None:
+            check_ATL06_data_against_hold_list(D_xover, hold_list)
 
     # cleanup the cache if it is too large
     if len(xover_cache.keys()) > 5:

ATL11/package_data/ATL11_output_attrs.csv

@@ -23,10 +23,6 @@ ref_surf,poly_exponent_y,counts,N_coeffs,int8,"""exponents for the y factors in
 ref_surf,x_atc,meters,N_pts,Float64,"""Along-track coordinate of the reference point, measured along the RGT from its first equator crossing.""","""Along track distance""","""ATL06""",../delta_time ../latitude ../longitude,0.0,3600000.0
 ref_surf,y_atc,meters,N_pts,Float64,"""Across-track coordinate of the reference point, measured along the RGT from its first equator crossing.""","""Across track distance""","""ATL06""",../delta_time ../latitude ../longitude,-20000.0,20000.0
 ref_surf,rgt_azimuth,degrees,N_pts,Float32,"""Reference track azimuth, in degrees east of local north""","""RGT azimuth""","""ATL06""",../delta_time ../latitude ../longitude,-180.0,180.0
-ref_surf,slope_change_rate_x,years^-1,N_pts,Float32,"""rate of change of the x component of the surface slope""","""x component of slope chnage rate""","""derived, ATL11 algorithm""",../delta_time ../latitude ../longitude,,
-ref_surf,slope_change_rate_y,years^-1,N_pts,Float32,"""rate of change of the y component of the  surface slope""","""y component of slope change rate""","""derived, ATL11 algorithm""",../delta_time ../latitude ../longitude,,
-ref_surf,slope_change_rate_x_sigma,years^-1,N_pts,Float32,"""Formal error in the rate of change of the x component of the surface slope""","""error of x component slope change rate""","""derived, ATL11 algorithm""",../delta_time ../latitude ../longitude,,
-ref_surf,slope_change_rate_y_sigma,years^-1,N_pts,Float32,"""Formal error in the rate of change of the y component of the surface slope""","""error of y component slope change rate""","""derived, ATL11 algorithm""",../delta_time ../latitude ../longitude,,
 ref_surf,misfit_chi2r,meters,N_pts,Float32,"""misfit chi square, divided by the number of degrees in the solution""","""misfit chi square""","""derived, ATL11 algorithm""",../delta_time ../latitude ../longitude,0.0,
 ref_surf,misfit_RMS,meters,N_pts,Float32,"""RMS misfit for the surface-polynomial fit""","""misfit RMS""","""derived, ATL11 algorithm""",../delta_time ../latitude ../longitude,0.0,
 ref_surf,dem_h,meters,N_pts,Float32,"""DEM elevation, derived from ATL06 /gtxx/atl06_segments/dem/dem_h""","""DEM elevation""","""ATL06""",../delta_time ../latitude ../longitude,-100.0,7500.0