ALOS-2 PALSAR L1.1 to NISAR RSLC HDF5 (#529)

* adding first dirty (but "working") version of the ALOS-2 to HDF5 software

* increase az. time precision, add Doppler, fix multiple bugs, and refactor code

* address PR reviews. Throw exception for invalid time type when saving attitude records. Remove unnecessary code.

* remove constraint to the number of polarization channels

* pull changes from #531: Quick update to the Geocode Cython binding constructor

* rename alos2 Python scripts

* read HDF5 groups only if they exist

* simplify code by importing functions from ALOS2NISARL0B.py, clean and refactor code, and update calls to pybind bindings

* Write the ALOS-2 RSLC product using an RSLC template

* allows for overwriting fields previously copied during h5_prep (e.g. isGeocoded)

Author: gshiroma

cxx/isce3/product/Serialization.h

@@ -45,19 +45,26 @@ namespace isce3 {
 
             // Load effective velocity LUT
             isce3::core::LUT2d<double> lut;
-            isce3::core::loadCalGrid(group, "effectiveVelocity", lut);
-            proc.effectiveVelocity(lut);
+            if (isce3::io::exists(group, "effectiveVelocity")) {
+                isce3::core::loadCalGrid(group, "effectiveVelocity", lut);
+                proc.effectiveVelocity(lut); 
+            }
 
-            // Load azimuth FM rate and Doppler centroid for primary frequency (A)
-            isce3::core::loadCalGrid(group, "frequencyA/azimuthFMRate", lut);
-            proc.azimuthFMRate(lut, 'A');
+            if (isce3::io::exists(group, "frequencyA/azimuthFMRate")) {
+                // Load azimuth FM rate and Doppler centroid for primary frequency (A)
+                isce3::core::loadCalGrid(group, "frequencyA/azimuthFMRate", lut);
+                proc.azimuthFMRate(lut, 'A');
+            }
             isce3::core::loadCalGrid(group, "frequencyA/dopplerCentroid", lut);
             proc.dopplerCentroid(lut, 'A');
 
             // Check for existence of secondary frequencies
             if (isce3::io::exists(group, "frequencyB")) {
-                isce3::core::loadCalGrid(group, "frequencyB/azimuthFMRate", lut);
-                proc.azimuthFMRate(lut, 'B');
+
+                if (isce3::io::exists(group, "frequencyB/azimuthFMRate")) {
+                    isce3::core::loadCalGrid(group, "frequencyB/azimuthFMRate", lut);
+                    proc.azimuthFMRate(lut, 'B');
+                }
                 isce3::core::loadCalGrid(group, "frequencyB/dopplerCentroid", lut);
                 proc.dopplerCentroid(lut, 'B');
             }

python/packages/isce3/stripmap/readers/l1/ALOS2/CEOS/AttitudeDataRecordType.py

@@ -0,0 +1,13 @@
+from isce3.parsers.CEOS.AttitudeDataRecordType import (AttitudeDataRecordHeaderType,
+    AttitudeDataRecordStateVectorType)
+
+def AttitudeDataRecordTrailerType(inlen):
+    '''
+    Attitude Data Record Trailer.
+    https://www.eorc.jaxa.jp/ALOS-2/en/doc/fdata/PALSAR-2_xx_Format_CEOS_E_f.pdf
+    '''
+
+    from isce3.parsers.CEOS.BasicTypes import BlankType
+
+    return BlankType(inlen)
+

python/packages/isce3/stripmap/readers/l1/ALOS2/CEOS/DataSetSummaryRecordType.py

@@ -0,0 +1,149 @@
+def DatasetSummaryRecordType():
+    '''
+    Data Set Summary Record.
+    https://www.eorc.jaxa.jp/ALOS-2/en/doc/fdata/PALSAR-2_xx_Format_CEOS_E_f.pdf
+    '''
+    from isce3.parsers.CEOS.BasicTypes import (BlankType, StringType, IntegerType, FloatType, MultiType)
+    from isce3.parsers.CEOS.CEOSHeaderType import CEOSHeaderType
+
+    #Common part of CEOS header (modified)
+    inlist = CEOSHeaderType().mapping 
+    inlist += [('DSRecordSequenceNumber', IntegerType(4)),
+               ('SARChannelIndicator', StringType(4)),
+               ('SceneIdentifier', StringType(32)),
+               ('SceneDesignator', StringType(16)),
+               ('InputSceneCenterTime', StringType(32)),
+               ('blanks1', BlankType(16)),
+               ('SceneCenterLatitude', BlankType(16)),
+               ('SceneCenterLongitude', BlankType(16)),
+               ('SceneCenterHeading', BlankType(16)),
+               ('EllipsoidDesignator', StringType(16)),
+               ('EllipsoidSemiMajorAxisInKm', FloatType(16)),
+               ('EllipsoidSemiMinorAxisInKm', FloatType(16)),
+               ('EarthMassIn1024Kg', FloatType(16)),
+               ('GravitationalConstant', FloatType(16)),
+               ('EllipsoidJ2Parameter', FloatType(16)),
+               ('EllipsoidJ3Parameter', FloatType(16)),
+               ('EllipsoidJ4Parameter', FloatType(16)),
+               ('blanks2', BlankType(16)),
+               ('AverageTerrainHeight', FloatType(16)),
+               ('SceneCenterLineNumber', IntegerType(8)),
+               ('SceneCenterPixelNumber', IntegerType(8)),
+               ('ProcessedSceneLengthInKm', FloatType(16)),
+               ('ProcessedSceneWidthInKm', FloatType(16)),
+               ('blanks3', BlankType(16)),
+               ('NumberOfSARChannels', IntegerType(4)),
+               ('blanks4', BlankType(4)),
+               ('SensorPlatformMissionIdentifier', StringType(16)),
+               ('SensorIDAndMode', StringType(32)),
+               ('OrbitNumber', IntegerType(8)),
+               ('SensorPlatformLatitude', BlankType(8)),
+               ('SensorPlatformLongitude', BlankType(8)),
+               ('SensorPlatformHeading', BlankType(8)),
+               ('SensorClockAngle', FloatType(8)),
+               ('SceneCenterIncidenceAngle', FloatType(8)),
+               ('blanks5', BlankType(8)),
+               ('RadarWavelengthInm', FloatType(16)),
+               ('MotionCompensationIndicator', StringType(2)),
+               ('RangePulseCodeSpecifier', StringType(16)),
+               ('RangePulseAmplitudeCoefficientDCTerm', FloatType(16)),
+               ('RangePulseAmplitudeCoefficientLinearTerm', FloatType(16)),
+               ('RangePulseAmplitudeCoefficientQuadraticTerm', FloatType(16)),
+               ('RangePulseAmplitudeCoefficientCubicTerm', FloatType(16)),
+               ('RangePulseAmplitudeCoefficientQuarticTerm', FloatType(16)),
+               ('RangePulsePhaseCoefficientDCTerm', FloatType(16)),
+               ('RangePulsePhaseCoefficientLinearTerm', FloatType(16)),
+               ('RangePulsePhaseCoefficientQuadraticTerm', FloatType(16)),
+               ('RangePulsePhaseCoefficientCubicTerm', FloatType(16)),
+               ('RangePulsePhaseCoefficientQuarticTerm', FloatType(16)),
+               ('DownlinkedDataChirpExtractionIndex', BlankType(8)),
+               ('blanks6', BlankType(8)),
+               ('SamplingRateInMHz', FloatType(16)),
+               ('RangeGateAtEarlyEdgeAtStartOfImageInusec', FloatType(16)),
+               ('RangePulseLengthInusec', FloatType(16)),
+               ('BasebandConversionFlag', StringType(4)),
+               ('RangeCompressedFlag', StringType(4)),
+               ('ReceiverGainCopolEarlyEdgeindB', FloatType(16)),
+               ('ReceiverGainXpolEarlyEdgeindB', FloatType(16)),
+               ('QuantizationBitsPerChannel', IntegerType(8)),
+               ('QuantizationDescriptor', StringType(12)),
+               ('DCBiasIComponent', FloatType(16)),
+               ('DCBiasQComponent', FloatType(16)),
+               ('GainImbalanceforIQ', FloatType(16)),
+               ('blanks7', BlankType(16)),
+               ('blanks8', BlankType(16)),
+               ('AntennaElectronicBoresightInDeg', FloatType(16)),
+               ('AntennaMechanicalBoresightInDeg', FloatType(16)),
+               ('EchoTrackerOnOffFlag', StringType(4)),
+               ('NominalPRFInmHz', FloatType(16)),
+               ('EffectiveTwoWayAntennaElevation3dBBeamwidthInDeg', FloatType(16)),
+               ('EffectiveTwoWayAntennaAzimuth3dBBeamwidthInDeg', FloatType(16)),
+               ('SatelliteEncodedBinaryTimeCode', IntegerType(16)),
+               ('SatelliteClockTime', StringType(32)),
+               ('SatelliteClockIncrementInnsec', IntegerType(16)),
+               ('ProcessingFacilityIdentifier', StringType(16)),
+               ('ProcessingSystemIdentifier', StringType(8)),
+               ('ProcessingVersionIdentifier', StringType(8)),
+               ('ProcessingFacilityProcessCode', StringType(16)),
+               ('ProductLevelCode', StringType(16)),
+               ('ProductTypeSpecifier', StringType(32)),
+               ('ProcessingAlgorithmIdentifier', StringType(32)),
+               ('NominalEffectiveAzimuthLooks', FloatType(16)),
+               ('NominalEffectiveRangeLooks', FloatType(16)),
+               ('BandwidthPerLookInAzimuthHz', FloatType(16)),
+               ('BandwidthPerLookInRangeHz', FloatType(16)),
+               ('TotalProcessorBandwidthInAzimuth', FloatType(16)),
+               ('TotalProcessorBandwidthInRange', FloatType(16)),
+               ('WeightingFunctionInAzimuth', StringType(32)),
+               ('WeightingFunctionInRange', StringType(32)),
+               ('DataInputSource', StringType(16)),
+               ('Nominal3dBResolutionInGroundRangeInm', FloatType(16)),
+               ('Nominal3dBResolutionInAzimuthInm', FloatType(16)),
+               ('ConstantRadiometricParameterBias', FloatType(16)),
+               ('LinearRadiometricParameterGain', FloatType(16)),
+               ('AlongTrackDopplerConstantTermInHz', FloatType(16)),
+               ('AlongTrackDopplerLinearTermInHzPerPixel', FloatType(16)),
+               ('AlongTrackDopplerQuadraticTermInHzPerPixel2', FloatType(16)),
+               ('blanks9', BlankType(16)),
+               ('CrossTrackDopplerConstantTermInHz', FloatType(16)),
+               ('CrossTrackDopplerLinearTermInHzPerPixel', FloatType(16)),
+               ('CrossTrackDopplerLinearTermInHzPerPixel2', FloatType(16)),
+               ('TimeDirectionIndicatorAlongPixel', StringType(8)),
+               ('TimeDirectionIndicatorAlongLine', StringType(8)),
+               ('AlongTrackDopplerRateConstantTermInHzPerSec', FloatType(16)),
+               ('AlongTrackDopplerRateLinearTermInHzPerSecPerPixel', FloatType(16)),
+               ('AlongTrackDopplerRateQuadraticTermInHzPerSecPerPixel2', FloatType(16)),
+               ('blanks10', BlankType(16)),
+               ('CrossTrackDopplerRateConstantTermInHzPerSec', FloatType(16)),
+               ('CrossTrackDopplerRateLinearTermInHzPerSecPerPixel', FloatType(16)),
+               ('CrossTrackDopplerRateQuadraticTermInHzPerSecPerPixel2', FloatType(16)),
+               ('blanks11', BlankType(16)),
+               ('LineContentIndicator', StringType(8)),
+               ('ClutterLockAppliedFlag', StringType(4)),
+               ('AutoFocusingAppliedFlag', StringType(4)),
+               ('LineSpacingInm', FloatType(16)),
+               ('PixelSpacingInm', FloatType(16)),
+               ('ProcessorRangeCompressionDesignator', StringType(16)),
+               ('DopplerCenterFrequencyConstantTerm', FloatType(16)),
+               ('DopplerCenterFrequencyLinearTerm', FloatType(16))]
+
+    #ALOS specific trailer
+    inlist += [('CalibrationDataIndicator', IntegerType(4)),
+               ('StartLineNumberCalibrationUpper', IntegerType(8)),
+               ('StopLineNumberCalibrationUpper', IntegerType(8)),
+               ('StartLineNumberCalibrationBottom', IntegerType(8)),
+               ('StopLineNumberCalibrationBottom', IntegerType(8)),
+               ('PRFSwitchingIndicator', IntegerType(4)),
+               ('LineLocatorOfPRFSwitching', IntegerType(8)),
+               ('blanks14', BlankType(16)),
+               ('YawSteeringModeFlag', IntegerType(4)),
+               ('ParameterTableNumber', IntegerType(4)),
+               ('NominalOffNadirAngle', FloatType(16)),
+               ('AntennaBeamNumber', IntegerType(4)),
+               ('blanks15', BlankType(28)),
+               ('blanks16', BlankType(120)),
+               ('NumberOfAnnotationPoints', IntegerType(8)),
+               ('blanks17', BlankType(2082))]
+
+    return MultiType( inlist )
+

python/packages/isce3/stripmap/readers/l1/ALOS2/CEOS/ImageFile.py

@@ -0,0 +1,109 @@
+##This is the first cut to get the HDF5 translation working.
+##Signal Record Iterator can be better designed
+##The code currently only iterates over the records - modification/ manipulation belongs to customer
+
+class ImageFile(object):
+    '''
+    Class for parsing ALOS-2 L1.1 CEOS Imagefile.
+    '''
+
+    def __init__(self, filename):
+        '''
+        Initialize object with leader filename.
+        '''
+        import os
+
+        #Save file name
+        self.name = filename 
+
+        #Save file size in bytes
+        self.size = os.stat(filename).st_size
+
+        #Leader file always seems to consist of same set of records.
+        #https://www.eorc.jaxa.jp/ALOS-2/en/doc/fdata/PALSAR-2_xx_Format_CEOS_E_f.pdf
+        self.fid = open(self.name, 'rb')
+
+        #Leader file descriptor
+        self.description = self.parseFileDescriptor()
+
+        #Line counter 
+        self.counter = 0
+
+    def close(self):
+        '''
+        Close file object.
+        '''
+        self.fid.close()
+
+    def parseFileDescriptor(self):
+        '''
+        Parse SAR Leaderfile descriptor record.
+        '''
+        from isce3.parsers.CEOS.ImageFileDescriptorType import ImageFileDescriptorType
+
+        #Description of record - seems to be common across missions
+        record = ImageFileDescriptorType()
+            
+        #Read the record
+        record.fromfile(self.fid)
+
+        #Sensor specific validators
+        #https://www.eorc.jaxa.jp/ALOS-2/en/doc/fdata/PALSAR-2_xx_Format_CEOS_E_f.pdf
+        assert(record.RecordSequenceNumber == 1)
+        assert(record.FirstRecordType == 50)
+        assert(record.RecordTypeCode == 192)
+        assert(record.SecondRecordSubType == 18)
+        assert(record.ThirdRecordSubType == 18)
+        assert(record.RecordLength == 720)
+        assert(record.RecordSequenceType == 'FSEQ')
+        assert(record.RecordCodeType == 'FTYP')
+        assert(record.RecordFieldType == 'FLGT')
+        assert(record.SampleDataLineNumberLocator == "13 4PB")
+        assert(record.SARChannelNumberLocator == "49 2PB")
+        assert(record.TimeOfSARDataLineLocator == "45 4PB")
+        assert(record.LeftFillCountLocator == "21 4PB")
+        assert(record.RightFillCountLocator == "29 4PB")
+        assert(record.SARDataFormatTypeCode == "C*8")
+        assert(record.NumberOfRightFillBitsWithinPixel == 0)
+        assert(record.NumberOfBytesPerDataGroup % record.NumberOfSamplesPerDataGroup == 0)
+        assert(record.NumberOfBytesOfSARDataPerRecord % (record.NumberOfBytesPerDataGroup//record.NumberOfSamplesPerDataGroup) == 0)
+
+        #Check length of record
+        assert(self.fid.tell() == record.RecordLength)
+
+        #Return the validated record
+        return record
+
+    def readNextLine(self):
+        '''
+        Read the next line from file.
+        '''
+        # from isce3.parsers.CEOS.SignalDataRecordType import SignalDataRecordType
+        from isce3.stripmap.readers.l1.ALOS2.CEOS.SignalDataRecordType import SignalDataRecordType
+
+        #Create record type with information from description
+        bytesperpixel = self.description.NumberOfBytesPerDataGroup // self.description.NumberOfSamplesPerDataGroup
+        pixels = self.description.NumberOfBytesOfSARDataPerRecord // bytesperpixel
+        record = SignalDataRecordType(pixels=pixels,
+                                      bytesperpixel=bytesperpixel)
+
+
+        #Read from file
+        record.fromfile(self.fid)
+        self.counter = self.counter + 1
+
+        #Sensor specific validators
+        assert(record.RecordSequenceNumber == (self.counter+1))
+        assert(record.FirstRecordType == 50)
+        assert(record.RecordTypeCode == 10)
+        assert(record.SecondRecordSubType == 18)
+        assert(record.ThirdRecordSubType == 20)
+        assert(record.ActualCountOfLeftFillPixels == 0)
+        assert(record.SARChannelCode == 0)
+        assert(record.ScanIDForScanSAR == 0)
+        assert(record.OnboardRangeCompressedFlag == 0)
+        assert(record.PulseTypeDesignator == 0)
+
+        return record
+
+