Skip to content

Commit 8e2fd3d

Browse files
jshimada47jshimada47
committed
Added support for NISAR Soil Moisture Aggregation algorithm.
1 parent ec5248c commit 8e2fd3d

File tree

5 files changed

+118
-9
lines changed

5 files changed

+118
-9
lines changed

‎tools/imagesets/centos7conda/distrib_nisar/Dockerfile‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,27 +8,36 @@ RUN apk add git
88
# add credentials on build
99
ARG GIT_OAUTH_TOKEN
1010
RUN cd /opt \
11+
&& git clone https://[email protected]/NISAR-ADT/SoilMoisture \
1112
&& git clone https://[email protected]/NISAR-ADT/QualityAssurance \
1213
&& git clone https://[email protected]/NISAR-ADT/CFChecker \
1314
&& git clone https://[email protected]/NISAR-ADT/calTools \
1415
&& cd /opt/QualityAssurance && git checkout 448db8d && rm -rf .git \
1516
&& cd /opt/CFChecker && git checkout R2 && rm -rf .git \
16-
&& cd /opt/calTools && git checkout 5607f81 && rm -rf .git
17+
&& cd /opt/calTools && git checkout 5607f81 && rm -rf .git \
18+
&& cd /opt/SoilMoisture && git checkout 80e14ac && rm -rf .git
1719

1820
FROM $distrib_img
1921

2022
RUN conda install testfixtures scikit-image
2123
RUN conda install cfunits --channel conda-forge
2224

25+
# Soil Moisture
26+
RUN conda install make --channel conda-forge
27+
RUN conda create -n SoilMoisture gcc gfortran netcdf-cxx4 netcdf-fortran --channel conda-forge
28+
2329
# copy the repo from the intermediate image
2430
COPY --from=0 /opt/QualityAssurance /opt/QualityAssurance
2531
COPY --from=0 /opt/CFChecker /opt/CFChecker
2632
COPY --from=0 /opt/calTools /opt/calTools
33+
COPY --from=0 /opt/SoilMoisture /opt/SoilMoisture
2734

2835
# install
2936
RUN cd /opt/QualityAssurance \
3037
&& python setup.py install \
3138
&& cd /opt/CFChecker \
3239
&& python setup.py install \
3340
&& cd /opt/calTools \
34-
&& python setup.py install
41+
&& python setup.py install \
42+
&& cd /opt/SoilMoisture \
43+
&& env CONDA_PREFIX=/opt/conda/envs/SoilMoisture PATH=/opt/conda/bin:/opt/conda/envs/SoilMoisture/bin:/usr/bin make install

‎tools/imagesets/imgset.py‎

Lines changed: 45 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,12 @@ def run_with_logging(dockercall, cmd, logger, printlog=True):
4444
# save command to log
4545
logger.info("++ " + cmdstr + "\n")
4646
pipe = subprocess.Popen(shlex.split(cmdstr), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
47+
48+
# Maximum number of seconds to wait for "docker run" to finish after
49+
# its child process exits. The observed times have been < 1 ms.
50+
# Use a relatively large number to flag a possible problem with Docker.
51+
timeout = 10
52+
4753
with pipe.stdout:
4854
for line in iter(pipe.stdout.readline, b''): # b'\n'-separated lines
4955
decoded = line.decode("utf-8")
@@ -52,6 +58,9 @@ def run_with_logging(dockercall, cmd, logger, printlog=True):
5258
decoded = decoded[:-1]
5359
logger.info(decoded)
5460
ret = pipe.poll()
61+
if ret is None:
62+
ret = pipe.wait(timeout=timeout)
63+
# ret will be None if exception TimeoutExpired was raised and caught.
5564
if ret != 0:
5665
raise subprocess.CalledProcessError(ret, cmdstr)
5766

@@ -223,8 +232,8 @@ def makedistrib(self):
223232

224233
def makedistrib_nisar(self):
225234
"""
226-
Install package to redistributable isce3 docker image with nisar qa and
227-
noise estimator caltool
235+
Install package to redistributable isce3 docker image with nisar qa,
236+
noise estimator caltool, and Soil Moisture applications
228237
"""
229238

230239
build_args = f"--build-arg distrib_img={self.imgname()} \
@@ -327,7 +336,7 @@ def distribrun(self, testdir, cmd, logfile=None, dataname=None, nisarimg=False,
327336

328337
def workflowtest(self, wfname, testname, dataname, pyname, suf="", description="", arg=""):
329338
"""
330-
Run the specified workflow test using the distrib image.
339+
Run the specified workflow test using either the distrib or the nisar image.
331340
332341
Parameters
333342
-------------
@@ -370,10 +379,21 @@ def workflowtest(self, wfname, testname, dataname, pyname, suf="", description="
370379
shutil.copyfile(pjoin(runconfigdir, inputrunconfig),
371380
pjoin(testdir, f"runconfig_{wfname}{suf}.yaml"))
372381
log = pjoin(testdir, f"output_{wfname}{suf}", "stdouterr.log")
373-
cmd = [f"time python3 -m {pyname} {arg} runconfig_{wfname}{suf}.yaml"]
382+
383+
if not testname.startswith("sm"):
384+
cmd = [f"time python3 -m {pyname} {arg} runconfig_{wfname}{suf}.yaml"]
385+
else:
386+
executable = pyname
387+
cmd = [f"time {executable} runconfig_{wfname}{suf}.yaml"]
388+
374389
try:
375-
self.distribrun(testdir, cmd, logfile=log, dataname=dataname,
376-
loghdlrname=f'wftest.{os.path.basename(testdir)}')
390+
if not testname.startswith("sm"):
391+
self.distribrun(testdir, cmd, logfile=log, dataname=dataname,
392+
loghdlrname=f'wftest.{os.path.basename(testdir)}')
393+
else:
394+
# Currently, the SM executables are in the nisar image.
395+
self.distribrun(testdir, cmd, logfile=log, dataname=dataname, nisarimg=True,
396+
loghdlrname=f"wftest.{os.path.basename(testdir)}")
377397
except subprocess.CalledProcessError as e:
378398
raise RuntimeError(f"Workflow test {testname} failed") from e
379399

@@ -481,6 +501,25 @@ def beamformtest(self, tests=None):
481501
except subprocess.CalledProcessError as e:
482502
raise RuntimeError(f"CalTool beamformer tool test {testname} failed") from e
483503

504+
def smtest(self, tests=None):
505+
if tests is None:
506+
tests = workflowtests['sm'].items()
507+
for testname, dataname in tests:
508+
# Note: we will eventually have multiple SM executables, each
509+
# of which implements a different algorithm. These executables
510+
# will run the same input test data. It's TBD whether they'll
511+
# be able to share the same runconfig. The output files should
512+
# be either written to different directories by executable or
513+
# should be named to indicate which executable was used, or both.
514+
#
515+
# Also, the current plan is for two of the SM executables to be
516+
# Fortran 90 binaries and the other two to be Python modules.
517+
sm_bindir = '/opt/conda/envs/SoilMoisture/bin'
518+
executables = [ 'NISAR_SM_DISAGG_SAS' ]
519+
cmd = [ ]
520+
for executable in executables:
521+
self.workflowtest("sm", testname, dataname, f"{sm_bindir}/{executable}")
522+
484523
def mintests(self):
485524
"""
486525
Only run first test from each workflow

‎tools/imagesets/runconfigs/sm_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
--Folder path to NISAR GCOV data
2+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/NISAR_L2_PR_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004.h5
3+
--Folder path to ECMWF sol moisture
4+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ERA5_SD_TP_ST_SM.nc
5+
--Folder path and file to static ancillary data: EZlat200m
6+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lat200m.float32
7+
--Folder path and file to static ancillary data: EZlon200m
8+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lon200m.float32
9+
--Folder path and file to static ancillary data: EZlat1km
10+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lat01km.float32
11+
--Folder path and file to static ancillary data: EZlon1km
12+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lon01km.float32
13+
--Folder path and file to static ancillary data: EZlat3km
14+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lat03km.float32
15+
--Folder path and file to static ancillary data: EZlon3km
16+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lon03km.float32
17+
--Folder path and file to static ancillary data: EZlat9km
18+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lat09km.float32
19+
--Folder path and file to static ancillary data: EZlon9km
20+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lon09km.float32
21+
--Folder path and file to static ancillary data: Landcover1km
22+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/dominantIGBP_M01_B20120101_003.uint8
23+
--Folder path and file to static ancillary data: WaterFrac1km
24+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/waterfrac_M01_002.float32
25+
--Folder path and file to static ancillary data: WaterFrac3km
26+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/waterfrac_M03_002.float32
27+
--Folder path and file to static ancillary data: BD200m
28+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/BD05cmD_EASE2_200m.nc
29+
--Folder path and file to static ancillary data: Urban1km
30+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/UrbanFraction_M01_001.float32
31+
--Folder path and file to static ancillary data: dem1km slope
32+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/DEMSLPSTD_M01_002.float32
33+
--Log file name with path
34+
output_sm/NISAR_SM_20200926_Disagg_ver1.log
35+
--Output HDF5 file name with path
36+
output_sm/NISAR_SM_20200926_Disagg_ver1.h5

‎tools/imagesets/workflowdata.py‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,9 +161,29 @@
161161
"dem.tif",
162162
"README.txt",
163163
],
164+
165+
"L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004":
166+
[
167+
"NISAR_L2_PR_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004.h5",
168+
"ERA5_SD_TP_ST_SM.nc",
169+
"ancillary/DEMSLPSTD_M01_002.float32",
170+
"ancillary/dominantIGBP_M01_B20120101_003.uint8",
171+
"ancillary/EASE2_lat01km.float32",
172+
"ancillary/EASE2_lat03km.float32",
173+
"ancillary/EASE2_lat09km.float32",
174+
"ancillary/EASE2_lat200m.float32",
175+
"ancillary/EASE2_lon01km.float32",
176+
"ancillary/EASE2_lon03km.float32",
177+
"ancillary/EASE2_lon09km.float32",
178+
"ancillary/EASE2_lon200m.float32",
179+
"ancillary/UrbanFraction_M01_001.float32",
180+
"ancillary/waterfrac_M01_002.float32",
181+
"ancillary/waterfrac_M03_002.float32",
182+
"README.txt",
183+
],
164184
}
165185

166-
# dictionaries definining mappig of workflow tests to data
186+
# dictionaries definining mapping of workflow tests to data
167187
# each key is the test name, value is the corresponding data
168188
workflowtests = {
169189
'rslc': {"rslc_" + name: "L0B_RRSD_" + name for name in [
@@ -245,4 +265,8 @@
245265
"instrumentTables_bf_19000101.h5", # instrument tables
246266
],
247267
},
268+
269+
'sm': {"sm_" + name: "L2_GCOV_" + name for name in [
270+
"001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004",
271+
]},
248272
}

‎tools/run.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ def run(*, steps, imgset, **kwargs):
2525
"noisesttest",
2626
"ptatest",
2727
"beamformtest",
28+
"smtest",
2829
"rslcqa",
2930
"gslcqa",
3031
"gcovqa",

0 commit comments

Comments
 (0)