Skip to content

Commit fd74d38

Browse files
jshimada47GitHub Enterprise
authored andcommitted
Merge pull request #957 from isce-3/SM_integrate
Added support for NISAR Soil Moisture Disaggregation algorithm
2 parents a3c8aac + f0d3241 commit fd74d38

File tree

7 files changed

+174
-17
lines changed

7 files changed

+174
-17
lines changed

tools/imagesets/centos7conda/distrib_nisar/Dockerfile

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,27 +8,32 @@ RUN apk add git
88
# add credentials on build
99
ARG GIT_OAUTH_TOKEN
1010
RUN cd /opt \
11+
&& git clone https://[email protected]/NISAR-ADT/SoilMoisture \
1112
&& git clone https://[email protected]/NISAR-ADT/QualityAssurance \
1213
&& git clone https://[email protected]/NISAR-ADT/CFChecker \
1314
&& git clone https://[email protected]/NISAR-ADT/calTools \
1415
&& cd /opt/QualityAssurance && git checkout 448db8d && rm -rf .git \
1516
&& cd /opt/CFChecker && git checkout R2 && rm -rf .git \
16-
&& cd /opt/calTools && git checkout 5607f81 && rm -rf .git
17+
&& cd /opt/calTools && git checkout 5607f81 && rm -rf .git \
18+
&& cd /opt/SoilMoisture && git checkout 80e14ac && rm -rf .git
1719

1820
FROM $distrib_img
1921

2022
RUN conda install testfixtures scikit-image
2123
RUN conda install cfunits --channel conda-forge
2224

25+
# Soil Moisture
26+
COPY spec-file.txt /tmp/spec-file.txt
27+
RUN conda create -n SoilMoisture --file /tmp/spec-file.txt && conda clean -ay
28+
2329
# copy the repo from the intermediate image
2430
COPY --from=0 /opt/QualityAssurance /opt/QualityAssurance
2531
COPY --from=0 /opt/CFChecker /opt/CFChecker
2632
COPY --from=0 /opt/calTools /opt/calTools
33+
COPY --from=0 /opt/SoilMoisture /opt/SoilMoisture
2734

2835
# install
29-
RUN cd /opt/QualityAssurance \
30-
&& python setup.py install \
31-
&& cd /opt/CFChecker \
32-
&& python setup.py install \
33-
&& cd /opt/calTools \
34-
&& python setup.py install
36+
RUN cd /opt/QualityAssurance && python setup.py install
37+
RUN cd /opt/CFChecker && python setup.py install
38+
RUN cd /opt/calTools && python setup.py install
39+
RUN cd /opt/SoilMoisture && conda run -n SoilMoisture make install
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
gcc
2+
gfortran
3+
make
4+
netcdf-cxx4
5+
netcdf-fortran
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# This file may be used to create an environment using:
2+
# $ conda create --name <env> --file <this file>
3+
# platform: linux-64
4+
@EXPLICIT
5+
https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2
6+
https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2021.10.8-ha878542_0.tar.bz2
7+
https://conda.anaconda.org/conda-forge/noarch/kernel-headers_linux-64-2.6.32-he073ed8_15.tar.bz2
8+
https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.36.1-hea4e1c9_2.tar.bz2
9+
https://conda.anaconda.org/conda-forge/linux-64/libgcc-devel_linux-64-11.2.0-h0952999_12.tar.bz2
10+
https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-11.2.0-h5c6108e_12.tar.bz2
11+
https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-11.2.0-he4da1e4_12.tar.bz2
12+
https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-11.2.0-h69a702a_12.tar.bz2
13+
https://conda.anaconda.org/conda-forge/linux-64/libgomp-11.2.0-h1d223b6_12.tar.bz2
14+
https://conda.anaconda.org/conda-forge/noarch/sysroot_linux-64-2.12-he073ed8_15.tar.bz2
15+
https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-1_gnu.tar.bz2
16+
https://conda.anaconda.org/conda-forge/linux-64/binutils_impl_linux-64-2.36.1-h193b22a_2.tar.bz2
17+
https://conda.anaconda.org/conda-forge/linux-64/binutils_linux-64-2.36-hf3e587d_4.tar.bz2
18+
https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-11.2.0-h1d223b6_12.tar.bz2
19+
https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h7f98852_4.tar.bz2
20+
https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.18.1-h7f98852_0.tar.bz2
21+
https://conda.anaconda.org/conda-forge/linux-64/jpeg-9e-h7f98852_0.tar.bz2
22+
https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h516909a_1.tar.bz2
23+
https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-11.2.0-he4da1e4_12.tar.bz2
24+
https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.11-h36c2ea0_1013.tar.bz2
25+
https://conda.anaconda.org/conda-forge/linux-64/make-4.3-hd18ef5c_1.tar.bz2
26+
https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.3-h9c3ff4c_0.tar.bz2
27+
https://conda.anaconda.org/conda-forge/linux-64/openssl-3.0.0-h7f98852_2.tar.bz2
28+
https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-11.2.0-h82a94d6_12.tar.bz2
29+
https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2
30+
https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.11-h36c2ea0_1013.tar.bz2
31+
https://conda.anaconda.org/conda-forge/linux-64/gcc-11.2.0-h702ea55_4.tar.bz2
32+
https://conda.anaconda.org/conda-forge/linux-64/gcc_linux-64-11.2.0-h39a9532_4.tar.bz2
33+
https://conda.anaconda.org/conda-forge/linux-64/gfortran_impl_linux-64-11.2.0-h7a446d4_12.tar.bz2
34+
https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h10796ff_3.tar.bz2
35+
https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.46.0-ha19adfc_0.tar.bz2
36+
https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.10.0-ha35d2d1_2.tar.bz2
37+
https://conda.anaconda.org/conda-forge/linux-64/libzip-1.8.0-h1c5bbd1_1.tar.bz2
38+
https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.11-h27826a3_1.tar.bz2
39+
https://conda.anaconda.org/conda-forge/linux-64/gfortran_linux-64-11.2.0-h777b47f_4.tar.bz2
40+
https://conda.anaconda.org/conda-forge/linux-64/krb5-1.19.2-h48eae69_3.tar.bz2
41+
https://conda.anaconda.org/conda-forge/linux-64/gfortran-11.2.0-h8811e0c_4.tar.bz2
42+
https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.81.0-h494985f_0.tar.bz2
43+
https://conda.anaconda.org/conda-forge/linux-64/curl-7.81.0-h494985f_0.tar.bz2
44+
https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.1-nompi_h7f166f4_103.tar.bz2
45+
https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.8.1-nompi_hb3fd0d9_101.tar.bz2
46+
https://conda.anaconda.org/conda-forge/linux-64/netcdf-cxx4-4.3.1-nompi_h8a6a9d2_105.tar.bz2
47+
https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.5.4-nompi_h2b6e579_100.tar.bz2

tools/imagesets/imgset.py

Lines changed: 48 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -232,8 +232,8 @@ def makedistrib(self):
232232

233233
def makedistrib_nisar(self):
234234
"""
235-
Install package to redistributable isce3 docker image with nisar qa and
236-
noise estimator caltool
235+
Install package to redistributable isce3 docker image with nisar qa,
236+
noise estimator caltool, and Soil Moisture applications
237237
"""
238238

239239
build_args = f"--build-arg distrib_img={self.imgname()} \
@@ -336,7 +336,7 @@ def distribrun(self, testdir, cmd, logfile=None, dataname=None, nisarimg=False,
336336

337337
def workflowtest(self, wfname, testname, dataname, pyname, suf="", description="", arg=""):
338338
"""
339-
Run the specified workflow test using the distrib image.
339+
Run the specified workflow test using either the distrib or the nisar image.
340340
341341
Parameters
342342
-------------
@@ -348,7 +348,9 @@ def workflowtest(self, wfname, testname, dataname, pyname, suf="", description="
348348
Test input dataset(s) to be mounted (e.g. "L0B_RRSD_REE1", ["L0B_RRSD_REE1", "L0B_RRSD_REE2"]).
349349
If None, no input datasets are used.
350350
pyname : str
351-
Name of the isce3 module to execute (e.g. "nisar.workflows.focus")
351+
Name of the isce3 module to execute (e.g. "nisar.workflows.focus") or,
352+
for Soil Moisture (SM) testing, the name of the SAS executable to run
353+
(e.g. "NISAR_SM_DISAGG_SAS")
352354
suf: str
353355
Suffix in runconfig and output directory name to differentiate between
354356
reference and secondary data in end-to-end tests
@@ -374,15 +376,34 @@ def workflowtest(self, wfname, testname, dataname, pyname, suf="", description="
374376
# distinguish between the runconfig files for each individual workflow)
375377
if testname.startswith("end2end"):
376378
inputrunconfig = f"{testname}_{wfname}{suf}.yaml"
379+
shutil.copyfile(pjoin(runconfigdir, inputrunconfig),
380+
pjoin(testdir, f"runconfig_{wfname}{suf}.yaml"))
381+
elif testname.startswith("soilm"):
382+
# Executable-dependent. Currently works only for Disaggregation.
383+
inputrunconfig = f"{testname}{suf}.txt"
384+
shutil.copyfile(pjoin(runconfigdir, inputrunconfig),
385+
pjoin(testdir, f"runconfig_{wfname}{suf}.txt"))
377386
else:
378387
inputrunconfig = f"{testname}{suf}.yaml"
379-
shutil.copyfile(pjoin(runconfigdir, inputrunconfig),
380-
pjoin(testdir, f"runconfig_{wfname}{suf}.yaml"))
388+
shutil.copyfile(pjoin(runconfigdir, inputrunconfig),
389+
pjoin(testdir, f"runconfig_{wfname}{suf}.yaml"))
381390
log = pjoin(testdir, f"output_{wfname}{suf}", "stdouterr.log")
382-
cmd = [f"time python3 -m {pyname} {arg} runconfig_{wfname}{suf}.yaml"]
391+
392+
if not testname.startswith("soilm"):
393+
cmd = [f"time python3 -m {pyname} {arg} runconfig_{wfname}{suf}.yaml"]
394+
else:
395+
executable = pyname
396+
# Executable-dependent. Currently works only for Disaggregation.
397+
cmd = [f"time {executable} runconfig_{wfname}{suf}.txt"]
398+
383399
try:
384-
self.distribrun(testdir, cmd, logfile=log, dataname=dataname,
385-
loghdlrname=f'wftest.{os.path.basename(testdir)}')
400+
if not testname.startswith("soilm"):
401+
self.distribrun(testdir, cmd, logfile=log, dataname=dataname,
402+
loghdlrname=f'wftest.{os.path.basename(testdir)}')
403+
else:
404+
# Currently, the SM executables are in the nisar image.
405+
self.distribrun(testdir, cmd, logfile=log, dataname=dataname, nisarimg=True,
406+
loghdlrname=f"wftest.{os.path.basename(testdir)}")
386407
except subprocess.CalledProcessError as e:
387408
raise RuntimeError(f"Workflow test {testname} failed") from e
388409

@@ -490,6 +511,24 @@ def beamformtest(self, tests=None):
490511
except subprocess.CalledProcessError as e:
491512
raise RuntimeError(f"CalTool beamformer tool test {testname} failed") from e
492513

514+
def soilmtest(self, tests=None):
515+
if tests is None:
516+
tests = workflowtests['soilm'].items()
517+
for testname, dataname in tests:
518+
# Note: we will eventually have multiple SM executables, each
519+
# of which implements a different algorithm. These executables
520+
# will run the same input test data. It's TBD whether they'll
521+
# be able to share the same runconfig. The output files should
522+
# be either written to different directories by executable or
523+
# should be named to indicate which executable was used, or both.
524+
#
525+
# Also, the current plan is for two of the SM executables to be
526+
# Fortran 90 binaries and the other two to be Python modules.
527+
soilm_bindir = '/opt/conda/envs/SoilMoisture/bin'
528+
executables = [ 'NISAR_SM_DISAGG_SAS' ]
529+
for executable in executables:
530+
self.workflowtest("soilm", testname, dataname, f"{soilm_bindir}/{executable}")
531+
493532
def mintests(self):
494533
"""
495534
Only run first test from each workflow
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
--Folder path to NISAR GCOV data
2+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/NISAR_L2_PR_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004.h5
3+
--Folder path to ECMWF sol moisture
4+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ERA5_SD_TP_ST_SM.nc
5+
--Folder path and file to static ancillary data: EZlat200m
6+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lat200m.float32
7+
--Folder path and file to static ancillary data: EZlon200m
8+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lon200m.float32
9+
--Folder path and file to static ancillary data: EZlat1km
10+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lat01km.float32
11+
--Folder path and file to static ancillary data: EZlon1km
12+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lon01km.float32
13+
--Folder path and file to static ancillary data: EZlat3km
14+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lat03km.float32
15+
--Folder path and file to static ancillary data: EZlon3km
16+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lon03km.float32
17+
--Folder path and file to static ancillary data: EZlat9km
18+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lat09km.float32
19+
--Folder path and file to static ancillary data: EZlon9km
20+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/EASE2_lon09km.float32
21+
--Folder path and file to static ancillary data: Landcover1km
22+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/dominantIGBP_M01_B20120101_003.uint8
23+
--Folder path and file to static ancillary data: WaterFrac1km
24+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/waterfrac_M01_002.float32
25+
--Folder path and file to static ancillary data: WaterFrac3km
26+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/waterfrac_M03_002.float32
27+
--Folder path and file to static ancillary data: BD200m
28+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/BD05cmD_EASE2_200m.nc
29+
--Folder path and file to static ancillary data: Urban1km
30+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/UrbanFraction_M01_001.float32
31+
--Folder path and file to static ancillary data: dem1km slope
32+
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/DEMSLPSTD_M01_002.float32
33+
--Log file name with path
34+
output_soilm/NISAR_SM_20200926_Disagg_ver1.log
35+
--Output HDF5 file name with path
36+
output_soilm/NISAR_SM_20200926_Disagg_ver1.h5

tools/imagesets/workflowdata.py

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,9 +161,29 @@
161161
"dem.tif",
162162
"README.txt",
163163
],
164+
165+
"L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004":
166+
[
167+
"NISAR_L2_PR_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004.h5",
168+
"ERA5_SD_TP_ST_SM.nc",
169+
"ancillary/DEMSLPSTD_M01_002.float32",
170+
"ancillary/dominantIGBP_M01_B20120101_003.uint8",
171+
"ancillary/EASE2_lat01km.float32",
172+
"ancillary/EASE2_lat03km.float32",
173+
"ancillary/EASE2_lat09km.float32",
174+
"ancillary/EASE2_lat200m.float32",
175+
"ancillary/EASE2_lon01km.float32",
176+
"ancillary/EASE2_lon03km.float32",
177+
"ancillary/EASE2_lon09km.float32",
178+
"ancillary/EASE2_lon200m.float32",
179+
"ancillary/UrbanFraction_M01_001.float32",
180+
"ancillary/waterfrac_M01_002.float32",
181+
"ancillary/waterfrac_M03_002.float32",
182+
"README.txt",
183+
],
164184
}
165185

166-
# dictionaries definining mappig of workflow tests to data
186+
# dictionaries definining mapping of workflow tests to data
167187
# each key is the test name, value is the corresponding data
168188
workflowtests = {
169189
'rslc': {"rslc_" + name: "L0B_RRSD_" + name for name in [
@@ -245,4 +265,8 @@
245265
"instrumentTables_bf_19000101.h5", # instrument tables
246266
],
247267
},
268+
269+
'soilm': {"soilm_" + name: "L2_GCOV_" + name for name in [
270+
"001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004",
271+
]},
248272
}

tools/run.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ def run(*, steps, imgset, **kwargs):
2525
"noisesttest",
2626
"ptatest",
2727
"beamformtest",
28+
"soilmtest",
2829
"rslcqa",
2930
"gslcqa",
3031
"gcovqa",

0 commit comments

Comments
 (0)