Skip to content

Commit f6409bf

Browse files
jshimada47jshimada47
committed
Implemented changes as per @gunter's review: cleanup of Dockerfile SM
commands, renaming of workflow test 'smtest' to 'soilmtest', cleanup of imgset.workflowtest().
1 parent 54ea2dd commit f6409bf

File tree

7 files changed

+72
-22
lines changed

7 files changed

+72
-22
lines changed

‎tools/imagesets/centos7conda/distrib_nisar/Dockerfile‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ RUN conda install testfixtures scikit-image
2323
RUN conda install cfunits --channel conda-forge
2424

2525
# Soil Moisture
26-
RUN conda create -n SoilMoisture gcc gfortran netcdf-cxx4 netcdf-fortran make --channel conda-forge
26+
COPY spec-file.txt /tmp/spec-file.txt
27+
RUN conda create -n SoilMoisture --file /tmp/spec-file.txt && conda clean -ay
2728

2829
# copy the repo from the intermediate image
2930
COPY --from=0 /opt/QualityAssurance /opt/QualityAssurance
@@ -32,11 +33,7 @@ COPY --from=0 /opt/calTools /opt/calTools
3233
COPY --from=0 /opt/SoilMoisture /opt/SoilMoisture
3334

3435
# install
35-
RUN cd /opt/QualityAssurance \
36-
&& python setup.py install \
37-
&& cd /opt/CFChecker \
38-
&& python setup.py install \
39-
&& cd /opt/calTools \
40-
&& python setup.py install \
41-
&& cd /opt/SoilMoisture \
42-
&& env CONDA_PREFIX=/opt/conda/envs/SoilMoisture PATH=/opt/conda/bin:/opt/conda/envs/SoilMoisture/bin:/usr/bin make install
36+
RUN cd /opt/QualityAssurance && python setup.py install
37+
RUN cd /opt/CFChecker && python setup.py install
38+
RUN cd /opt/calTools && python setup.py install
39+
RUN cd /opt/SoilMoisture && conda run -n SoilMoisture make install

‎tools/imagesets/centos7conda/distrib_nisar/requirements.soil-moisture.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
gcc
2+
gfortran
3+
make
4+
netcdf-cxx4
5+
netcdf-fortran

‎tools/imagesets/centos7conda/distrib_nisar/spec-file.txt‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# This file may be used to create an environment using:
2+
# $ conda create --name <env> --file <this file>
3+
# platform: linux-64
4+
@EXPLICIT
5+
https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2
6+
https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2021.10.8-ha878542_0.tar.bz2
7+
https://conda.anaconda.org/conda-forge/noarch/kernel-headers_linux-64-2.6.32-he073ed8_15.tar.bz2
8+
https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.36.1-hea4e1c9_2.tar.bz2
9+
https://conda.anaconda.org/conda-forge/linux-64/libgcc-devel_linux-64-11.2.0-h0952999_12.tar.bz2
10+
https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-11.2.0-h5c6108e_12.tar.bz2
11+
https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-11.2.0-he4da1e4_12.tar.bz2
12+
https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-11.2.0-h69a702a_12.tar.bz2
13+
https://conda.anaconda.org/conda-forge/linux-64/libgomp-11.2.0-h1d223b6_12.tar.bz2
14+
https://conda.anaconda.org/conda-forge/noarch/sysroot_linux-64-2.12-he073ed8_15.tar.bz2
15+
https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-1_gnu.tar.bz2
16+
https://conda.anaconda.org/conda-forge/linux-64/binutils_impl_linux-64-2.36.1-h193b22a_2.tar.bz2
17+
https://conda.anaconda.org/conda-forge/linux-64/binutils_linux-64-2.36-hf3e587d_4.tar.bz2
18+
https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-11.2.0-h1d223b6_12.tar.bz2
19+
https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h7f98852_4.tar.bz2
20+
https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.18.1-h7f98852_0.tar.bz2
21+
https://conda.anaconda.org/conda-forge/linux-64/jpeg-9e-h7f98852_0.tar.bz2
22+
https://conda.anaconda.org/conda-forge/linux-64/libev-4.33-h516909a_1.tar.bz2
23+
https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-11.2.0-he4da1e4_12.tar.bz2
24+
https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.11-h36c2ea0_1013.tar.bz2
25+
https://conda.anaconda.org/conda-forge/linux-64/make-4.3-hd18ef5c_1.tar.bz2
26+
https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.3-h9c3ff4c_0.tar.bz2
27+
https://conda.anaconda.org/conda-forge/linux-64/openssl-3.0.0-h7f98852_2.tar.bz2
28+
https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-11.2.0-h82a94d6_12.tar.bz2
29+
https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20191231-he28a2e2_2.tar.bz2
30+
https://conda.anaconda.org/conda-forge/linux-64/zlib-1.2.11-h36c2ea0_1013.tar.bz2
31+
https://conda.anaconda.org/conda-forge/linux-64/gcc-11.2.0-h702ea55_4.tar.bz2
32+
https://conda.anaconda.org/conda-forge/linux-64/gcc_linux-64-11.2.0-h39a9532_4.tar.bz2
33+
https://conda.anaconda.org/conda-forge/linux-64/gfortran_impl_linux-64-11.2.0-h7a446d4_12.tar.bz2
34+
https://conda.anaconda.org/conda-forge/linux-64/hdf4-4.2.15-h10796ff_3.tar.bz2
35+
https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.46.0-ha19adfc_0.tar.bz2
36+
https://conda.anaconda.org/conda-forge/linux-64/libssh2-1.10.0-ha35d2d1_2.tar.bz2
37+
https://conda.anaconda.org/conda-forge/linux-64/libzip-1.8.0-h1c5bbd1_1.tar.bz2
38+
https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.11-h27826a3_1.tar.bz2
39+
https://conda.anaconda.org/conda-forge/linux-64/gfortran_linux-64-11.2.0-h777b47f_4.tar.bz2
40+
https://conda.anaconda.org/conda-forge/linux-64/krb5-1.19.2-h48eae69_3.tar.bz2
41+
https://conda.anaconda.org/conda-forge/linux-64/gfortran-11.2.0-h8811e0c_4.tar.bz2
42+
https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.81.0-h494985f_0.tar.bz2
43+
https://conda.anaconda.org/conda-forge/linux-64/curl-7.81.0-h494985f_0.tar.bz2
44+
https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.1-nompi_h7f166f4_103.tar.bz2
45+
https://conda.anaconda.org/conda-forge/linux-64/libnetcdf-4.8.1-nompi_hb3fd0d9_101.tar.bz2
46+
https://conda.anaconda.org/conda-forge/linux-64/netcdf-cxx4-4.3.1-nompi_h8a6a9d2_105.tar.bz2
47+
https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.5.4-nompi_h2b6e579_100.tar.bz2

‎tools/imagesets/imgset.py‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -348,7 +348,9 @@ def workflowtest(self, wfname, testname, dataname, pyname, suf="", description="
348348
Test input dataset(s) to be mounted (e.g. "L0B_RRSD_REE1", ["L0B_RRSD_REE1", "L0B_RRSD_REE2"]).
349349
If None, no input datasets are used.
350350
pyname : str
351-
Name of the isce3 module to execute (e.g. "nisar.workflows.focus")
351+
Name of the isce3 module to execute (e.g. "nisar.workflows.focus") or,
352+
for Soil Moisture (SM) testing, the name of the SAS executable to run
353+
(e.g. "NISAR_SM_DISAGG_SAS")
352354
suf: str
353355
Suffix in runconfig and output directory name to differentiate between
354356
reference and secondary data in end-to-end tests
@@ -380,14 +382,14 @@ def workflowtest(self, wfname, testname, dataname, pyname, suf="", description="
380382
pjoin(testdir, f"runconfig_{wfname}{suf}.yaml"))
381383
log = pjoin(testdir, f"output_{wfname}{suf}", "stdouterr.log")
382384

383-
if not testname.startswith("sm"):
385+
if not testname.startswith("soilm"):
384386
cmd = [f"time python3 -m {pyname} {arg} runconfig_{wfname}{suf}.yaml"]
385387
else:
386388
executable = pyname
387389
cmd = [f"time {executable} runconfig_{wfname}{suf}.yaml"]
388390

389391
try:
390-
if not testname.startswith("sm"):
392+
if not testname.startswith("soilm"):
391393
self.distribrun(testdir, cmd, logfile=log, dataname=dataname,
392394
loghdlrname=f'wftest.{os.path.basename(testdir)}')
393395
else:
@@ -501,9 +503,9 @@ def beamformtest(self, tests=None):
501503
except subprocess.CalledProcessError as e:
502504
raise RuntimeError(f"CalTool beamformer tool test {testname} failed") from e
503505

504-
def smtest(self, tests=None):
506+
def soilmtest(self, tests=None):
505507
if tests is None:
506-
tests = workflowtests['sm'].items()
508+
tests = workflowtests['soilm'].items()
507509
for testname, dataname in tests:
508510
# Note: we will eventually have multiple SM executables, each
509511
# of which implements a different algorithm. These executables
@@ -514,11 +516,10 @@ def smtest(self, tests=None):
514516
#
515517
# Also, the current plan is for two of the SM executables to be
516518
# Fortran 90 binaries and the other two to be Python modules.
517-
sm_bindir = '/opt/conda/envs/SoilMoisture/bin'
519+
soilm_bindir = '/opt/conda/envs/SoilMoisture/bin'
518520
executables = [ 'NISAR_SM_DISAGG_SAS' ]
519-
cmd = [ ]
520521
for executable in executables:
521-
self.workflowtest("sm", testname, dataname, f"{sm_bindir}/{executable}")
522+
self.workflowtest("soilm", testname, dataname, f"{soilm_bindir}/{executable}")
522523

523524
def mintests(self):
524525
"""

‎tools/imagesets/runconfigs/soilm_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,6 @@ input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_0
3131
--Folder path and file to static ancillary data: dem1km slope
3232
input_L2_GCOV_001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004/ancillary/DEMSLPSTD_M01_002.float32
3333
--Log file name with path
34-
output_sm/NISAR_SM_20200926_Disagg_ver1.log
34+
output_soilm/NISAR_SM_20200926_Disagg_ver1.log
3535
--Output HDF5 file name with path
36-
output_sm/NISAR_SM_20200926_Disagg_ver1.h5
36+
output_soilm/NISAR_SM_20200926_Disagg_ver1.h5

‎tools/imagesets/workflowdata.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,7 @@
266266
],
267267
},
268268

269-
'sm': {"sm_" + name: "L2_GCOV_" + name for name in [
270-
"001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004",
269+
'soilm': {"soilm_" + name: "L2_GCOV_" + name for name in [
270+
"001_005_A_219_4020_HH_20200926T135152_20200926T135219_P01101_M_P_001-004",
271271
]},
272272
}

‎tools/run.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ def run(*, steps, imgset, **kwargs):
2525
"noisesttest",
2626
"ptatest",
2727
"beamformtest",
28-
"smtest",
28+
"soilmtest",
2929
"rslcqa",
3030
"gslcqa",
3131
"gcovqa",

0 commit comments

Comments
 (0)