Skip to content

Commit 4603e30

Browse files
oelbertoelbert
authored
Merge branch 'NOAA-GFDL:develop' into develop
2 parents a4517e0 + 14be4f5 commit 4603e30

File tree

1 file changed

+50
-17
lines changed

1 file changed

+50
-17
lines changed

‎Makefile‎

Lines changed: 50 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
SHELL := /bin/bash
22
include docker/Makefile.image_names
3-
include Makefile.data_download
43
ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
54

5+
6+
67
define BROWSER_PYSCRIPT
78
import os, webbrowser, sys
89

@@ -35,6 +36,19 @@ BUILD_FLAGS ?=
3536

3637
### Testing variables
3738

39+
NUM_RANKS ?=6
40+
MPIRUN_ARGS ?=--oversubscribe --mca btl_vader_single_copy_mechanism none
41+
MPIRUN_CALL ?=mpirun -np $(NUM_RANKS) $(MPIRUN_ARGS)
42+
TEST_ARGS ?=-v
43+
FV3CORE_THRESH_ARGS=--threshold_overrides_file=$(PACE_PATH)/pyFV3/tests/savepoint/translate/overrides/standard.yaml
44+
PHYSICS_THRESH_ARGS=--threshold_overrides_file=$(PACE_PATH)/pySHiELD/tests/savepoint/translate/overrides/standard.yaml
45+
46+
TEST_DATA_LOC ?=test_data/
47+
TEST_DATA_VERSION ?=8.1.3
48+
TEST_DATA_HOST ?= https://portal.nccs.nasa.gov/datashare/astg/smt/pace-regression-data/
49+
TEST_RESOLUTION ?= c12
50+
TEST_CONFIG ?= $(TEST_DATA_LOC)$(TEST_RESOLUTION)_$(NUM_RANKS)ranks
51+
3852
RUN_FLAGS ?=--rm
3953
ifeq ("$(CONTAINER_CMD)","")
4054
PACE_PATH?=$(ROOT_DIR)
@@ -46,31 +60,24 @@ else
4660
endif
4761
endif
4862
ifeq ("$(CONTAINER_CMD)","")
49-
EXPERIMENT_DATA_RUN=$(EXPERIMENT_DATA)
63+
EXPERIMENT_DATA_RUN=$(TEST_CONFIG)
5064
else
5165
ifeq ("$(CONTAINER_CMD)","srun")
52-
EXPERIMENT_DATA_RUN=$(EXPERIMENT_DATA)
66+
EXPERIMENT_DATA_RUN=$(TEST_CONFIG)
5367
else
54-
EXPERIMENT_DATA_RUN=$(PACE_PATH)/test_data/$(FORTRAN_SERIALIZED_DATA_VERSION)/$(EXPERIMENT)
68+
EXPERIMENT_DATA_RUN=$(TEST_CONFIG)
5569
endif
5670
endif
5771
ifeq ($(DEV),y)
5872
VOLUMES += -v $(ROOT_DIR):/pace
5973
else
60-
VOLUMES += -v $(EXPERIMENT_DATA):$(EXPERIMENT_DATA_RUN)
74+
VOLUMES += -v $(EXPERIMENT_DATA_RUN)
6175
endif
6276
ifeq ($(CONTAINER_CMD),docker)
6377
CONTAINER_FLAGS=run $(RUN_FLAGS) $(VOLUMES) --env GT_CACHE_ROOT=/pace/.gt_cache $(PACE_IMAGE)
6478
else
6579
CONTAINER_FLAGS=
6680
endif
67-
NUM_RANKS ?=6
68-
MPIRUN_ARGS ?=--oversubscribe --mca btl_vader_single_copy_mechanism none
69-
MPIRUN_CALL ?=mpirun -np $(NUM_RANKS) $(MPIRUN_ARGS)
70-
TEST_ARGS ?=-v
71-
TEST_TYPE=$(word 3, $(subst _, ,$(EXPERIMENT)))
72-
FV3CORE_THRESH_ARGS=--threshold_overrides_file=$(PACE_PATH)/fv3core/tests/savepoint/translate/overrides/$(TEST_TYPE).yaml
73-
PHYSICS_THRESH_ARGS=--threshold_overrides_file=$(PACE_PATH)/physics/tests/savepoint/translate/overrides/$(TEST_TYPE).yaml
7481

7582
###
7683

@@ -108,29 +115,55 @@ notebook:
108115
DEV=y \
109116
$(MAKE) enter
110117

118+
get_standard_test_data:
119+
if [ ! -d $(TEST_DATA_LOC) ]; then \
120+
mkdir -p $(TEST_DATA_LOC); \
121+
fi ; \
122+
if [ ! -f $(TEST_CONFIG)_standard/dycore/input.nml ] ; then \
123+
wget $(TEST_DATA_HOST)/8.1.3_c12_6ranks_standard.tar.gz; \
124+
tar -xzvf $(ROOT_DIR)/8.1.3_c12_6ranks_standard.tar.gz; \
125+
mv $(ROOT_DIR)/8.1.3/* $(TEST_DATA_LOC); \
126+
rm -rf 8.1.3*; \
127+
fi
128+
129+
get_physics_test_data:
130+
if [ ! -d $(TEST_DATA_LOC) ]; then \
131+
mkdir -p $(TEST_DATA_LOC); \
132+
fi ; \
133+
if [ ! -f $(TEST_CONFIG)_baroclinic/physics/input.nml ] ; then \
134+
wget $(TEST_DATA_HOST)/8.1.3_c12_6ranks_baroclinic.physics.tar.gz ; \
135+
tar -xzvf $(ROOT_DIR)/8.1.3_c12_6ranks_baroclinic.physics.tar.gz; \
136+
mv $(ROOT_DIR)/8.1.3/* $(TEST_DATA_LOC); \
137+
rm -rf 8.1.3*; \
138+
fi
139+
140+
get_test_data:
141+
$(MAKE) get_standard_test_data; \
142+
$(MAKE) get_physics_test_data
143+
111144
test_util:
112145
if [ $(shell $(CHECK_CHANGED_SCRIPT) util) != false ]; then \
113146
$(MAKE) -C util test; \
114147
fi
115148

116149
savepoint_tests: build ## dycore-only savepoint tests
117150
TARGET=dycore $(MAKE) get_test_data
118-
$(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && pytest --data_path=$(EXPERIMENT_DATA_RUN)/dycore/ $(TEST_ARGS) $(FV3CORE_THRESH_ARGS) $(PACE_PATH)/fv3core/tests/savepoint"
151+
$(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && pytest --data_path=$(TEST_DATA_LOC)/standard/dycore/ $(TEST_ARGS) $(FV3CORE_THRESH_ARGS) $(PACE_PATH)/pyFV3/tests/savepoint"
119152

120153
savepoint_tests_mpi: build
121154
TARGET=dycore $(MAKE) get_test_data
122-
$(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python3 -m mpi4py -m pytest --maxfail=1 --data_path=$(EXPERIMENT_DATA_RUN)/dycore/ $(TEST_ARGS) $(FV3CORE_THRESH_ARGS) -m parallel $(PACE_PATH)/fv3core/tests/savepoint"
155+
$(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python3 -m mpi4py -m pytest --maxfail=1 --data_path=$(TEST_DATA_LOC)/dycore/ $(TEST_ARGS) $(FV3CORE_THRESH_ARGS) -m parallel $(PACE_PATH)/pyFV3/tests/savepoint"
123156

124157
dependencies.svg: dependencies.dot
125158
dot -Tsvg $< -o $@
126159

127160
physics_savepoint_tests: build
128161
TARGET=physics $(MAKE) get_test_data
129-
$(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && pytest --data_path=$(EXPERIMENT_DATA_RUN)/physics/ $(TEST_ARGS) $(PHYSICS_THRESH_ARGS) $(PACE_PATH)/physics/tests/savepoint"
162+
$(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && pytest --data_path=$(EXPERIMENT_DATA_RUN)/physics/ $(TEST_ARGS) $(PHYSICS_THRESH_ARGS) $(PACE_PATH)/pySHiELD/tests/savepoint"
130163

131164
physics_savepoint_tests_mpi: build
132165
TARGET=physics $(MAKE) get_test_data
133-
$(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python -m mpi4py -m pytest --maxfail=1 --data_path=$(EXPERIMENT_DATA_RUN)/physics/ $(TEST_ARGS) $(PHYSICS_THRESH_ARGS) -m parallel $(PACE_PATH)/physics/tests/savepoint"
166+
$(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python -m mpi4py -m pytest --maxfail=1 --data_path=$(EXPERIMENT_DATA_RUN)/physics/ $(TEST_ARGS) $(PHYSICS_THRESH_ARGS) -m parallel $(PACE_PATH)/pySHiELD/tests/savepoint"
134167

135168
test_main: build
136169
$(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && pytest $(TEST_ARGS) $(PACE_PATH)/tests/main"
@@ -147,7 +180,7 @@ test_mpi_54rank:
147180

148181
driver_savepoint_tests_mpi: build
149182
TARGET=pace $(MAKE) get_test_data
150-
$(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python -m mpi4py -m pytest --maxfail=1 --data_path=$(EXPERIMENT_DATA_RUN)/pace/ $(TEST_ARGS) $(PHYSICS_THRESH_ARGS) -m parallel $(PACE_PATH)/physics/tests/savepoint"
183+
$(CONTAINER_CMD) $(CONTAINER_FLAGS) bash -c "$(SAVEPOINT_SETUP) && cd $(PACE_PATH) && $(MPIRUN_CALL) python -m mpi4py -m pytest --maxfail=1 --data_path=$(EXPERIMENT_DATA_RUN)/physics/ $(TEST_ARGS) $(PHYSICS_THRESH_ARGS) -m parallel $(PACE_PATH)/pySHiELD/tests/savepoint"
151184

152185
docs: ## generate Sphinx HTML documentation
153186
$(MAKE) -C docs html

0 commit comments

Comments
 (0)