Skip to content

Commit f1e382c

Browse files
gmao-rreichlegmao-rreichle
authored
refactored ldas_setup (#107)
2 parents 55528a5 + 5fe4072 commit f1e382c

File tree

7 files changed

+1853
-1913
lines changed

7 files changed

+1853
-1913
lines changed

‎CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1111

1212
### Added
1313

14+
- Added ntasks-per-node
15+
1416
### Changed
1517

18+
- Cleaned up ldas_setup. Split it to ldas.py and setup_utils.py,
19+
1620
### Fixed
1721

22+
- Fixed Restart = 1 when the domain is not global
23+
1824
### Removed
1925

2026
### Deprecated

‎GEOSldas_App/CMakeLists.txt

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ ecbuild_add_executable (
2020
LIBS GEOSlandassim_GridComp)
2121

2222
set (scripts
23+
ldas_setup
24+
setup_utils.py
2325
process_hist.csh
2426
ens_forcing/average_ensemble_forcing.py
2527
ens_forcing/ensemble_forc.py
@@ -34,7 +36,7 @@ install (
3436
DESTINATION bin
3537
)
3638

37-
set(file ldas_setup)
39+
set(file ldas.py)
3840
configure_file(${file} ${file} @ONLY)
3941
install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${file} DESTINATION bin)
4042

‎GEOSldas_App/GEOSldas_LDAS.rc

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
1-
####################################################################################
2-
# #
3-
# GEOSldas Resource Parameters #
4-
# #
5-
# Values below override the hardcoded default values #
6-
# in *.F90 calls to MAPL_GetResource(). #
7-
# #
8-
# Users can further override the values below by #
9-
# editing the "exeinp" file during ldas setup. #
10-
# #
11-
####################################################################################
1+
###################################################################################
2+
# #
3+
# GEOSldas Resource Parameters #
4+
# #
5+
# Values below override the hardcoded default values #
6+
# in *.F90 calls to MAPL_GetResource(). #
7+
# #
8+
# Users can further override the values below by #
9+
# editing the "exeinp" input file for ldas_setup. #
10+
# #
11+
###################################################################################
1212

1313

1414
# ---- Using Catchment[CN] offline?

‎GEOSldas_App/ldas.py

Lines changed: 1501 additions & 0 deletions
Large diffs are not rendered by default.

‎GEOSldas_App/ldas_setup

Lines changed: 19 additions & 1901 deletions
Large diffs are not rendered by default.

‎GEOSldas_App/lenkf_j_template.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#SBATCH --job-name={MY_JOB}
1717
#SBATCH --qos={MY_QOS}
1818
#SBATCH --constraint={MY_CONSTRAINT}
19+
{MY_NODES}
1920
'''
2021
,
2122
"NAS": '''#!/bin/csh -f

‎GEOSldas_App/setup_utils.py

Lines changed: 312 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,312 @@
1+
#!/usr/bin/env python3
2+
3+
import os
4+
import sys
5+
6+
from collections import OrderedDict
7+
from datetime import timedelta
8+
9+
10+
def generate_echo(inpfile, ladas_cpl = 0):
11+
"""
12+
Echo generator of inpfile, ignore line starts with "## "
13+
"""
14+
if ladas_cpl == 0 :
15+
use_rc_defaults = 'GEOSldas=>' # use defaults for LDAS
16+
else :
17+
use_rc_defaults = 'GEOSagcm=>' # use defaults for AGCM
18+
19+
with open (inpfile) as fin :
20+
for line in fin:
21+
if use_rc_defaults in line:
22+
line = line.split(use_rc_defaults)[1]
23+
yield line
24+
25+
def parseInputFile(inpfile, ladas_cpl=0):
26+
"""
27+
Parse the input file and return a dict of options
28+
Inpfile : input file
29+
Output : dict, if inpfile does not exist, return empty {}
30+
"""
31+
if not os.path.exists(inpfile):
32+
assert ladas_cpl > 0, " No exeinput file only if ladas_cpl > 0"
33+
return {}
34+
35+
inpdict = OrderedDict()
36+
errstr = "line [%d] of [%s] is not in the form 'key: value'"
37+
38+
echoLines = generate_echo(inpfile, ladas_cpl=ladas_cpl)
39+
40+
linenum = 0
41+
for line in echoLines:
42+
line = line.strip()
43+
linenum += 1
44+
if not line:
45+
continue
46+
# handle comments
47+
position = line.find('#')
48+
if position==0: # comment line
49+
continue
50+
if position>0: # strip out comment
51+
line = line[:position]
52+
# we expect a line to be of the form
53+
# key : value
54+
assert ':' in line, errstr % (linenum, inpfile)
55+
56+
key, val = line.split(':',1)
57+
key = key.strip()
58+
val = val.strip()
59+
if not key or not val:
60+
print ("WARNING: " + errstr % (linenum, inpfile))
61+
continue
62+
#raise Exception(errstr % (linenum, inpfile))
63+
if key in inpdict:
64+
raise Exception('Duplicate key [%s] in [%s]' % (key, inpfile))
65+
inpdict[key] = val.strip()
66+
return inpdict
67+
68+
def echoInputFile(inpfile, ladas_cpl = 0):
69+
"""
70+
Echo inpfile, ignore line starts with "## "
71+
"""
72+
echoLines = generate_echo(inpfile, ladas_cpl=ladas_cpl)
73+
74+
for line in echoLines:
75+
sys.stdout.write(line)
76+
sys.stdout.flush()
77+
78+
def printExeInputSampleFile():
79+
"""
80+
Print sample exeinp file to screen
81+
"""
82+
print ('####################################################################################')
83+
print ('# #')
84+
print ('# REQUIRED INPUTS #')
85+
print ('# #')
86+
print ('# The inputs to ldas_setup can be provided in an "exeinp" file such as this #')
87+
print ('# sample file. #')
88+
print ('# When ldas_setup is called by fvsetup to set up an experiment with the coupled #')
89+
print ('# land-atm data assimilation system, the inputs to ldas_setup are provided as #')
90+
print ('# optional command line arguments. #')
91+
print ('# #')
92+
print ('####################################################################################')
93+
print ()
94+
print ('############################################################')
95+
print ('# #')
96+
print ('# EXPERIMENT INFO #')
97+
print ('# #')
98+
print ('# Format for start/end times is yyyymmdd hhmmss. #')
99+
print ('# #')
100+
print ('############################################################')
101+
print ()
102+
print ('EXP_ID:')
103+
print ('EXP_DOMAIN:')
104+
print ('NUM_LDAS_ENSEMBLE:')
105+
print ('BEG_DATE:')
106+
print ('END_DATE:')
107+
print ()
108+
print ('############################################################')
109+
print ('# #')
110+
print ('# RESTART INFO #')
111+
print ('# #')
112+
print ('# (i) Select "RESTART" option: #')
113+
print ('# #')
114+
print ('# Use one of the following options if you *have* a #')
115+
print ('# GEOSldas restart file: #')
116+
print ('# #')
117+
print ('# RESTART: 1 #')
118+
print ('# YES, have restart file from GEOSldas #')
119+
print ('# in SAME tile space (grid) with SAME boundary #')
120+
print ('# conditions and SAME snow model parameter (WEMIN). #')
121+
print ('# The restart domain can be for the same or #')
122+
print ('# a larger one. #')
123+
print ('# #')
124+
print ('# RESTART: 2 #')
125+
print ('# YES, have restart file from GEOSldas but #')
126+
print ('# in a DIFFERENT tile space (grid) or with #')
127+
print ('# DIFFERENT boundary conditions or DIFFERENT snow #')
128+
print ('# model parameter (WEMIN). #')
129+
print ('# Restart *must* be for the GLOBAL domain. #')
130+
print ('# #')
131+
print ('# Use one of the following options if you DO NOT have a #')
132+
print ('# GEOSldas restart file #')
133+
print ('# (works for global domain ONLY!): #')
134+
print ('# #')
135+
print ('# RESTART: 0 #')
136+
print ('# Cold start from some old restart for Jan 1, 0z. #')
137+
print ('# #')
138+
print ('# RESTART: M #')
139+
print ('# Re-tile from archived MERRA-2 restart file. #')
140+
print ('# #')
141+
print ('# -------------------------------------------------------- #')
142+
print ('# IMPORTANT: #')
143+
print ('# Except for RESTART=1, SPIN-UP is REQUIRED in almost #')
144+
print ('# all cases. #')
145+
print ('# -------------------------------------------------------- #')
146+
print ('# #')
147+
print ('# #')
148+
print ('# (ii) Specify experiment ID/location of restart file: #')
149+
print ('# #')
150+
print ('# For RESTART=1 or RESTART=2: #')
151+
print ('# Specify RESTART_ID, RESTART_PATH, RESTART_DOMAIN with #')
152+
print ('# restarts stored as follows: #')
153+
print ('# RESTART_PATH/RESTART_ID/output/RESTART_DOMAIN/rs/ #')
154+
print ('# #')
155+
print ('# For RESTART=0 or RESTART=M: #')
156+
print ('# There is no need to specify RESTART_ID, RESTART_PATH, #')
157+
print ('# and RESTART_DOMAIN. #')
158+
print ('# #')
159+
print ('############################################################')
160+
print ()
161+
print ('RESTART:')
162+
print ('#RESTART_ID:')
163+
print ('#RESTART_PATH:')
164+
print ('#RESTART_DOMAIN:')
165+
print ()
166+
print ('############################################################')
167+
print ('# #')
168+
print ('# SURFACE METEOROLOGICAL FORCING #')
169+
print ('# #')
170+
print ('# Surface meteorological forcing time step is in seconds. #')
171+
print ('# #')
172+
print ('# NOTE: #')
173+
print ('# When forcing is on cube-sphere (CS) grid, must use: #')
174+
print ('# - Model tile space (BCS) derived from same CS grid. #')
175+
print ('# - Nearest-neighbor interpolation (MET_HINTERP: 0). #')
176+
print ('# #')
177+
print ('# For more information, see: #')
178+
print ('# GEOSldas/doc/README.MetForcing_and_BCS.md #')
179+
print ('# #')
180+
print ('############################################################')
181+
print ()
182+
print ('MET_TAG:')
183+
print ('MET_PATH:')
184+
print ('FORCE_DTSTEP:')
185+
print ()
186+
print ('############################################################')
187+
print ('# #')
188+
print ('# LAND BOUNDARY CONDITIONS (BCS) #')
189+
print ('# #')
190+
print ('# Path to and (atmospheric) resolution of BCS. #')
191+
print ('# Path includes BCS_VERSION. #')
192+
print ('# #')
193+
print ('# For more information, see: #')
194+
print ('# GEOSldas/doc/README.MetForcing_and_BCS.md #')
195+
print ('# [..]/GEOSsurface_GridComp/Utils/Raster/make_bcs #')
196+
print ('# #')
197+
print ('############################################################')
198+
print ()
199+
print ('BCS_PATH:')
200+
print ('BCS_RESOLUTION:')
201+
print ()
202+
print ('############################################################')
203+
204+
current_directory = os.path.dirname(__file__) # path where ldas_setup is
205+
# rc template files are in [current_directory]/../etc/
206+
# add defaults from GEOSldas_LDAS.rc
207+
_fn = current_directory+'/../etc/GEOSldas_LDAS.rc'
208+
echoInputFile(_fn)
209+
_fn = current_directory+'/../etc/GEOS_SurfaceGridComp.rc'
210+
echoInputFile(_fn)
211+
212+
213+
def getExeKeys(option):
214+
# required keys for exe input file depending on restart option
215+
rqdExeInpKeys = {'1' : ['EXP_ID', 'EXP_DOMAIN', 'NUM_LDAS_ENSEMBLE', 'BEG_DATE',
216+
'END_DATE', 'RESTART_PATH', 'RESTART_DOMAIN', 'RESTART_ID',
217+
'MET_TAG', 'MET_PATH', 'FORCE_DTSTEP', 'BCS_PATH',
218+
'BCS_RESOLUTION'],
219+
'0' : ['EXP_ID', 'EXP_DOMAIN', 'NUM_LDAS_ENSEMBLE', 'BEG_DATE',
220+
'END_DATE',
221+
'MET_TAG', 'MET_PATH', 'FORCE_DTSTEP', 'BCS_PATH',
222+
'BCS_RESOLUTION']
223+
}
224+
225+
assert option == '0' or option == '1', '"%s" option is not recognized ' % option
226+
return rqdExeInpKeys[option]
227+
228+
def getResourceKeys(option):
229+
# ------
230+
# Required resource manager input fields
231+
# ------
232+
RmInpKeys ={'required' : ['account', 'walltime', 'ntasks_model'],
233+
'optional' : ['job_name', 'qos', 'oserver_nodes', 'writers-per-node', 'ntasks-per-node', 'constraint']
234+
}
235+
assert option in ['required', 'optional'],' "%s" option is not supported' % option
236+
237+
return RmInpKeys[option]
238+
239+
240+
def printResourceInputSampleFile():
241+
"""
242+
print sample resource manager input file
243+
"""
244+
requiredKeys = getResourceKeys('required')
245+
optionalKeys = getResourceKeys('optional')
246+
247+
print ('#')
248+
print ('# REQUIRED inputs')
249+
print ('#')
250+
print ('# NOTE:')
251+
print ('# - account = computational project number')
252+
print ('# [At NCCS: Use command "getsponsor" to see available account number(s).]' )
253+
print ('# - walltime = walltime requested; format is HH:MM:SS (hours/minutes/seconds)')
254+
print ('# - ntasks_model = number of processors requested for the model (typically 126; output server is not included)')
255+
print ('#')
256+
for key in requiredKeys:
257+
print (key + ':')
258+
print ()
259+
print ('#')
260+
print ('# OPTIONAL inputs')
261+
print ('#')
262+
print ('# NOTE:')
263+
print ('# - job_name = name of experiment; default is "exp_id"')
264+
print ('# - qos = quality-of-service; do not specify by default; specify "debug" for faster but limited service')
265+
print ('# - oserver_nodes = number of nodes for oserver ( default is 0, for future use )')
266+
print ('# - writers-per-node = tasks per oserver_node for writing ( default is 5, for future use );')
267+
print ('# IMPORTANT REQUIREMENT: total #writers = writers-per-node * oserver_nodes >= 2;')
268+
print ('# jobs will hang when oserver_nodes = writers-per-node = 1.')
269+
print ('# - ntasks-per-node = requesting fewer ntasks-per-node than total number of cores per node increases allocated memory;')
270+
print ('# ntasks_model should be a multiple of ntasks-per-node')
271+
print ('# - constraint = name of chip set(s) (NCCS default is "[mil|cas]", NAS default is "cas_ait")')
272+
print ('#')
273+
for key in optionalKeys:
274+
print ('#'+key + ':')
275+
276+
def printInputSampleFile(cmdLineArgs):
277+
'''
278+
./ldas_setup sample ...
279+
280+
"sample" sub-command:
281+
'--exeinp' and '--batinp' are mutually exclusive command line arguments.
282+
Specifying one will set it to True and set the other one to False.
283+
That is, we can have either: {'exeinp': False, 'batinp': True }
284+
or: {'exeinp': True, 'batinp': False}
285+
'''
286+
if cmdLineArgs['exeinp']:
287+
printExeInputSampleFile()
288+
elif cmdLineArgs['batinp']:
289+
printResourceInputSampleFile()
290+
else:
291+
raise Exception('unrecognized sample option')
292+
293+
def printDictionary(d):
294+
"""
295+
Private method: print a 'flat' dictionary
296+
"""
297+
298+
for key, val in d.items():
299+
print (key.ljust(24), ':', val)
300+
301+
def hours_to_hhmmss(hours):
302+
303+
# Convert hours to timedelta
304+
td = timedelta(hours=hours)
305+
306+
# Extract hours, minutes, seconds
307+
total_seconds = int(td.total_seconds())
308+
hours, remainder = divmod(total_seconds, 3600)
309+
minutes, seconds = divmod(remainder, 60)
310+
311+
# Format as HHMMSS
312+
return f"{hours:02d}{minutes:02d}{seconds:02d}"

0 commit comments

Comments
 (0)