diff --git a/.circleci/config.yml b/.circleci/config.yml index c763e45a..cd042ffd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,11 +1,11 @@ version: 2.1 # Anchors in case we need to override the defaults from the orb -#baselibs_version: &baselibs_version v7.29.0 -#bcs_version: &bcs_version v11.6.0 +#baselibs_version: &baselibs_version v8.5.0 +#bcs_version: &bcs_version v12.0.0 orbs: - ci: geos-esm/circleci-tools@4 + ci: geos-esm/circleci-tools@5 workflows: build-test: @@ -21,7 +21,10 @@ workflows: #baselibs_version: *baselibs_version repo: GEOSgcm checkout_fixture: true - mepodevelop: true + # V12 code uses a special branch for now. + fixture_branch: feature/sdrabenh/gcm_v12 + # We comment out this as it will "undo" the fixture_branch + #mepodevelop: true persist_workspace: true # Needs to be true to run fv3/gcm experiment, costs extra # Run AMIP GCM (1 hour, no ExtData) @@ -45,7 +48,7 @@ workflows: - docker-hub-creds matrix: parameters: - compiler: [gfortran, ifort] + compiler: [ifort] requires: - build-GEOSgcm-on-<< matrix.compiler >> repo: GEOSgcm diff --git a/CHANGELOG.md b/CHANGELOG.md index ee03615e..12e9f7d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added choice 'blank' for qos in remap_restart python pacakage +- Added hydrostatic question in remap_restart python pacakage +- Added function to remap from stretched restarts to stretched restarts +- Added remapping for GEOS-IT restarts +- Added new res C1120 +- NOTE: If running on SLES15 remap tests will not be zero diff for GOCART RST but are zero diff for all other +- Add support for v13 BCs + ### Changed ### Fixed +- Fixed issue with SLES15 in remapping + ### Removed +- Removed detection of OS at NCCS as it is all SLES15 now + ### Deprecated ## [2.1.7] - 2025-04-23 diff --git a/coupled_diagnostics/g5lib/confocn.py b/coupled_diagnostics/g5lib/confocn.py index 89f6ada6..4484a422 100644 --- a/coupled_diagnostics/g5lib/confocn.py +++ b/coupled_diagnostics/g5lib/confocn.py @@ -1,17 +1,17 @@ ''' -@EXPDSC +{{ EXPDSC }} ### Put detailed experiment description here ### ''' import os from datetime import date -expid='@EXPID' -cmpexp='@EXPID' -data_path='@EXPDIR' +expid='{{ EXPID }}' +cmpexp='{{ EXPID }}' +data_path='{{ EXPDIR }}' plot_path=data_path+'/plots_ocn' -basin_mask='@COUPLEDIR/basin_mask/@OGCM_IMx@OGCM_JM/basin_mask.nc' -grid_spec='@COUPLEDIR/a@AGCM_IMx@AGCM_JM_o@OGCM_IMx@OGCM_JM/INPUT/grid_spec.nc' +basin_mask='{{ COUPLEDIR }}/basin_mask/{{ OGCM_IM }}x{{ OGCM_JM }}/basin_mask.nc' +grid_spec='{{ COUPLEDIR }}/a{{ AGCM_IM }}x{{ AGCM_JM }}_o{{ OGCM_IM }}x{{ OGCM_JM }}/INPUT/grid_spec.nc' start_year=1980 end_year=1981 diff --git a/coupled_diagnostics/g5lib/plotocn.j b/coupled_diagnostics/g5lib/plotocn.j index 1c4fd213..ce9b3db8 100755 --- a/coupled_diagnostics/g5lib/plotocn.j +++ b/coupled_diagnostics/g5lib/plotocn.j @@ -1,5 +1,5 @@ #!/bin/csh -#SBATCH --job-name=@PLOT_T +#SBATCH --job-name={{ PLOT_T }} #SBATCH --ntasks=20 #SBATCH --time=12:00:00 #SBATCH -o plotocn.out @@ -7,7 +7,7 @@ source /usr/share/modules/init/csh module purge module load python/GEOSpyD/Ana2019.10_py2.7 -set SCRDIR=@GEOSDIR/coupled_diagnostics +set SCRDIR={{ GEOSDIR }}/coupled_diagnostics set G5LIBDIR=$SCRDIR set ANADIR=$SCRDIR/analysis set VERIFICATION=$SCRDIR/verification @@ -18,9 +18,9 @@ else if ( ($HOSTNAME =~ "pfe"*) || ($HOSTNAME =~ r[0-9]*i[0-9]*n[0-9]*) || ($HOS setenv OCEANVAL /nobackup/gmao_SIteam/ModelData/oceanval/verification endif -set EXPID=@EXPID -set HOMDIR=@HOMDIR -set EXPDIR=@EXPDIR +set EXPID={{ EXPID }} +set HOMDIR={{ HOMDIR }} +set EXPDIR={{ EXPDIR }} set WORKDIR=$EXPDIR/plot setenv PYTHONPATH ${G5LIBDIR}:${VERIFICATION}:${HOMDIR}/.. diff --git a/post/stats.F90 b/post/stats.F90 index 70d9de3b..7232faab 100644 --- a/post/stats.F90 +++ b/post/stats.F90 @@ -1053,8 +1053,16 @@ end subroutine init_levs zlev(lev).eq.750 .or. zlev(lev).eq.600 .or. & zlev(lev).eq.500 .or. zlev(lev).eq.400 .or. & zlev(lev).eq.300 .or. zlev(lev).eq.200 .or. & - zlev(lev).eq.100 .or. zlev(lev).eq.70 .or. & - zlev(lev).eq.30 .or. zlev(lev).eq.10 ) then + zlev(lev).eq.100 .or. zlev(lev).eq.90 .or. & + zlev(lev).eq.80 .or. zlev(lev).eq.70 .or. & + zlev(lev).eq.60 .or. zlev(lev).eq.50 .or. & + zlev(lev).eq.40 .or. zlev(lev).eq.30 .or. & + zlev(lev).eq.20 .or. zlev(lev).eq.10 .or. & + zlev(lev).eq.9 .or. zlev(lev).eq.8 .or. & + zlev(lev).eq.7 .or. zlev(lev).eq.6 .or. & + zlev(lev).eq.5 .or. zlev(lev).eq.4 .or. & + zlev(lev).eq.3 .or. zlev(lev).eq.2 .or. & + zlev(lev).eq.1 ) then write(6,1005) int(zlev(lev)),trim(regions(iregion)),& trim(fields_3d(n)%name),nymd,nhms,hour,& rms(iregion,lev,nfield,nt,1),& diff --git a/pre/NSIDC-OSTIA_SST-ICE_blend/regrid_forcing_esmf.F90 b/pre/NSIDC-OSTIA_SST-ICE_blend/regrid_forcing_esmf.F90 index 6b6ae02b..c42716a8 100644 --- a/pre/NSIDC-OSTIA_SST-ICE_blend/regrid_forcing_esmf.F90 +++ b/pre/NSIDC-OSTIA_SST-ICE_blend/regrid_forcing_esmf.F90 @@ -31,6 +31,10 @@ module GenESMFGridCompMod integer :: PJM integer :: GPIM integer :: GPJM + integer :: OIM + integer :: OJM + integer :: GOIM + integer :: GOJM type(ESMF_Time) :: start_time type(ESMF_Time) :: end_time logical :: select_time @@ -277,6 +281,15 @@ subroutine Initialize ( GC, IMPORT, EXPORT, CLOCK, RC ) PrivateState%ogrid=ogrid PrivateState%pgrid=pgrid +! Query Ogrid to save IM and JM + call MAPL_GridGet(ogrid, localCellCountPerDim=COUNTS, & + globalCellCountPerDim=dims, RC=STATUS) + VERIFY_(STATUS) + PrivateState%oim=counts(1) + PrivateState%ojm=counts(2) + PrivateState%goim=dims(1) + PrivateState%gojm=dims(2) + ! Query Pgrid to save IM and JM call MAPL_GridGet(pgrid, localCellCountPerDim=COUNTS, & globalCellCountPerDim=dims, RC=STATUS) @@ -346,6 +359,8 @@ subroutine RUN ( GC, IMPORT, EXPORT, CLOCK, RC ) integer :: IM, JM integer :: IM_WORLD, JM_WORLD + integer :: OIM, OJM + integer :: OIM_WORLD, OJM_WORLD type(ESMF_Grid) :: ogrid, pgrid type(ESMF_VM) :: vm type(ESMF_Time) :: currentTime, dateN @@ -360,6 +375,7 @@ subroutine RUN ( GC, IMPORT, EXPORT, CLOCK, RC ) type (MAPL_MetaComp), pointer :: MAPL logical :: amIRoot,dowrite type(ESMF_Time) :: start_interval, end_interval + real :: gmax, gmin !============================================================================= @@ -405,6 +421,11 @@ subroutine RUN ( GC, IMPORT, EXPORT, CLOCK, RC ) UNIT_W = getfile(filename) + OIM = PrivateState%oim + OJM = PrivateState%ojm + OIM_WORLD = PrivateState%goim + OJM_WORLD = PrivateState%gojm + IM = PrivateState%pim JM = PrivateState%pjm IM_WORLD = PrivateState%gpim @@ -484,13 +505,17 @@ subroutine RUN ( GC, IMPORT, EXPORT, CLOCK, RC ) ! read(unit_r) odata call MAPL_VarRead(unit_r, grid=ogrid, a=odata, rc=status) VERIFY_(STATUS) - ! transform data from ocean (tripolar or Reynolds) to mit-cubed + ! Check global max/min + call MAPL_MaxMin('In data: ', odata) + ! transform data from ocean (tripolar or Reynolds) to cubed call privateState%regridder%regrid(odata,pdata,rc=status) VERIFY_(status) if (privateState%fix_fraction) then where(pdata > 1.0) pdata = 1.0 where(pdata < 0.0) pdata = 0.0 end if + ! Check global max/min + call MAPL_MaxMin('Out data: ', pdata) ! write(unit_w) pdata if (doWrite) then call MAPL_VarWrite(unit_w, grid=pgrid, a=pdata, rc=status) diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 33a3cd30..0ec9247f 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -124,9 +124,11 @@ def remap(self): NPE = 160 PARTITION ='' - QOS = config['slurm_pbs']['qos'] + QOS ='' + + qos = config['slurm_pbs']['qos'] TIME = "1:00:00" - if QOS != "debug": TIME="12:00:00" + if qos != "debug": TIME="3:00:00" NNODE = '' job = '' @@ -134,6 +136,8 @@ def remap(self): job = "PBS" CONSTRAINT = 'cas_ait' NNODE = (NPE-1)//40 + 1 + if (qos != ''): + QOS = "#PBS -q "+qos else: job = "SLURM" partition = config['slurm_pbs']['partition'] @@ -141,8 +145,11 @@ def remap(self): PARTITION = "#SBATCH --partition=" + partition CONSTRAINT = '"[cas|mil]"' + if (qos != ''): + QOS = "#SBATCH --qos="+qos account = config['slurm_pbs']['account'] + # even if the (MERRA-2) input restarts are binary, the output restarts will always be nc4 (remap_bin2nc.py) suffix = '_rst.' + suffix out_rstfile = expid + os.path.basename(in_rstfile).split('_rst')[0].split('.')[-1]+suffix @@ -364,7 +371,7 @@ def has_catch_rst(text): "type": "text", "name": "slurm_pbs:qos", "message": message_qos, - "default": "debug", + "default": "", }, { diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 8a8ea85b..76677d16 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # # remap_restarts package: -# remap_command_line.py parses and converts the command-line arguments and converts the information +# remap_command_line.py parses and converts the command-line arguments and converts the information # into matching "Answers" for the questionary (remap_questions.py) # import os @@ -64,10 +64,11 @@ def parse_args(program_description): p_command.add_argument('-ocnmdlout', default='data', help='Ocean model for new restarts', choices=choices_omodel) p_command.add_argument('-in_stretch', default=False, help='Stretched CS params of input restarts', choices=choices_stretch) p_command.add_argument('-out_stretch', default=False, help='Stretched CS params for new restarts', choices=choices_stretch) - + # Unlike remap_questions.py, command-line feature does not deduce Catch vs. CatchCN[40,45] for simplicity, thus requires input argument p_command.add_argument('-catch_model',default='catch', help='Catchment[CN] model', choices=choices_catchmodel) + p_command.add_argument('-nonhydrostatic', action='store_true', help=" non hydrostatic upper air") p_command.add_argument('-nobkg', action='store_true', help="Do not remap bkg files") p_command.add_argument('-nolcv', action='store_true', help="Do not write lcv file") p_command.add_argument('-np', action='store_true', help="No prompt. Overwrite config files without prompting questions") @@ -77,10 +78,11 @@ def parse_args(program_description): p_command.add_argument('-out_bc_base',default="", help="Boundary conditions base dir (w/o bc_version and resolution info) for new restarts") p_command.add_argument('-zoom', help= "Zoom parameter (search radius) for input surface restarts") - p_command.add_argument('-qos', default="debug", help="slurm_pbs quality-of-service", choices=['debug', 'allnccs', 'normal']) + p_command.add_argument('-qos', default="", help="slurm_pbs quality-of-service", choices=['', 'debug', 'allnccs', 'normal']) account = get_account() p_command.add_argument('-account', default=account, help="slurm_pbs account") - p_command.add_argument('-partition', default='', help="slurm_pbs partition") + p_command.add_argument('-partition', default='', help="slurm_pbs partition") + p_command.add_argument('-rs', default='3', help="Flag indicating which restarts to regrid: 1 (upper air); 2 (surface); 3 (both)", choices=['1','2','3']) # Parse using parse_known_args so we can pass the rest to the remap scripts @@ -128,6 +130,7 @@ def get_answers_from_command_line(cml): answers["output:shared:stretch"] = cml.out_stretch answers["output:analysis:bkg"] = not cml.nobkg answers["output:analysis:lcv"] = not cml.nolcv + answers["input:air:hydrostatic"] = not cml.nonhydrostatic if cml.rs == '1': answers["output:air:remap"] = True answers["output:surface:remap_water"] = False @@ -143,7 +146,7 @@ def get_answers_from_command_line(cml): answers["output:air:agcm_import_rst"] = not cml.noagcm_import_rst - if cml.zoom: + if cml.zoom: answers["input:surface:zoom"] = cml.zoom else: # zoom_default fills 'input:shared:agrid' @@ -162,7 +165,7 @@ def get_answers_from_command_line(cml): answers["slurm_pbs:account"] = cml.account answers["slurm_pbs:qos"] = cml.qos answers["slurm_pbs:partition"] = cml.partition - + return answers if __name__ == "__main__": @@ -175,7 +178,7 @@ def get_answers_from_command_line(cml): with open("raw_command.yaml", "w") as f: yaml.dump(config, f) - config = get_config_from_answers(answers, config_tpl= True) + config = get_config_from_answers(answers, config_tpl= True) with open("params_from_command.yaml", "w") as f: yaml.dump(config, f) diff --git a/pre/remap_restart/remap_lake_landice_saltwater.py b/pre/remap_restart/remap_lake_landice_saltwater.py index b8b26988..a2d03449 100755 --- a/pre/remap_restart/remap_lake_landice_saltwater.py +++ b/pre/remap_restart/remap_lake_landice_saltwater.py @@ -139,10 +139,12 @@ def remap(self): cmd = exe + out_til + ' ' + in_til + ' InData/'+ saltwater_internal + ' 0 ' + str(zoom) self.run_and_log(cmd, log_name) - # split Saltwater - if config['output']['surface']['split_saltwater']: - print("\nSplitting Saltwater...\n") - cmd = bindir+'/SaltIntSplitter.x ' + out_til + ' ' + 'OutData/' + saltwater + # split Saltwater Internal + # NOTE: split_saltwater==True means that the input restarts are already split. + # So we do not split them again. + if not config['output']['surface']['split_saltwater']: + print("\nSplitting Saltwater Internal...\n") + cmd = bindir+'/SaltIntSplitter.x ' + out_til + ' ' + 'OutData/' + saltwater_internal # subprocess.call(shlex.split(cmd)) openwater = '' seaice = '' @@ -152,6 +154,17 @@ def remap(self): cmd = exe + out_til + ' ' + in_til + ' InData/'+ saltwater_import + ' 0 ' + str(zoom) self.run_and_log(cmd, log_name) + # split Saltwater Import + # NOTE: split_saltwater==True means that the input restarts are already split. + # So we do not split them again. + if not config['output']['surface']['split_saltwater']: + print("\nSplitting Saltwater Import...\n") + cmd = bindir+'/SaltIntSplitter.x ' + out_til + ' ' + 'OutData/' + saltwater_import +# subprocess.call(shlex.split(cmd)) + openwater = '' + seaice = '' + self.run_and_log(cmd, log_name) + if (openwater): cmd = exe + out_til + ' ' + in_til + ' InData/' + openwater + ' 0 ' + str(zoom) self.run_and_log(cmd, log_name) diff --git a/pre/remap_restart/remap_params.tpl b/pre/remap_restart/remap_params.tpl index 7ac89ee8..f7db3b5c 100644 --- a/pre/remap_restart/remap_params.tpl +++ b/pre/remap_restart/remap_params.tpl @@ -7,7 +7,7 @@ input: air: drymass: 1 - hydrostatic: 0 + hydrostatic: true shared: MERRA-2: false GEOS-IT: false @@ -64,5 +64,5 @@ output: lcv: false slurm_pbs: account: - qos: + qos: '' partition: '' diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 84db9492..618e956c 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -1,17 +1,13 @@ #!/usr/bin/env python3 # # remap_restarts package: -# interactive questionary to create a yaml configuration file (remap_params.yaml) and +# interactive questionary to create a yaml configuration file (remap_params.yaml) and # a matching command line argument string (remap_restarts.CMD) # # import os -import subprocess -import shlex import ruamel.yaml -import shutil import questionary -import glob from remap_utils import * def echo_level(x): @@ -36,10 +32,10 @@ def echo_bcs(x,opt): return False def default_partition(x): - if x['slurm_pbs:qos'] == 'debug': - x['slurm_pbs:partition'] = 'compute' - return False - return True + if x['slurm_pbs:qos'] == 'debug': + x['slurm_pbs:partition'] = 'compute' + return False + return True def validate_merra2_time(text): if len(text) == 10 : @@ -47,7 +43,7 @@ def validate_merra2_time(text): if hh in ['03','09','15','21']: return True else: - return False + return False else: return False @@ -72,7 +68,7 @@ def SITE_GEOSIT(x): if GEOS_SITE == "NAS": x['input:shared:GEOS-IT']= False return False - return True + return True def ask_questions(): @@ -121,7 +117,13 @@ def ask_questions(): "validate": lambda text: validate_geosit_time(text), "when": lambda x: x.get("input:shared:GEOS-IT", False) and not x.get("input:shared:MERRA-2", False), }, - + { + "type": "confirm", + "name": "input:air:hydrostatic", + "message": "Is the upper air input hydrostatic? (If you are not sure, don't change the default 'True')\n", + "default": True, + "when": lambda x: not x['input:shared:MERRA-2'], + }, { "type": "path", "name": "output:shared:out_dir", @@ -155,7 +157,7 @@ def ask_questions(): "when": lambda x: not x['input:shared:MERRA-2'] and not x['input:shared:GEOS-IT'] and not fvcore_info(x), }, - { + { "type": "select", "name": "input:shared:omodel", "message": "Select ocean model of input restarts:\n", @@ -164,7 +166,7 @@ def ask_questions(): "when": lambda x: not x['input:shared:MERRA-2'] and not x['input:shared:GEOS-IT'], }, - { + { "type": "select", "name": "input:shared:ogrid", "message": message_ogrid_in, @@ -208,7 +210,7 @@ def ask_questions(): { "type": "select", "name": "output:shared:stretch", - "message": message_stretch, + "message": message_stretch, "choices": choices_stretch[1:3], "when": lambda x : x['output:shared:stretch'], }, @@ -217,7 +219,7 @@ def ask_questions(): "type": "select", "name": "output:shared:agrid", "message": "Select resolution of SG001 grid for new restarts: \n", - "choices": choices_res_SG001, + "choices": choices_res_SG001, "when": lambda x : x.get('output:shared:stretch') == 'SG001', }, @@ -225,7 +227,7 @@ def ask_questions(): "type": "select", "name": "output:shared:agrid", "message": "Select resolution of SG002 grid for new restarts: \n", - "choices": choices_res_SG002, + "choices": choices_res_SG002, "when": lambda x : x.get('output:shared:stretch') == 'SG002', }, @@ -265,7 +267,7 @@ def ask_questions(): "type": "text", "name": "output:air:nlevel", "message": "Enter number of atmospheric levels for new restarts: (71 72 91 127 132 137 144 181)\n", - "default": "72", + "default": "181", }, # to show the message, we ask output first @@ -290,7 +292,7 @@ def ask_questions(): "name": "output:shared:bc_version", "message": message_bc_ops_new, "choices": choices_bc_ops, - "default": "NL3", + "default": "v13", "when": lambda x: x["input:shared:MERRA-2"] or x["input:shared:GEOS-IT"], }, @@ -299,7 +301,7 @@ def ask_questions(): "name": "output:shared:bc_version", "message": "Select BCs version for new restarts:\n", "choices": choices_bc_ops, - "default": "NL3", + "default": "v13", "when": lambda x: not x["input:shared:MERRA-2"] and not x["input:shared:GEOS-IT"], }, @@ -308,7 +310,7 @@ def ask_questions(): "name": "output:shared:bc_version", "message": message_bc_other_new, "choices": choices_bc_other, - "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06','v11','v12'], + "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v06','v11','v12','GM4'], }, { @@ -316,7 +318,7 @@ def ask_questions(): "name": "output:shared:bc_version", "message": "\nSelect BCs version for new restarts:\n", "choices": choices_bc_other, - "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06','v11','v12'], + "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v06','v11','v12','GM4'], }, { @@ -376,8 +378,8 @@ def ask_questions(): { "type": "confirm", "name": "output:air:agcm_import_rst", - "message": f'''Remap agcm_import_rst (a.k.a. IAU) file needed for REPLAY runs? - (NOTE: Preferred method is to regenerate IAU file, + "message": f'''Remap agcm_import_rst (a.k.a. IAU) file needed for REPLAY runs? + (NOTE: Preferred method is to regenerate IAU file, but IF requested, remapping will be performed.)''', "default": False, "when": lambda x: echo_level(x), @@ -453,20 +455,29 @@ def ask_questions(): "name": "slurm_pbs:partition", "message": message_partition, "default": '', + "when": lambda x : default_partition(x), }, ] answers = questionary.prompt(questions) answers['input:shared:rst_dir'] = os.path.abspath(answers['input:shared:rst_dir']) answers['output:shared:out_dir'] = os.path.abspath(answers['output:shared:out_dir']) - if answers.get('input:air:nlevel') : del answers['input:air:nlevel'] - if answers["output:surface:remap"] and not answers["input:shared:MERRA-2"] and not answers["input:shared:GEOS-IT"]: + if answers.get('input:air:nlevel'): + del answers['input:air:nlevel'] + if answers["output:surface:remap"] and not answers["input:shared:MERRA-2"] and not answers["input:shared:GEOS-IT"]: answers["input:surface:catch_model"] = catch_model(answers) answers["output:surface:remap_water"] = answers["output:surface:remap"] answers["output:surface:remap_catch"] = answers["output:surface:remap"] del answers["output:surface:remap"] + if answers["input:shared:MERRA-2"]: + answers["input:air:hydrostatic"] = True + # Due to the order of questions above, if a user asks + # for MERRA2, they will not be asked for GEOS-IT so + # we set to false so the next if-block doesn't run + answers["input:shared:GEOS-IT"] = False + if answers["input:shared:GEOS-IT"]: + answers["input:air:hydrostatic"] = True - return answers if __name__ == "__main__": diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index 6436c076..b7266c4a 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -160,22 +160,27 @@ def remap(self): NPE = 5400; nwrit = 6 PARTITION ='' - QOS = config['slurm_pbs']['qos'] + QOS ='' + qos = config['slurm_pbs']['qos'] TIME = "1:00:00" if NPE > 532: assert config['slurm_pbs']['qos'] != 'debug', "qos should be 'allnccs' for NCCS or 'normal' for NAS" - TIME = "12:00:00" + TIME = "3:00:00" NNODE = '' job='' if GEOS_SITE == "NAS": CONSTRAINT = 'cas_ait' NNODE = (NPE-1)//40 + 1 job='PBS' + if (qos != ''): + QOS = "#PBS -q "+qos else: job='SLURM' partition = config['slurm_pbs']['partition'] if (partition != ''): PARTITION = "#SBATCH --partition=" + partition + if (qos != ''): + QOS = "#SBATCH --qos="+qos CONSTRAINT = '"[cas|mil]"' @@ -184,7 +189,7 @@ def remap(self): # to interp_restarts.x. Per the code we use: # -stretched_grid target_lon target_lat stretch_fac # If we are not running stretched grid, we should pass in a blank string - stretch = config['output']['shared']['stretch'] + stretch = config['input']['shared']['stretch'] stretch_str = "" if stretch: if stretch == 'SG001': @@ -200,7 +205,24 @@ def remap(self): # note "reversed" order of args (relative to order in definition of STRETCH_GRID) - stretch_str = "-stretched_grid " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac) + stretch_str = "-stretched_grid_in " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac) + + stretch = config['output']['shared']['stretch'] + if stretch: + if stretch == 'SG001': + stretch_fac = STRETCH_GRID['SG001'][0] + target_lat = STRETCH_GRID['SG001'][1] + target_lon = STRETCH_GRID['SG001'][2] + elif stretch == 'SG002': + stretch_fac = STRETCH_GRID['SG002'][0] + target_lat = STRETCH_GRID['SG002'][1] + target_lon = STRETCH_GRID['SG002'][2] + else: + exit("This stretched grid option is not supported " + str(stretch)) + + # note "reversed" order of args (relative to order in definition of STRETCH_GRID) + + stretch_str = stretch_str+" -stretched_grid_out " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac) # Now, let's create the input.nml file # We need to create a namelist for the upper air remapping @@ -223,21 +245,63 @@ def remap(self): cd {out_dir}/upper_data /bin/touch input.nml -# The MERRA fvcore_internal_restarts don't include W or DZ, but we can add them by setting -# HYDROSTATIC = 0 which means HYDROSTATIC = FALSE +set mpi_type = "openmpi" +if ($?I_MPI_ROOT ) then + set mpi_type = "intel" +endif -if ($?I_MPI_ROOT) then - # intel scaling suggestions - #-------------------------- - setenv I_MPI_ADJUST_ALLREDUCE 12 - setenv I_MPI_ADJUST_GATHERV 3 - setenv I_MPI_SHM_HEAP_VSIZE 512 - setenv PSM2_MEMORY large - setenv I_MPI_EXTRA_FILESYSTEM 1 - setenv I_MPI_EXTRA_FILESYSTEM_FORCE gpfs - setenv ROMIO_FSTYPE_FORCE "gpfs:" +if ($mpi_type =~ "openmpi") then +setenv OMPI_MCA_shmem_mmap_enable_nfs_warning 0 +# pre-connect MPI procs on mpi_init +setenv OMPI_MCA_mpi_preconnect_all 1 +# options fo bcast: 0|ignore, 1|basic_linear, 2|chain, 3|pipeline, 4|split_binary_tree, 5|binary_tree, 6|binomial, 7|knomial, 8|scatter_allgather, 9|scatter_allgather_ring +setenv OMPI_MCA_coll_tuned_bcast_algorithm 7 +# options for scatter: 0|ignore, 1|basic_linear, 2|binomial, 3|linear_nb +setenv OMPI_MCA_coll_tuned_scatter_algorithm 2 +# options for reduce_scatter: 0|ignore, 1|non-overlapping, 2|recursive_halving, 3|ring, 4|butterfly +setenv OMPI_MCA_coll_tuned_reduce_scatter_algorithm 3 +# options for allreduce: 0|ignore, 1|basic_linear, 2|nonoverlapping, 3|recursive_doubling, 4|ring, 5|segmented_ring, 6|rabenseifner +setenv OMPI_MCA_coll_tuned_allreduce_algorithm 3 +# options for allgather: 0|ignore, 1|linear, 2|bruck, 3|recursive_doubling, 4|ring, 5|neighbor, 6|two_proc, 7|sparbit +setenv OMPI_MCA_coll_tuned_allgather_algorithm 4 +# options for allgatherv: 0|ignore, 1|default, 2|bruck, 3|ring, 4|neighbor, 5|two_proc, 6|sparbit +setenv OMPI_MCA_coll_tuned_allgatherv_algorithm 3 +# options for gather: 0 ignore, 1 basic linear, 2 binomial, 3 linear with synchronization +setenv OMPI_MCA_coll_tuned_gather_algorithm 1 +# options for barrier: 0|ignore, 1|linear, 2|double_ring, 3|recursive_doubling, 4|bruck, 5|two_proc, 6|tree +setenv OMPI_MCA_coll_tuned_barrier_algorithm 0 +# required for a tuned flag to be effective +setenv OMPI_MCA_coll_tuned_use_dynamic_rules 1 +# disable file locks +setenv OMPI_MCA_sharedfp "^lockedfile,individual" +## HDF5: disable slow locks (promise not to open half-written files) +#setenv HDF5_USE_FILE_LOCKING FALSE +else +setenv I_MPI_FABRICS ofi +setenv I_MPI_OFI_PROVIDER psm3 +setenv I_MPI_ADJUST_SCATTER 2 +setenv I_MPI_ADJUST_SCATTERV 2 +setenv I_MPI_ADJUST_GATHER 2 +setenv I_MPI_ADJUST_GATHERV 3 +setenv I_MPI_ADJUST_ALLGATHER 3 +setenv I_MPI_ADJUST_ALLGATHERV 3 +setenv I_MPI_ADJUST_ALLREDUCE 12 +setenv I_MPI_ADJUST_REDUCE 10 +setenv I_MPI_ADJUST_BCAST 11 +setenv I_MPI_ADJUST_REDUCE_SCATTER 4 +setenv I_MPI_ADJUST_BARRIER 9 +#setenv I_MPI_SHM_HEAP_VSIZE 512 +#setenv I_MPI_EXTRA_FILESYSTEM 1 +#setenv I_MPI_EXTRA_FILESYSTEM_FORCE "gpfs" +#setenv ROMIO_FSTYPE_FORCE "gpfs:" +#setenv I_MPI_TUNING_MODE auto +#setenv I_MPI_TUNING_BIN_DUMP tuning-results.dat +#setenv I_MPI_DEBUG 6 +#setenv MPS_STAT_LEVEL 4 +env | grep 'I_MPI\|FI_' endif + set infiles = () set outfils = () foreach infile ( *_restart_in ) @@ -260,7 +324,14 @@ def remap(self): set ioflag = "" endif -set drymassFLG = {drymassFLG} +set hydrostaticIN = {hydrostatic} +if ( $hydrostaticIN == 'True' ) then + set hydrostaticflag = "-in_hydrostatic T" +else + set hydrostaticflag = "-in_hydrostatic F" +endif + +set drymassFLG = {drymass} if ($drymassFLG) then set dmflag = "" else @@ -268,20 +339,20 @@ def remap(self): endif {Bin}/esma_mpirun -np {NPE} $interp_restartsX -im {imout} -lm {nlevel} \\ - -do_hydro {hydrostatic} $ioflag $dmflag -nwriter {nwrit} {stretch_str} + $hydrostaticflag {stretch_str} $dmflag -nwriter {nwrit} $ioflag """ account = config['slurm_pbs']['account'] - drymassFLG = config['input']['air']['drymass'] + drymass = config['input']['air']['drymass'] hydrostatic = config['input']['air']['hydrostatic'] nlevel = config['output']['air']['nlevel'] log_name = out_dir+'/remap_upper_log' job_name = 'remap_upper' remap_upper_script = remap_template.format(Bin=bindir, account = account, \ - out_dir = out_dir, log_name = log_name, job_name= job_name, drymassFLG = drymassFLG, \ + out_dir = out_dir, log_name = log_name, job_name= job_name, drymass = drymass, \ imout = imout, nwrit = nwrit, NPE = NPE, NNODE = NNODE, \ - QOS = QOS, TIME = TIME, CONSTRAINT = CONSTRAINT, PARTITION = PARTITION, nlevel = nlevel, hydrostatic = hydrostatic, - stretch_str = stretch_str) + QOS = QOS, TIME = TIME, CONSTRAINT = CONSTRAINT, PARTITION = PARTITION, nlevel = nlevel, \ + hydrostatic = hydrostatic, stretch_str = stretch_str) script_name = './remap_upper.j' @@ -350,7 +421,8 @@ def find_rst(self): if len(files) >0: restarts_in.append(files[0]) if (len(restarts_in) == 0) : - print("\n try restart file names without time stamp\n") + print("\n Try restart file names without time stamp or suffix (e.g., .nc4)\n") + print("\n It expects restart file names as xx_internal_rst, e.g., fvcore_internal_rst \n") for f in self.air_restarts : fname = rst_dir+ '/'+f if os.path.exists(fname): diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 0a803d82..99ab3866 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -28,11 +28,11 @@ # define "choices", "message" strings, and "validate" lists that are used multiple times # (and related definitions, even if they are used just once). -choices_bc_ops = ['NL3', 'ICA', 'GM4', 'Other'] +choices_bc_ops = ['v13', 'NL3', 'ICA', 'Other'] -choices_bc_other = ['v06','v11','v12'] +choices_bc_other = ['v06','v11','v12','GM4'] -choices_bc_cmd = ['NL3', 'ICA', 'GM4', 'v06', 'v11','v12'] +choices_bc_cmd = ['NL3', 'ICA', 'GM4', 'v06', 'v11','v12', 'v13'] choices_omodel = ['data', 'MOM5', 'MOM6'] @@ -78,21 +78,20 @@ message_bc_ops = f'''\n BCs version | ADAS tags | GCM tags typically used with BCs version -----------------|----------------------|----------------------------------------- - GM4: Ganymed-4_0 | 5_12_2 ... 5_16_5 | Ganymed-4_0 ... Heracles-5_4_p3 + v13: v13 | future | 12.0 ... present + NL3: Icarus-NLv3 | 5_25_1 ... present | Icarus_NL, 10.19 ... 11.7 ICA: Icarus | 5_17_0 ... 5_24_0_p1 | Icarus, Jason ... 10.18 - NL3: Icarus-NLv3 | 5_25_1 ... present | Icarus_NL, 10.19 ... present ---------------------------------------------------------------------------------- - Other: Additional choices used in model or DAS development. - \n\n ''' + Other: Additional choices used in model or DAS development.\n\n''' message_bc_ops_in = ("Select boundary conditions (BCs) version of input restarts:\n" + message_bc_ops) message_bc_ops_new = ("Select boundary conditions (BCs) version for new restarts:\n" + message_bc_ops) message_bc_other = f'''\n - - v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb\n - v11: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2\n - v12: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix \n\n'''\ + v06: NL3 + JPL veg height + PEATMAP + MODIS snow alb + v11: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + v12: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix + GM4: Ganymed-4_0\n'''\ message_bc_other_in = ("Select BCs version of input restarts:\n" + message_bc_other) message_bc_other_new = ("Select BCs version for new restarts:\n" + message_bc_other) @@ -113,7 +112,7 @@ message_ogrid_new = "Select data ocean grid/resolution of output restarts:\n" -message_qos = "SLURM or PBS quality-of-service (qos)? (If resolution is c1440 or higher, enter 'allnccs' for NCCS or 'normal' for NAS.)\n" +message_qos = "SLURM or PBS quality-of-service (qos)? (Use default 'debug' to get resource faster; or Enter 'allnccs' for NCCS or 'normal' for NAS if resolution is c1440 or higher; or leave it blank)\n" message_account = "Select/enter SLURM or PBS account:\n" @@ -125,20 +124,20 @@ #SBATCH --ntasks={NPE} #SBATCH --job-name={job_name} #SBATCH --output={log_name} -#SBATCH --qos={QOS} #SBATCH --time={TIME} #SBATCH --constraint={CONSTRAINT} {PARTITION} +{QOS} """, "PBS": """#!/bin/csh -f #PBS -l walltime={TIME} #PBS -l select={NNODE}:ncpus=40:mpiprocs=40:model={CONSTRAINT} #PBS -N {job_name} -#PBS -q {QOS} #PBS -W group_list={account} #PBS -o {log_name} #PBS -j oe {PARTITION} +{QOS} """ } @@ -449,7 +448,7 @@ def get_command_line_from_answers(answers): nobkg = '' if answers["output:analysis:bkg"] else " -nobkg " nolcv = '' if answers["output:analysis:lcv"] else " -nolcv " - + nonhydrostatic = '' if answers["input:air:hydrostatic"] else " -nonhydrostatic " label = ' -lbl ' if answers["output:shared:label"] else "" in_bc_base = ' -in_bc_base ' + answers.get("input:shared:bc_base") @@ -480,7 +479,9 @@ def get_command_line_from_answers(answers): noagcm_import_rst = '' if answers["output:air:agcm_import_rst"] else " -noagcm_import_rst " account = " -account " + answers["slurm_pbs:account"] - qos = " -qos " + answers["slurm_pbs:qos"] + qos = '' + if answers["slurm_pbs:qos"] != '': + qos = " -qos " + answers["slurm_pbs:qos"] partition = '' if answers["slurm_pbs:partition"] != '': partition = " -partition " + answers["slurm_pbs:partition"] @@ -509,6 +510,7 @@ def get_command_line_from_answers(answers): wemout + \ label + \ nobkg + \ + nonhydrostatic + \ noagcm_import_rst + \ nolcv + \ out_rs + \ @@ -650,5 +652,5 @@ def remove_ogrid_comment(x, opt): return False if __name__ == '__main__' : - config = yaml_to_config('c24Toc12.yaml') + config = yaml_to_config('remap_params.tpl') print_config(config)