Merge branch 'main' into bugfix/movestat-fix

Author: sdrabenh

CHANGELOG.md

@@ -11,12 +11,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Remapping seaicetherom_import_rst and saltwater_import_rst if they are in the restart directory
+
 ### Fixed
 
 - Fixed issue with `movestat` which only allowed single-month seasons
 
 ### Removed
 
+- Removed detection of OS at NCCS as it is all SLES15 now
+
 ### Deprecated
 
 ## [2.1.6] - 2025-01-23

pre/remap_restart/remap_catchANDcn.py

@@ -44,8 +44,8 @@ def remap(self):
         in_rstfiles = glob.glob(rst_dir+'/*'+model+'_*')
      if len(in_rstfiles) == 0:
         return
-     in_rstfile = in_rstfiles[0]  
-             
+     in_rstfile = in_rstfiles[0]
+
 
      cwdir          = os.getcwd()
      bindir         = os.path.dirname(os.path.realpath(__file__))
@@ -60,8 +60,8 @@ def remap(self):
      surflay        = config['output']['surface']['surflay']
      in_tilefile    = config['input']['surface']['catch_tilefile']
 
-     if "gmao_SIteam/ModelData" in out_bc_base: 
-        assert  GEOS_SITE == "NAS", "wrong site to run the package" 
+     if "gmao_SIteam/ModelData" in out_bc_base:
+        assert  GEOS_SITE == "NAS", "wrong site to run the package"
 
      if not in_tilefile :
         agrid        = config['input']['shared']['agrid']
@@ -86,7 +86,7 @@ def remap(self):
 
      out_bc_landdir = get_landdir(out_bc_base, out_bc_version, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch, grid=EASE_grid)
 
-     label = get_label(config) 
+     label = get_label(config)
 
      suffix     = time+'z.nc4' + label
 
@@ -100,22 +100,22 @@ def remap(self):
 
      print("\nRemapping " + model + ".....\n")
  # determine NPE based on *approximate* number of input and output tile
-      
+
      in_Ntile  = 0
      mime = mimetypes.guess_type(in_rstfile)
      if mime[0] and 'stream' in mime[0]: # binary
        in_Ntile  = 1684725 # if it is binary, it is safe to assume the maximum is M09
      else : # nc4 file
        ds = nc.Dataset(in_rstfile)
        in_Ntile = ds.dimensions['tile'].size
-       
+
      out_Ntile = 0
      with open( out_bc_landdir+'/clsm/catchment.def') as f:
        out_Ntile = int(next(f))
      max_Ntile = max(in_Ntile, out_Ntile)
      NPE = 0
      if   (max_Ntile <=  112573) : # no more than EASEv2_M36
-        NPE = 40      
+        NPE = 40
      elif (max_Ntile <= 1684725) : # no more than EASEv2_M09
         NPE = 80
      elif (max_Ntile <= 2496756) : # no more than C720
@@ -140,9 +140,7 @@ def remap(self):
        if (partition != ''):
          PARTITION = "#SBATCH --partition=" + partition
 
-       CONSTRAINT = '"[cas|sky]"'
-       if BUILT_ON_SLES15:
-         CONSTRAINT = 'mil'
+       CONSTRAINT = '"[cas|mil]"'
 
      account    = config['slurm_pbs']['account']
      # even if the (MERRA-2) input restarts are binary, the output restarts will always be nc4 (remap_bin2nc.py)
@@ -156,7 +154,7 @@ def remap(self):
      InData_dir = out_dir+'/InData/'
      print ("mkdir -p " + InData_dir)
      os.makedirs(InData_dir, exist_ok = True)
-    
+
      f = os.path.basename(in_rstfile)
      dest = InData_dir+'/'+f
      # file got copy because the computing node cannot access archive
@@ -176,8 +174,8 @@ def remap(self):
 
 set params = ( -model {model}  -time {time} -in_tilefile {in_tilefile} )
 set params = ( $params -out_bcs {out_bcs} -out_tilefile {out_tilefile} -out_dir {out_dir} )
-set params = ( $params -surflay {surflay} -in_wemin {in_wemin} -out_wemin {out_wemin} ) 
-set params = ( $params -in_rst {in_rstfile} -out_rst {out_rstfile} ) 
+set params = ( $params -surflay {surflay} -in_wemin {in_wemin} -out_wemin {out_wemin} )
+set params = ( $params -in_rst {in_rstfile} -out_rst {out_rstfile} )
 $esma_mpirun_X $mk_catchANDcnRestarts_X $params
 
 """
@@ -256,7 +254,7 @@ def ask_catch_questions():
    catch_input_shared_rst_dir = ''
    def has_rs_rc_out(path):
      if os.path.exists(path):
-        dirs = os.listdir(path)   
+        dirs = os.listdir(path)
         if 'rs' in dirs and 'rc_out' in dirs:
            nonlocal catch_input_shared_rst_dir
            catch_input_shared_rst_dir = path
@@ -273,7 +271,7 @@ def has_catch_rst(text):
      rst_dir   = catch_input_shared_rst_dir + '/rs/ens0000/Y'+yyyy +'/M'+mm+'/'
      rst_files = glob.glob(rst_dir+'*catch*_internal_rst.'+yyyy+mm+dd+'_'+hh+'00')
      if len(rst_files) !=1 :
-       return False 
+       return False
      return True
 
    questions =[
@@ -290,19 +288,19 @@ def has_catch_rst(text):
             "message": (message_datetime + " and rst file must exist.)\n"),
             "validate": lambda text: has_catch_rst(text)
         },
-        
+
         {
             "type": "path",
             "name": "output:shared:out_dir",
-            "message": message_out_dir, 
+            "message": message_out_dir,
         },
 
         {
             "type": "text",
             "name": "output:shared:expid",
             "message": message_expid,
             "default": "",
-        },  
+        },
 
         {
             "type": "select",
@@ -319,15 +317,15 @@ def has_catch_rst(text):
             "choices": choices_bc_ops,
             "default": "NL3",
         },
-        {   
+        {
             "type": "select",
             "name": "output:shared:bc_version",
             "message": message_bc_other_new,
             "choices": choices_bc_other,
             "when": lambda x:  x["output:shared:bc_version"] == 'Other',
         },
 
-        {   
+        {
             "type": "select",
             "name": "output:surface:EASE_grid",
             "message": "Select grid for new restart:\n",
@@ -380,29 +378,29 @@ def has_catch_rst(text):
             "name": "slurm_pbs:partition",
             "message": message_partition,
             "default": '',
-        },  
+        },
 
    ]
 
-   answers = questionary.prompt(questions)   
+   answers = questionary.prompt(questions)
    yyyy = answers['input:shared:yyyymmddhh'][0:4]
    mm   = answers['input:shared:yyyymmddhh'][4:6]
    dd   = answers['input:shared:yyyymmddhh'][6:8]
    hh   = answers['input:shared:yyyymmddhh'][8:10]
-  
+
    rst_dir      = answers['input:shared:rst_dir']+'/rs/ens0000/Y'+yyyy +'/M'+mm+'/'
    rst_file     = glob.glob(rst_dir+'*catch*_internal_rst.'+yyyy+mm+dd+'_'+hh+'00')[0]
    idx1         = rst_file.find('catch')
    idx2         = rst_file.find('_internal_rst')
    catch_model  = rst_file[idx1:idx2]
-   
+
    tile_dir     = answers['input:shared:rst_dir']+'/rc_out/'
    in_tilefiles = glob.glob(tile_dir+'*.til')
    for file in in_tilefiles:
       answers['input:surface:catch_tilefile'] = file
       if 'MAPL_' in file:
          answers['input:surface:catch_tilefile'] = file
-       
+
    answers['input:shared:rst_dir'] = rst_dir
    answers['input:surface:catch_model'] =  catch_model
    answers['input:surface:wemin']  = "13"

pre/remap_restart/remap_lake_landice_saltwater.py

@@ -96,8 +96,10 @@ def remap(self):
      os.makedirs(OutData_dir)
 
 
-     saltwater = ''
-     seaice    = ''
+     saltwater_internal    = ''
+     saltwater_import      = ''
+     seaicethermo_internal = ''
+     seaicethermo_import   = ''
      landice   = ''
      lake      = ''
      route     = ''
@@ -108,12 +110,14 @@ def remap(self):
         if os.path.exists(dest) : shutil.remove(dest)
         print('\nCopy ' + rst + ' to ' +dest)
         shutil.copy(rst,dest)
-        if 'saltwater' in f : saltwater = f
-        if 'seaice'    in f : seaice    = f
-        if 'landice'   in f : landice   = f
-        if 'lake'      in f : lake      = f
-        if 'roue'      in f : route     = f
-        if 'openwater' in f : openwater = f
+        if 'saltwater_internal'     in f : saltwater_internal    = f
+        if 'saltwater_import'       in f : saltwater_import      = f
+        if 'seaicethermo_internal'  in f : seaicethermo_internal = f
+        if 'seaicethermo_import'    in f : seaicethermo_import   = f
+        if 'landice'                in f : landice   = f
+        if 'lake'                   in f : lake      = f
+        if 'roue'                   in f : route     = f
+        if 'openwater'              in f : openwater = f
 
      in_til  = InData_dir+'/' + os.path.basename(in_tile_file)
      out_til = OutData_dir+'/'+ os.path.basename(out_tile_file)
@@ -131,8 +135,8 @@ def remap(self):
      if os.path.exists(log_name):
         os.remove(log_name)
 
-     if (saltwater):
-       cmd = exe + out_til + ' ' + in_til + ' InData/'+ saltwater + ' 0 ' + str(zoom)
+     if (saltwater_internal):
+       cmd = exe + out_til + ' ' + in_til + ' InData/'+ saltwater_internal + ' 0 ' + str(zoom)
        self.run_and_log(cmd, log_name)
 
        # split Saltwater
@@ -144,12 +148,21 @@ def remap(self):
          seaice  = ''
          self.run_and_log(cmd, log_name)
 
+     if (saltwater_import):
+       cmd = exe + out_til + ' ' + in_til + ' InData/'+ saltwater_import + ' 0 ' + str(zoom)
+       self.run_and_log(cmd, log_name)
+
      if (openwater):
        cmd = exe + out_til + ' ' + in_til + ' InData/' + openwater + ' 0 ' + str(zoom)
        self.run_and_log(cmd, log_name)
 
-     if (seaice):
-       cmd = exe + out_til + ' ' + in_til + ' InData/' + seaice + ' 0 ' + str(zoom)
+     if (seaicethermo_internal):
+       cmd = exe + out_til + ' ' + in_til + ' InData/' + seaicethermo_internal + ' 0 ' + str(zoom)
+       print('\n'+cmd)
+       self.run_and_log(cmd, log_name)
+
+     if (seaicethermo_import):
+       cmd = exe + out_til + ' ' + in_til + ' InData/' + seaicethermo_import + ' 0 ' + str(zoom)
        print('\n'+cmd)
        self.run_and_log(cmd, log_name)
 
@@ -197,7 +210,9 @@ def find_rst(self):
                  "landice_internal_rst"     ,
                  "openwater_internal_rst"   ,
                  "saltwater_internal_rst"   ,
-                 "seaicethermo_internal_rst"]
+                 "saltwater_import_rst"   ,
+                 "seaicethermo_internal_rst",
+                 "seaicethermo_import_rst"]
 
      rst_dir = self.config['input']['shared']['rst_dir']
      yyyymmddhh_ = str(self.config['input']['shared']['yyyymmddhh'])

pre/remap_restart/remap_upper.py

@@ -78,9 +78,9 @@ def remap(self):
      label = get_label(config)
      suffix = yyyymmddhh_[0:8]+'_'+yyyymmddhh_[8:10] +'z' + types + label
 
-     in_bc_base     = config['input']['shared']['bc_base'] 
+     in_bc_base     = config['input']['shared']['bc_base']
      if "gmao_SIteam/ModelData" in in_bc_base:
-        assert  GEOS_SITE == "NAS", "wrong site to run the package" 
+        assert  GEOS_SITE == "NAS", "wrong site to run the package"
 
      in_bc_version  = config['input']['shared']['bc_version']
      agrid       = config['input']['shared']['agrid']
@@ -90,7 +90,7 @@ def remap(self):
      topo_bcsdir = get_topodir(in_bc_base, in_bc_version,  agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch)
      topoin = glob.glob(topo_bcsdir+'/topo_DYN_ave*.data')[0]
 
-     out_bc_base    = config['output']['shared']['bc_base'] 
+     out_bc_base    = config['output']['shared']['bc_base']
      out_bc_version = config['output']['shared']['bc_version']
      agrid       = config['output']['shared']['agrid']
      ogrid       = config['output']['shared']['ogrid']
@@ -125,7 +125,7 @@ def remap(self):
        cmd = '/bin/ln -s  ' + rst + ' ' + f
        print('\n'+cmd)
        subprocess.call(shlex.split(cmd))
- 
+
      # link topo file
 
      cmd = '/bin/ln -s ' + topoin + ' .'
@@ -162,7 +162,7 @@ def remap(self):
      PARTITION =''
      QOS  = config['slurm_pbs']['qos']
      TIME = "1:00:00"
-     if NPE > 532: 
+     if NPE > 532:
         assert config['slurm_pbs']['qos'] != 'debug', "qos should be 'allnccs' for NCCS or 'normal' for NAS"
         TIME = "12:00:00"
      NNODE = ''
@@ -177,9 +177,7 @@ def remap(self):
        if (partition != ''):
          PARTITION = "#SBATCH --partition=" + partition
 
-       CONSTRAINT = '"[cas|sky]"'
-       if BUILT_ON_SLES15:
-         CONSTRAINT = 'mil'
+       CONSTRAINT = '"[cas|mil]"'
 
      # We need to create an input.nml file which is different if we are running stretched grid
      # If we are running stretched grid, we need to pass in the target lon+lat and stretch factor
@@ -200,7 +198,7 @@ def remap(self):
         else:
           exit("This stretched grid option is not supported " + str(stretch))
 
-        # note "reversed" order of args (relative to order in definition of STRETCH_GRID)  
+        # note "reversed" order of args (relative to order in definition of STRETCH_GRID)
 
         stretch_str = "-stretched_grid " + str(target_lon) + " " + str(target_lat) + " " + str(stretch_fac)
 

pre/remap_restart/remap_utils.py

@@ -17,14 +17,8 @@
 
 #During cmake step, the string will be changed according to the system
 
-BUILT_ON_SLES15 = "@BUILT_ON_SLES15@"
 GEOS_SITE       = "@GEOS_SITE@"
 
-if BUILT_ON_SLES15 == "TRUE":
-    BUILT_ON_SLES15 = True
-else:
-    BUILT_ON_SLES15 = False
-
 # top-level directory for BCs (machine-dependent)
 
 choices_bc_base  =[ "NCCS/Discover : /discover/nobackup/projects/gmao/bcs_shared/fvInput/ExtData/esm/tiles",