Adaption to EWC setup: (#42)

1. use ECTRANS_ID=none to save the output data in the local directory $MSSDIR
   instead of transfer by ectrans

2. define pythonpath and include  local (miniforge) environment path into $PATH

3. allow cleanup of old data, e.g. older than 2 dyas to avoid disk overflow

4. add specific label $LABEL to the output files

5. replace ml.drop (out-dated) by ml.drop_vars in add_anillery.py

Author: jgrooss

bin/add_ancillary.py

@@ -273,7 +273,7 @@ def main():
             ml["pv"].attrs["units"] = VARIABLES["pv"][1]
             ml["pv"].attrs["standard_name"] = VARIABLES["pv"][2]
         finally:
-            ml = ml.drop("metpy_crs")
+            ml = ml.drop_vars("metpy_crs")
     if option.n2:
         print("Adding N2...")
         try:

bin/convert.sh

@@ -2,13 +2,13 @@
 #Copyright (C) 2021 by Forschungszentrum Juelich GmbH
 #Author(s): Joern Ungermann, May Baer
 
-export mlfile=mss/${BASE}.ml.nc
-export plfile=mss/${BASE}.pl.nc
-export alfile=mss/${BASE}.al.nc
-export tlfile=mss/${BASE}.tl.nc
-export pvfile=mss/${BASE}.pv.nc
-export sfcfile=mss/${BASE}.sfc.nc
-export tmpfile=mss/.${BASE}.tmp
+export mlfile=mss/${BASE}.${LABEL}ml.nc
+export plfile=mss/${BASE}.${LABEL}pl.nc
+export alfile=mss/${BASE}.${LABEL}al.nc
+export tlfile=mss/${BASE}.${LABEL}tl.nc
+export pvfile=mss/${BASE}.${LABEL}pv.nc
+export sfcfile=mss/${BASE}.${LABEL}sfc.nc
+export tmpfile=mss/.${BASE}.${LABEL}tmp
 
 if [ ! -f grib/${BASE}.ml.grib ]; then
    echo FATAL `date` Model level file is missing
@@ -32,9 +32,9 @@ if [[ x$PV_LEVELS != x"" ]]; then
 fi
 
 echo adding gph
-ls
+
 $PYTHON $BINDIR/compute_geopotential_on_ml.py grib/${BASE}.ml.grib grib/${BASE}.ml2.grib -o ${tmpfile}
-ls
+
 cdo -f nc4c -t ecmwf copy ${tmpfile} ${tmpfile}_z
 ncatted -O \
     -a standard_name,z,o,c,geopotential_height \

bin/get_ecmwf.sh

@@ -91,6 +91,30 @@ export time_units="hours since ${init_date}"
 # Convert grib to netCDF, set init time
 . $BINDIR/convert.sh
 
+if [ $ECTRANS_ID == "none" ]
+then
+    echo "no ectrans transfer -- move data to " $MSSDIR
+  if [ x$TRANSFER_MODEL_LEVELS == x"yes" ]; then
+      mv $mlfile $MSSDIR
+  fi
+  if [ -f $tlfile ]; then
+      mv $tlfile $MSSDIR
+  fi
+  if [ -f $plfile ]; then
+      mv $plfile $MSSDIR
+  fi
+  if [ -f $pvfile ]; then
+      mv $pvfile $MSSDIR
+
+  fi
+  if [ -f $alfile ]; then
+      mv $alfile $MSSDIR
+  fi
+  if [ -f $sfcfile ]; then
+      mv $sfcfile $MSSDIR
+  fi
+else  
+
 if ecaccess-association-list | grep -q $ECTRANS_ID; then
   echo "Transfering files to "$ECTRANS_ID 
   if [ x$TRANSFER_MODEL_LEVELS == x"yes" ]; then
@@ -112,15 +136,30 @@ if ecaccess-association-list | grep -q $ECTRANS_ID; then
       ectrans -remote $ECTRANS_ID -source $sfcfile -target $sfcfile -overwrite -remove
   fi
 fi
-
+fi
 if [[ x$CLEANUP == x"yes" ]]
 then
+  export CYMD=${CLEANUP_YEAR}${CLEANUP_MONTH}${CLEANUP_DAY}
+  export CBASE=${DATASET}.${CYMD}T${HH}.${FCSTEP}
+  echo cleanup $CBASE
+    
   # clean up locally
-  for f in $mlfile $tlfile $plfile $pvfile $alfile $sfcfile grib/${BASE}*.grib;
+  for f in $mlfile $tlfile $plfile $pvfile $alfile $sfcfile grib/${CBASE}*.grib;
   do
       if [ -f $f ];
       then
           rm $f
       fi
   done
+  if [ $ECTRANS_ID == "none" ]
+  then
+    # clean up MSS server dir
+    for f in $MSSDIR/${CBASE}*.nc
+    do
+	if [ -f $f ];
+	then
+            rm $f
+	fi
+    done
+  fi
 fi

settings.example

@@ -1,16 +1,22 @@
 # Do not leave any whitespaces
 # variable=value
 
+export PATH=$HOME/miniforge3/envs/retrieval/bin/:$PATH
+
 # write data to the $SCRATCH directory with more available disk quota
 export WORKDIR=$HOME/data-retrieval
-
-export PYTHON=python3
-export CLEANUP=no
+export MSSDIR=/mswms-data/mssdata/ecmwf
+export PYTHON=$HOME/miniforge3/envs/retrieval/bin/python
 
 # configure area, grid, and transfer
 export AREA=75/-15/30/42.5
 export GRID=1.0/1.0
-export ECTRANS_ID=MSS-Data-Transfer
+# Set label for output files
+export LABEL=
+
+# set ECTRANS_ID to "none" if write to local $MSSDIR
+# export ECTRANS_ID=MSS-Data-Transfer
+export ECTRANS_ID=none
 
 # set init and forecast times for testing
 # (default 36h from today 00:00) 
@@ -22,3 +28,7 @@ export HH=00
 export FCSTEP=036
 export STEP=0/to/36/by/6
 
+export CLEANUP=yes
+export CLEANUP_DAY=`date --date=-2days +%d`
+export CLEANUP_MONTH=`date --date=-2days +%m`
+export CLEANUP_YEAR=`date --date=-2days +%Y`