Inclide trigger starting conversion jobs using the aviso system

Author: jgrooss

aviso/aviso_readme.txt

@@ -0,0 +1,29 @@
+Use of aviso to trigger job scripts on the European Weather Cloud (EWC)
+=======================================================================
+
+
+* The aviso service runs automatically on EWC machines with the data setup
+  It can be activated and deactivated by the command
+    sudo systemctl start aviso.service
+    sudo systemctl stop aviso.service
+  The status can be viewd be
+    sudo systemctl status aviso.service
+
+* For test purposes it my be used to test the behaviour on past trigger
+  events for a given (past) time range. This can be done by the listen
+  command within an env, where aviso is installed, like
+    sudo systemctl stop aviso.service
+    mamba activate avisoenv
+    aviso listen --from 2025-03-05T00:00:00.0Z --to 2025-03-05T08:00:00.0Z
+  
+* The file config.yaml in the direactory $HOME/.aviso guides the triggering
+
+* if multiple steps are given in a trigger command, like
+   step: [36,72,144]
+  the programs called by the trigger are executed sequentially, possibly
+  not in the time order of the triggers
+
+* the script get_ecmwf_aviso.sh creates ECMWF grib files and converts
+  them to MSS-convorm NetCDF files. One can speed up the process by
+  parallel download the grib files using "DOWNLOAD_ONLY=yes" in the file
+  settings.config and starting convert_all.sh separately.

aviso/config.yaml.parallel

@@ -0,0 +1,56 @@
+listeners:
+  - event: mars
+    request:
+      class: od
+      expver: 1
+      domain: g
+      stream: oper
+      step: 36
+    triggers:
+      - type: command
+        working_dir: $HOME/data-retrieval/aviso
+        command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out
+  - event: mars
+    request:
+      class: od
+      expver: 1
+      domain: g
+      stream: oper
+      step: 72
+    triggers:
+      - type: command
+        working_dir: $HOME/aviso
+        command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out
+  - event: mars
+    request:
+      class: od
+      expver: 1
+      domain: g
+      stream: oper
+      step: 108
+    triggers:
+      - type: command
+        working_dir: $HOME/aviso
+        command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out
+  - event: mars
+    request:
+      class: od
+      expver: 1
+      domain: g
+      stream: oper
+      step: 144
+    triggers:
+      - type: command
+        working_dir: $HOME/aviso
+        command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out
+  - event: mars
+    request:
+      class: od
+      expver: 1
+      domain: g
+      stream: oper
+      step: 228
+    triggers:
+      - type: command
+        working_dir: $HOME/aviso
+        command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out

aviso/config.yaml.sequential

@@ -0,0 +1,12 @@
+listeners:
+  - event: mars
+    request:
+      class: od
+      expver: 1
+      domain: g
+      stream: oper
+      step: [36, 72, 108, 144, 228]
+    triggers:
+      - type: command
+        working_dir: $HOME/data-retrieval/aviso
+        command: $HOME/data-retrieval/bin/get_ecmwf_aviso.sh --date ${request.date} --time ${request.time} --step ${request.step} > mars_${request.time}_${request.step}.out

bin/convert_all.sh

@@ -31,7 +31,7 @@ fi
 # script should start at 06h/18h to look for 00 12h forecast
 export h_exit=`date --date="+6hours" +%H`
 
-for FCSTEP in 036 072 108 144 228
+for FCSTEP in $FCSTEPS
 do
 
 # Set path, filenames and variables used later in the script
@@ -42,7 +42,7 @@ do
     export init_date=${DATE}T${TIME}
     echo BASE: $BASE
     export time_units="hours since ${init_date}"
-    echo time_units: "hours since ${init_date}"
+
 
     lockfile=grib/${DATASET}.${YMD}T${HH}.${FCSTEP}.ready
     echo `date` waiting for lockfile $lockfile
@@ -58,6 +58,7 @@ do
 	fi
     done
     rm $lockfile
+
     # Convert grib to netCDF, set init time
     echo `date`: converting ${FCSTEP}h forecast
     . $BINDIR/convert.sh
@@ -76,7 +77,6 @@ do
 	fi
 	if [ -f $pvfile ]; then
 	    mv $pvfile $MSSDIR
-	    
 	fi
 	if [ -f $alfile ]; then
 	    mv $alfile $MSSDIR

settings.default

@@ -3,6 +3,10 @@
 
 export PYTHON=python
 export CLEANUP=no
+export CLEANUP_DAY=`date --date=-2day +%d`
+export CLEANUP_MONTH=`date --date=-2day +%m`
+export CLEANUP_YEAR=`date --date=-2day +%Y`
+
 export MODEL_REDUCTION="-d lev,0,0 -d lev,16,28,4 -d lev,32,124,2"
 export DATASET=ecmwf