Merge pull request #52 from VERITAS-Observatory/v2.2.0-dev

vV2.2.0 dev - improve testing.

Author: GernotMaier

config/config_ATM61_Zd65_production.dat

@@ -12,7 +12,7 @@ GROPTICS_CONFIG="VERITAS_NewArray.cfg"
 
 # CARE stdHV
 CARE_CONFIG_std="CARE_V6_Std.txt"
-NSB_LIST_std="200 900"
+NSB_LIST_std="50 75 100 130 160 200 250 300 350 400 450 600 750 900"
 
 # CARE redHV
 CARE_CONFIG_redHV="CARE_V6_RHV.txt"

config/config_ATM62_Zd65_production.dat

@@ -1,6 +1,6 @@
 RUN_START=661000
-N_RUNS=2000
-N_SHOWER=3000
+N_RUNS=3000
+N_SHOWER=2000
 ZENITH=65
 ATMOSPHERE=62
 

docs/changes/52.feature.md

@@ -0,0 +1 @@
+Ensure VBF file consistency by running vbfReindex over all generated files. Ensure also that no corrupt file end up in the generated dataset.

docs/changes/53.maintenance.md

@@ -0,0 +1 @@
+Introduction of towncrier-generated changelogs. Add CI running pre-commit.

scripts/mergevbf.sh

@@ -46,12 +46,12 @@ generate_mergevbf_submission_script()
     CONTAINER_EXTERNAL_DIR="$CONTAINER_EXTERNAL_DIR -v \"$LOG_DIR:/workdir/external/log/\""
 
     if [[ $VTSSIMPIPE_CONTAINER == "docker" ]]; then
-        CARE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_MERGEVBF_IMAGE}"
+        MERGE_EXE="docker run --rm $CONTAINER_EXTERNAL_DIR ${VTSSIMPIPE_CONTAINER_URL}${VTSSIMPIPE_MERGEVBF_IMAGE}"
     elif [[ $VTSSIMPIPE_CONTAINER == "apptainer" ]]; then
-        CARE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_MERGEVBF_IMAGE/:/_}.sif"
+        MERGE_EXE="apptainer exec --cleanenv ${CONTAINER_EXTERNAL_DIR//-v/--bind} ${VTSSIMPIPE_CONTAINER_DIR}/${VTSSIMPIPE_MERGEVBF_IMAGE/:/_}.sif"
     fi
 
-    batch_size=250
+    batch_size=251
     vbf_id="0"
     MERGEDFILE=$(get_merge_file_name "$WOBBLE" "$NSB" "$vbf_id")
     TMP_FL_LIST="$MERGEVBF_DATA_DIR"/tmp_file_list_${WOBBLE}_${NSB}.dat
@@ -68,19 +68,24 @@ generate_mergevbf_submission_script()
         echo "MERGEDFILE=$MERGEDFILE"
         echo
         echo "find \"$CARE_DATA_DIR\" -type f -name \"*.vbf\" -exec basename {} \; | sed 's|^|/workdir/external/care/|' | sort -n > \"$TMP_FL_LIST\""
-        echo "split -d -l $batch_size \"$TMP_FL_LIST\" \"$TMP_FL_SPLIT_LIST\""
         echo
+        echo "while read -r file; do"
+        echo "   ${MERGE_EXE} bash -c \"/workdir/VBF/bin/vbfReindex "\$file"\""
+        echo "done < \"$TMP_FL_LIST\""
+        echo
+        echo "split -d -l $batch_size \"$TMP_FL_LIST\" \"$TMP_FL_SPLIT_LIST\""
         echo "for flist in \"$TMP_FL_SPLIT_LIST\"*; do"
         echo "   RUNNUMBER=\$(head -n 1 \"\$flist\" | awk -F '[^0-9]+' '{print \$2}')"
-        echo "   echo \$RUNNUMBER"
+        echo "   echo \"\$RUNNUMBER\""
         echo "   if [[ -e \"${MERGEVBF_DATA_DIR}/${MERGEDFILE}\${RUNNUMBER}.vbf.zst\" ]] && [[ ! -e \"${MERGEVBF_DATA_DIR}/${MERGEDFILE}\${RUNNUMBER}.vbf\" ]]; then"
         echo "      echo \" Found ${MERGEVBF_DATA_DIR}/${MERGEDFILE}\${RUNNUMBER}.vbf.zst; skipping job\""
         echo "      continue"
         echo "   fi"
         echo "   MERGEVBF=\"./bin/mergeVBF /workdir/external/mergevbf/\$(basename \"\$flist\")  /workdir/external/mergevbf/\${MERGEDFILE}\${RUNNUMBER}.vbf \${RUNNUMBER}\""
+        echo "   REINDEX=\"/workdir/VBF/bin/vbfReindex /workdir/external/mergevbf/${MERGEDFILE}\${RUNNUMBER}.vbf\""
         echo "   ZSTD_VBF=\"zstd -f /workdir/external/mergevbf/${MERGEDFILE}\${RUNNUMBER}.vbf\""
         echo "   echo \"LOG FILE ${MERGEVBF_DATA_DIR}/${MERGEDFILE}\${RUNNUMBER}.log\""
-        echo "   ${CARE_EXE} bash -c \"cd /workdir/EventDisplay_v4 && \${MERGEVBF} && \${ZSTD_VBF}\" > ${MERGEVBF_DATA_DIR}/${MERGEDFILE}\${RUNNUMBER}.log 2>&1"
+        echo "   ${MERGE_EXE} bash -c \"cd /workdir/EventDisplay_v4 && \${MERGEVBF} && \${REINDEX} && \${ZSTD_VBF}\" > ${MERGEVBF_DATA_DIR}/${MERGEDFILE}\"\${RUNNUMBER}\".log 2>&1"
         echo
         echo "   [ -e \"${MERGEVBF_DATA_DIR}/${MERGEDFILE}\${RUNNUMBER}.vbf.zst\" ] && rm -f \"${MERGEVBF_DATA_DIR}/${MERGEDFILE}\${RUNNUMBER}.vbf\""
         echo "done"