Merge pull request #59 from VERITAS-Observatory/v490.5-dev1

GernotMaier · web-flow · commit 2e4f40f72d6a · 2023-12-15T11:39:23.000+01:00
v490.5 dev1 - updated epoch definitions
diff --git a/DispBDTs/copy_dispBDTs.sh b/DispBDTs/copy_dispBDTs.sh
@@ -1,17 +1,17 @@
 #!/bin/bash
 # copy dispBDT files from IRF production site
 #
-# hardwired 
+# hardwired
 # - SIMTYPE (e.g., CARE_June2020)
 # - ANALYSISTYPE (e.g., AP)
 #
 
 IRFVERSION=$(cat ../IRFVERSION)
 ANALYSISTYPE="${VERITAS_ANALYSIS_TYPE:0:2}"
 SIMTYPE="CARE_UV_2212"
-SIMTYPE="CARE_RedHV"
 SIMTYPE="GRISU"
 SIMTYPE="CARE_June2020"
+SIMTYPE="CARE_RedHV"
 
 echo "COPY dispBDT for ${IRVERSION}, analysis type ${ANALYSISTYPE}, and simulation type ${SIMTYPE}"
 
@@ -46,6 +46,7 @@ do
             else
                 EPOCHS=$(cat ../IRF_EPOCHS_WINTER.dat | sort -u)
             fi
+            # FIXEPOCH EPOCHS="V6_2023_2023s"
         fi
         for E in $EPOCHS
         do
@@ -64,16 +65,18 @@ do
             # check log file for successful training
             for B in BDTDisp BDTDispError BDTDispSign
             do
+                echo "Parameters ${Z} ${B}"
                 if [[ -d ${IDIR}/${B} ]]; then
-                    CHECKF=$(tail -n 1 ${IDIR}/${B}/mvaAngRes_${ZE}-${B}.log | grep -v Delete | wc -l)
-                    if [[ $CHECKF != "0" ]]; then
+                    CHECKF=$(grep "Delete method" ${IDIR}/${B}/mvaAngRes_${ZE}-${B}.log | wc -l)
+                    if [[ $CHECKF != "4" ]]; then
                         echo "ERROR training file not complete in ${IDIR}/${B}/"
                     else
                         # expect 4=NTel xml files
                         NFILE=$(ls -1 ${IDIR}/${B}/*.xml | wc -l)
                         if [[ $NFILE == "4" ]]; then
                             cp -v -u ${IDIR}/${B}/*.xml ${ODIR}
                             cp -v -u ${IDIR}/${B}/*.log ${ODIR}
+                            gzip -f -v ${ODIR}/*.xml
                         else
                             echo "ERROR found only $NFILE xml files (expected 4) in ${IDIR}/${B}/"
                         fi
diff --git a/GammaHadronBDTs/copy_GammaHadron_V6_BDTs.sh b/GammaHadronBDTs/copy_GammaHadron_V6_BDTs.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 # copy gamma/hadron BDT files from IRF production site
 #
-# hardwired 
+# hardwired
 # - ANALYSISTYPE (e.g., AP)
 # - Directory with BDT output
 #
@@ -13,7 +13,7 @@ BDTDIR="$VERITAS_USER_DATA_DIR/analysis/Results/v490/${ANALYSISTYPE}/BDTtraining
 if [[ $ANALYSISTYPE == "AP" ]]; then
     CUTLIST="NTel2-Moderate NTel2-Soft NTel3-Hard"
 else
-    CUTLIST="NTel2-Soft"
+    CUTLIST="NTel2-SuperSoft"
 fi
 
 echo "COPY gamma/hadron BDTs for ${IRFVERSION}, analysis type ${ANALYSISTYPE}"
@@ -26,6 +26,7 @@ do
     else
         EPOCHS=$(cat ../IRF_EPOCHS_WINTER.dat | sort -u)
     fi
+    # FIXEPOCH EPOCHS="V6_2023_2023s"
     for E in $EPOCHS
     do
         for C in $CUTLIST
diff --git a/GammaHadronCutFiles/ANASUM.GammaHadron-Cut-NTel2-PointSource-Moderate-TMVA-BDT.dat b/GammaHadronCutFiles/ANASUM.GammaHadron-Cut-NTel2-PointSource-Moderate-TMVA-BDT.dat
@@ -121,3 +121,5 @@ v) TMVA Cuts
 * TMVA_MVACut V6_2021_2021s_ATM62 00 0.168 01 0.166 10 0.172 11 0.187 12 0.158 20 0.178 21 0.182 22 0.085 30 0.183 31 0.193 32 0.134 40 0.142 41 0.193 42 0.159 50 0.107 51 0.16 52 0.173 60 0.0638 61 0.121 62 0.171 70 0.0502 71 0.0983 72 0.174 80 0.065 81 0.0561 82 0.168
 * TMVA_MVACut V6_2021_2022w_ATM61 00 0.154 01 0.167 10 0.162 11 0.178 12 0.154 20 0.173 21 0.175 22 0.0867 30 0.178 31 0.187 32 0.131 40 0.136 41 0.186 42 0.152 50 0.0988 51 0.161 52 0.17 60 0.0666 61 0.122 62 0.173 70 0.0502 71 0.104 72 0.186 80 0.0661 81 0.0556 82 0.166
 * TMVA_MVACut V6_2022_2022s_ATM62 00 0.167 01 0.168 10 0.171 11 0.184 12 0.159 20 0.179 21 0.183 22 0.0897 30 0.18 31 0.19 32 0.135 40 0.137 41 0.188 42 0.157 50 0.0961 51 0.154 52 0.165 60 0.0618 61 0.119 62 0.168 70 0.0514 71 0.0943 72 0.167 80 0.0676 81 0.0506 82 0.157
+* TMVA_MVACut V6_2022_2023w_ATM61 00 0.159 01 0.152 10 0.166 11 0.172 12 0.158 20 0.176 21 0.177 22 0.0414 30 0.181 31 0.185 32 0.141 40 0.137 41 0.184 42 0.162 50 0.11 51 0.154 52 0.168 60 0.0652 61 0.118 62 0.175 70 0.0502 71 0.106 72 0.179 80 0.0631 81 0.061 82 0.168
+* TMVA_MVACut V6_2023_2023s_ATM62 01 0.152 10 0.17 11 0.185 12 0.159 20 0.177 21 0.183 22 0.0446 30 0.18 31 0.186 32 0.137 40 0.14 41 0.189 42 0.16 50 0.103 51 0.167 52 0.166 60 0.0666 61 0.125 62 0.173 70 0.0499 71 0.105 72 0.171 80 0.066 81 0.0548 82 0.169
diff --git a/GammaHadronCutFiles/ANASUM.GammaHadron-Cut-NTel2-PointSource-Soft-TMVA-BDT.dat b/GammaHadronCutFiles/ANASUM.GammaHadron-Cut-NTel2-PointSource-Soft-TMVA-BDT.dat
@@ -122,3 +122,5 @@ cut values for different epochs and its corresponding directory where to find .r
 * TMVA_MVACut V6_2021_2021s_ATM62 00 0.141 01 0.141 10 0.147 11 0.155 12 0.0192 20 0.169 21 0.154 22 0.0578 30 0.147 31 0.171 32 0.0982 40 0.0974 41 0.162 42 0.115 50 0.0497 51 0.114 52 0.133 60 0.0318 61 0.0608 62 0.142 70 0.0535 71 0.0399 72 0.134 80 0.0673 81 0.0523 82 0.11
 * TMVA_MVACut V6_2021_2022w_ATM61 00 0.141 01 0.144 10 0.141 11 0.141 12 0.0154 20 0.157 21 0.155 22 0.0572 30 0.146 31 0.165 32 0.0954 40 0.0959 41 0.159 42 0.117 50 0.05 51 0.115 52 0.137 60 0.035 61 0.0611 62 0.14 70 0.0569 71 0.0376 72 0.134 80 0.0695 81 0.0537 82 0.11
 * TMVA_MVACut V6_2022_2022s_ATM62 00 0.136 01 0.148 10 0.149 11 0.149 12 0.0174 20 0.157 21 0.156 22 0.0581 30 0.139 31 0.162 32 0.0979 40 0.0974 41 0.157 42 0.122 50 0.047 51 0.113 52 0.129 60 0.0306 61 0.0597 62 0.14 70 0.0534 71 0.0381 72 0.125 80 0.0649 81 0.0513 82 0.103
+* TMVA_MVACut V6_2022_2023w_ATM61 00 0.137 01 0.132 10 0.146 11 0.143 12 0.0198 20 0.158 21 0.155 22 0.0232 30 0.14 31 0.163 32 0.103 40 0.101 41 0.152 42 0.122 50 0.0525 51 0.112 52 0.128 60 0.0333 61 0.0622 62 0.133 70 0.055 71 0.0367 72 0.129 80 0.0647 81 0.0542 82 0.107
+* TMVA_MVACut V6_2023_2023s_ATM62 00 0.14 01 0.137 10 0.154 11 0.145 12 0.0201 20 0.162 21 0.151 22 0.0249 30 0.145 31 0.17 32 0.11 40 0.102 41 0.163 42 0.118 50 0.0569 51 0.111 52 0.13 60 0.0312 61 0.0612 62 0.136 70 0.0542 71 0.0389 72 0.132 80 0.0671 81 0.0511 82 0.109
diff --git a/GammaHadronCutFiles/ANASUM.GammaHadron-Cut-NTel2-PointSource-SuperSoft-NN-TMVA-BDT.dat b/GammaHadronCutFiles/ANASUM.GammaHadron-Cut-NTel2-PointSource-SuperSoft-NN-TMVA-BDT.dat
@@ -122,3 +122,5 @@ cut values for different epochs and its corresponding directory where to find .r
 * TMVA_MVACut V6_2021_2021s_ATM62 00 0.146 01 0.174 02 0.0228 10 0.16 11 0.166 12 0.0255 20 0.176 21 0.175 22 0.092 30 0.161 31 0.188 32 0.127 40 0.126 41 0.177 42 0.139 50 0.0877 51 0.141 52 0.147 60 0.0355 61 0.0938 62 0.16 70 0.0505 71 0.0739 72 0.169 80 0.068 81 0.0501 82 0.135
 * TMVA_MVACut V6_2021_2022w_ATM61 00 0.144 01 0.155 02 0.0164 10 0.161 11 0.154 12 0.0257 20 0.17 21 0.168 22 0.0919 30 0.163 31 0.179 32 0.127 40 0.121 41 0.174 42 0.141 50 0.0855 51 0.141 52 0.151 60 0.0338 61 0.0958 62 0.162 70 0.0567 71 0.0753 72 0.167 80 0.0672 81 0.0502 82 0.14
 * TMVA_MVACut V6_2022_2022s_ATM62 00 0.157 01 0.168 02 0.0311 10 0.158 11 0.162 12 0.0257 20 0.168 21 0.172 22 0.0926 30 0.157 31 0.181 32 0.123 40 0.118 41 0.172 42 0.14 50 0.0798 51 0.141 52 0.148 60 0.0326 61 0.0941 62 0.15 70 0.0477 71 0.0663 72 0.16 80 0.0678 81 0.0449 82 0.135
+* TMVA_MVACut V6_2022_2023w_ATM61 00 0.146 01 0.148 02 0.0226 10 0.145 11 0.148 12 0.0272 20 0.166 21 0.159 22 0.0763 30 0.16 31 0.165 32 0.131 40 0.117 41 0.166 42 0.149 50 0.0896 51 0.139 52 0.153 60 0.0419 61 0.0989 62 0.163 70 0.0528 71 0.0737 72 0.166 80 0.0727 81 0.0538 82 0.14
+* TMVA_MVACut V6_2023_2023s_ATM62 00 0.156 01 0.161 02 0.0295 10 0.161 11 0.163 12 0.028 20 0.18 21 0.172 22 0.127 30 0.166 31 0.181 32 0.146 40 0.123 41 0.166 42 0.155 50 0.0833 51 0.135 52 0.165 60 0.0297 61 0.0879 62 0.166 70 0.053 71 0.0624 72 0.171 80 0.0675 81 0.0503 82 0.132
diff --git a/GammaHadronCutFiles/ANASUM.GammaHadron-Cut-NTel3-PointSource-Hard-TMVA-BDT.dat b/GammaHadronCutFiles/ANASUM.GammaHadron-Cut-NTel3-PointSource-Hard-TMVA-BDT.dat
@@ -121,3 +121,5 @@ v) TMVA Cuts
 * TMVA_MVACut V6_2021_2021s_ATM62 10 0.15 11 0.033 20 0.15 21 0.143 30 0.15 31 0.153 32 0.108 40 0.161 41 0.155 42 0.0441 50 0.126 51 0.172 52 0.11 60 0.087 61 0.146 62 0.129 70 0.0721 71 0.126 72 0.119 80 0.0782 81 0.0923 82 0.138
 * TMVA_MVACut V6_2021_2022w_ATM61 10 0.143 11 0.0742 20 0.145 21 0.137 30 0.147 31 0.155 32 0.104 40 0.16 41 0.15 42 0.0481 50 0.122 51 0.16 52 0.104 60 0.095 61 0.157 62 0.114 70 0.078 71 0.132 72 0.133 80 0.0769 81 0.0925 82 0.146
 * TMVA_MVACut V6_2022_2022s_ATM62 10 0.149 11 0.00667 20 0.149 21 0.142 30 0.147 31 0.157 32 0.105 40 0.155 41 0.161 42 0.0445 50 0.122 51 0.162 52 0.107 60 0.0876 61 0.145 62 0.118 70 0.0714 71 0.13 72 0.123 80 0.0799 81 0.0892 82 0.145
+* TMVA_MVACut V6_2022_2023w_ATM61 10 0.144 11 0.0419 20 0.146 21 0.132 30 0.15 31 0.142 32 0.105 40 0.146 41 0.147 42 0.0533 50 0.13 51 0.153 52 0.124 60 0.0895 61 0.143 62 0.134 70 0.0861 71 0.132 72 0.149 80 0.08 81 0.106 82 0.161
+* TMVA_MVACut V6_2023_2023s_ATM62 10 0.148 11 0.0306 20 0.143 21 0.133 30 0.147 31 0.15 32 0.105 40 0.159 41 0.155 42 0.0486 50 0.129 51 0.168 52 0.115 60 0.0926 61 0.149 62 0.135 70 0.0825 71 0.135 72 0.144 80 0.0808 81 0.104 82 0.168
diff --git a/IRF_EPOCHS_SUMMER.dat b/IRF_EPOCHS_SUMMER.dat
@@ -9,3 +9,4 @@ V6_2018_2019
 V6_2020_2020s
 V6_2021_2021s
 V6_2022_2022s
+V6_2023_2023s
diff --git a/IRF_EPOCHS_WINTER.dat b/IRF_EPOCHS_WINTER.dat
@@ -9,3 +9,4 @@ V6_2018_2019
 V6_2019_2020w
 V6_2020_2021w
 V6_2021_2022w
+V6_2022_2023w
diff --git a/IRF_EPOCHS_obsfilter.dat b/IRF_EPOCHS_obsfilter.dat
@@ -6,3 +6,4 @@ V6_2016_2017
 V6_2018_2019
 V6_2021_2022w
 V6_2022_2022s
+V6_2022_2023w
diff --git a/ParameterFiles/ThroughputCorrection.runparameter b/ParameterFiles/ThroughputCorrection.runparameter
@@ -3,7 +3,7 @@ Throughput corrections
 
 Basic correction:
 
-Apply size scale corrections based on s=g*T, where g is the relative gain and T the transmission of the optical part (mirror reflectivity). 
+Apply size scale corrections based on s=g*T, where g is the relative gain and T the transmission of the optical part (mirror reflectivity).
 
 Partially extracted from here:
 https://veritas.sao.arizona.edu/wiki/index.php/Mirror_Reflectivity
@@ -17,14 +17,14 @@ https://veritas.sao.arizona.edu/wiki/Flux_Calibration_/_Energy_scale_2020
 
 Different possibilities for measuring the baseline...
 
-transmission/reflectivity baseline for each telescope 
+transmission/reflectivity baseline for each telescope
 (estimated from end of V5 / beginning of V6):
 TB 1 0.91
 TB 2 0.93
 TB 3 0.95
 TB 4 0.95
 
-transmission/reflectivity baseline for each telescope 
+transmission/reflectivity baseline for each telescope
 (estimated from calibration curve in https://veritas.sao.arizona.edu/wiki/images/0/0e/IOReaderDetectorConfigV6.txt at 340nm):
 TB 1 0.80
 TB 2 0.82
@@ -57,6 +57,8 @@ Table with average reflectivities (from David Hanna, see elog end of March 2020)
 * T V6_2021_2021s 0.670 0.701 0.774 0.647
 * T V6_2021_2022w 0.697 0.680 0.767 0.652
 * T V6_2022_2022s 0.682 0.684 0.764 0.645
+* T V6_2022_2023w 0.723 0.682 0.743 0.636
+* T V6_2023_2023s 0.759 0.689 0.732 0.666
 
 Table with g-factors (from single PE, linked by Qi in https://www.hep.physics.mcgill.ca/~veritas/gain)
   t ID T1 T2 T3 T4
@@ -79,6 +81,8 @@ Table with g-factors (from single PE, linked by Qi in https://www.hep.physics.mc
 * G V6_2021_2021s 0.937 0.987 1.022 1.054
 * G V6_2021_2022w 0.927 0.993 0.999 1.047
 * G V6_2022_2022s 0.932 0.959 0.970 0.991
+* G V6_2022_2023w 0.952 0.957 0.976 0.972
+* G V6_2023_2023s 0.897 0.953 0.986 0.991
 
 Table with average-year s-factors (from Mireia Nievas, jupyter-notebook)
   s = T * G
@@ -103,6 +107,8 @@ Table with average-year s-factors (from Mireia Nievas, jupyter-notebook)
 * s V6_2021_2021s 0.628 0.692 0.792 0.682
 * s V6_2021_2022w 0.646 0.676 0.766 0.683
 * s V6_2022_2022s 0.636 0.656 0.741 0.639
+* s V6_2022_2023w 0.688 0.653 0.725 0.619
+* s V6_2023_2023s  0.681 0.656 0.722 0.660
 
 Note: mirror swaps/jumps/breaks in T2 (2014_2015), T3 (2014_2015, 2017_2018), T4 (2013_2014).
 
diff --git a/ParameterFiles/VERITAS.Epochs.runparameter b/ParameterFiles/VERITAS.Epochs.runparameter
@@ -41,10 +41,14 @@ short period with <3% throughput differences only)
 * EPOCH V6_2021_2021s 2021-04-28 2021-11-16
 * EPOCH V6_2021_2022w 2021-11-17 2022-05-09
 * EPOCH V6_2022_2022s 2022-05-10 2022-11-08
-TODO - replace V6_2021_2022w by V6_2022_2023w
+* EPOCH V6_2022_2023w 2022-11-09 2023-05-01
+* EPOCH V6_2023_2023s 2023-05-01 2023-10-31
+TODO - replace V6_2022_2023w by V6_2023_2024w
 (when throughput parameters are available)
-* EPOCH V6_2021_2022w 2022-11-09 2023-05-01
-* EPOCH V6_2022_2022s 2023-05-01 2023-11-08
+* EPOCH V6_2022_2023w 2023-11-01 2024-04-22
+TODO - replace V6_2023_2023s by V6_2024_2024s
+(when throughput parameters are available)
+* EPOCH V6_2023_2023s 2024-04-23 2024-11-15
 
 =================================================
 
@@ -72,6 +76,7 @@ WINTER:
 * ATMOSPHERE 61 2020-11-05  2021-04-27
 * ATMOSPHERE 61 2021-11-17  2022-05-09
 * ATMOSPHERE 61 2022-11-09  2023-05-01
+* ATMOSPHERE 61 2023-11-01  2024-04-22
 
 SUMMER:
 * ATMOSPHERE 22 2007-06-04  2007-11-23
@@ -91,7 +96,8 @@ SUMMER:
 * ATMOSPHERE 62 2020-05-08  2020-11-04
 * ATMOSPHERE 62 2021-04-28  2021-11-16
 * ATMOSPHERE 62 2022-05-10  2022-11-08
-* ATMOSPHERE 62 2023-05-02  2023-11-08
+* ATMOSPHERE 62 2023-05-02  2023-10-31
+* ATMOSPHERE 62 2024-04-23  2024-11-15
 
 =================================================
 
diff --git a/download_irfs.sh b/download_irfs.sh
@@ -3,15 +3,15 @@
 #
 
 VERSION=$(cat IRFVERSION)
-VERSION="${VERSION}.2"
+VERSION="${VERSION}.5"
 
 if [ -z "$1" ]; then
 echo "
 ./download_irfs.sh <filelist>
 
 download IRFs from UCLA for Eventdisplay version $VERSION
 
-   <filelist> list of tar packages to be downloaded 
+   <filelist> list of tar packages to be downloaded
               (usually found in directory ./transfer)
 
 "
diff --git a/transfer/pack_irfs_pack_ucla.sh b/transfer/pack_irfs_pack_ucla.sh
@@ -1,12 +1,12 @@
 #!/bin/bash
 # pack IRFs for UCLA
 # - IRFs packed in reasonable sized tar packages
-# 
+#
 # this script should be followed by put_irfs_to_ucla.sh
 #
 
 # directory with all packages
-DDIR="tar_packages_v490.2"
+DDIR="tar_packages_v490.5"
 
 # list of cuts
 CLISTNV=$(cat ../IRF_GAMMAHADRONCUTS.dat)
@@ -86,7 +86,7 @@ pack_dispbdts()
                 mv -f ${D}.tar ${DDIR}/
             else
                 echo "ERROR directory DispBDTs/${C}/ does not exist"
-            fi    
+            fi
         done
     done
 }
@@ -179,7 +179,7 @@ pack_effectivareas_V4V5()
                        tar -cvf ${D}.tar EffectiveAreas/*${T}*${C}*${I}*${A}*.root
                        mv -v ${D}.tar ./${DDIR}/
                    else
-                      echo "Packing EffectiveAreas $F ${C} $I $A ${T} (no files found)" 
+                      echo "Packing EffectiveAreas $F ${C} $I $A ${T} (no files found)"
                    fi
                done
             done
@@ -194,7 +194,7 @@ pack_effectiveareas_V6
 pack_effectivareas_V4V5
 
 pack_dispbdts
- 
+
 pack_gammahadronbdts
 
 cd ${P}
diff --git a/transfer/put_irfs_to_ucla.sh b/transfer/put_irfs_to_ucla.sh
@@ -2,7 +2,7 @@
 # upload IRFs to UCLA
 # - uploading in tar packages
 # - one tar package per directory
-# 
+#
 # prepare by hand the upload directory:
 # bbftp -u bbftp -V -S -m -p 12 -e "mkdir /veritas/upload/EVNDISP/v490" gamma1.astro.ucla.edu
 #
@@ -14,7 +14,7 @@ command -v bbftp >/dev/null 2>&1 || { echo >&2 "bbftp is not installed. Aborting
 
 # Eventdisplay version
 VERSION=$(cat ../IRFVERSION)
-VERSION="${VERSION}.2"
+VERSION="${VERSION}.5"
 if [[ $IRFTYPE ]]; then
    VERSION=${VERSION}${IRFTYPE}
 fi
