Merge branch 'hyperopt_penalty' of github.com:NNPDF/nnpdf into hyperopt_penalty

Author: Tommaso Giani

.github/workflows/fitbot.yml

@@ -8,10 +8,12 @@ on:
 
 # some general variables
 env:
-  N3FIT_MAXNREP: 20 # total number of replicas to fit
-  POSTFIT_NREP: 16 # requested replicas for postfit
-  REFERENCE_SET: NNBOT-955eb2bcc-2025-06-17 # reference set for exact results
-  STABLE_REFERENCE_SET: NNBOT-955eb2bcc-2025-06-17 # reference set for last tag
+  N3FIT_MAXNREP: 30 # total number of replicas to fit
+  POSTFIT_NREP: 15 # requested minimum replicas for postfit
+  # IMPORTANT
+  # WHEN CHANGING THE REFERENCE SET, THE NEW REFERENCE MUST BE MANUALLY UPLOADED TO THE SERVER
+  REFERENCE_SET: NNBOT-99108504e-2025-11-22 # reference set for exact results
+  STABLE_REFERENCE_SET: NNBOT-99108504e-2025-11-22 # reference set for last tag
   PYTHONHASHSEED: "0"
 
 jobs:
@@ -55,12 +57,12 @@ jobs:
         cd $RUNFOLDER
         cp developing.yml $RUNCARD.yml
         vp-setupfit $RUNCARD.yml
-    # run n3fit replicas sequentially
+    # try running the n3fit replicas in parallel
     - name: Running n3fit
       shell: bash -l {0}
       run: |
         cd $RUNFOLDER
-        for ((i=1; i<=$N3FIT_MAXNREP; i+=1)); do n3fit $RUNCARD.yml $i ; done
+        n3fit $RUNCARD.yml 1 -r $N3FIT_MAXNREP
     # performing DGLAP
     - name: Running dglap
       shell: bash -l {0}
@@ -79,17 +81,16 @@ jobs:
       run: |
         conda activate nnpdfenv
         cd $RUNFOLDER
-        postfit $POSTFIT_NREP $RUNCARD
-        res=$(vp-upload $RUNCARD 2>&1)
-        echo ${res}
-        while echo ${res} | grep ERROR >/dev/null
-        do
-          sleep 30s
-          res=$(vp-upload $RUNCARD 2>&1)
-        done
-        url=$( echo "${res}" | grep https )
-        echo "FIT_URL=$url" >> $GITHUB_ENV
-    # running validphys report
+        postfit $POSTFIT_NREP $RUNCARD --at-least-nrep
+        ln -s ${PWD}/${RUNCARD} ${CONDA_PREFIX}/share/NNPDF/results
+        tar -czf ${RUNCARD}.tar.gz ${RUNCARD}
+        echo "PATH_TO_SAVE=${PWD}/${RUNCARD}.tar.gz" >> ${GITHUB_ENV}
+    - name: Keep the fit as an artifact
+      if: ${{ !cancelled() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ env.RUNCARD }}.tar.gz
+        path: ${{ env.PATH_TO_SAVE }}
     - name: Building and upload report
       shell: bash -l {0}
       run: |
@@ -121,6 +122,6 @@ jobs:
             - Fit Name: ${{ env.RUNCARD }}
             - Fit Report wrt master: ${{ env.REPORT_URL }}
             - Fit Report wrt latest stable reference: ${{ env.REPORT_URL_STABLE }}
-            - Fit Data: ${{ env.FIT_URL }}
+            - Fit Data: fit data is kept as an artifact. Please, remember to upload it to the server if the reference is changed.
 
           Check the report **carefully**, and please  buy me a :coffee: , or better, a GPU :wink:!

doc/sphinx/source/n3fit/methodology.rst

@@ -346,3 +346,58 @@ The figure above provides a schematic representation of this feature scaling met
 2. ``[number of points]`` points are kept (dark blue), while other points are discarded (light blue).
 3. A cubic spline function is used to do the interpolation between the points that have not been
    discarded.
+
+
+Diagonal basis
+--------------
+
+Performing the training and validation split without diagonalising the :math:`t_0` covmat :math:`C_{0}` neglects
+any correlations that may be present between training and validation data. To remedy this,
+we rotate to a basis in which the correlation matrix is diagonal before performing any training/validation split.
+Starting from the definition of the :math:`\chi^2` function in the NNPDF methodology, we have
+
+.. math::
+
+    \chi^2 &= (D-T)^T C_0^{-1} (D-T) \\
+           &= (D-T)^T R^{-1} R C_0^{-1} R R^{-1} (D-T) \\
+           &= (D-T)^T R^{-1} \left( R^{-1} C_0 R^{-1} \right)^{-1} R^{-1} (D-T) \\
+           &\equiv \tilde{\epsilon}^T \rho^{-1} \tilde{\epsilon} \, ,
+
+where we have defined :math:`\tilde{\epsilon} \equiv R^{-1}(D-T)` and :math:`\rho = R^{-1} C_0 R^{-1}`.
+
+Choosing :math:`R_{ii} = \sqrt{C_{0, ii}}`, we have that :math:`R^{-1} C_0 R^{-1}` coincides with the usual definition of the correlation matrix.
+
+Next, we move to the basis in which :math:`\rho` is diagonal. Writing :math:`\rho = \tilde{U}^T \tilde{\Lambda} \tilde{U}`, we find
+
+.. math::
+
+    \chi^2 &= \tilde{\epsilon}^T \rho^{-1} \tilde{\epsilon} \\
+           &= \tilde{\epsilon}^T (\tilde{U}^T \tilde{\Lambda} \tilde{U})^{-1} \tilde{\epsilon} \\
+           &= \tilde{\epsilon}^T \tilde{U}^T \tilde{\Lambda}^{-1} \tilde{U} \tilde{\epsilon} \\
+           &\equiv \dbtilde{\epsilon}^T \tilde{\Lambda}^{-1} \dbtilde{\epsilon} \, ,
+
+where on the last line we have defined
+
+.. math::
+
+    \dbtilde{\epsilon} \equiv \tilde{U}\tilde{\epsilon} = \tilde{U}R^{-1}(D-T).
+
+In index notation, this reads
+
+.. math::
+
+    \dbtilde{\epsilon_i} = \tilde{U}_{ij} \frac{(D-T)_j}{\sqrt{C_{0, jj}}}
+
+The transformed data :math:`\dbtilde{\epsilon}` is statistically independent in the diagonal basis of the correlation matrix :math:`\rho`.
+Computing the covariance of :math:`\dbtilde{\epsilon}`,
+
+.. math::
+
+    \mathbb{E}[\dbtilde{\epsilon}\dbtilde{\epsilon}^T]
+      &= \mathbb{E} \big[ (\tilde{U} R^{-1}(D-T)) (\tilde{U} R^{-1}(D-T))^T \big] \\
+      &= \tilde{U} R^{-1} \mathbb{E}[(D-T)(D-T)^T] R^{-1} \tilde{U}^T \\
+      &= \tilde{U} \rho \tilde{U}^T \\
+      &= \tilde{U}\tilde{U}^T \tilde{\Lambda} \tilde{U}\tilde{U}^T \\
+      &= \tilde{\Lambda} \, ,
+
+we find that it is diagonal, which demonstrates that the training/validation data are statistically independent indeed.

extra_tests/regression_checks.py

@@ -17,7 +17,7 @@
 runcard_and_replicas = {
     "normal_fit": 72,
     "central": 16,
-    "diagonal": 45,
+    "no_diagonal": 45,
     "feature_scaling": 81,
     "flavour": 29,
     "no_msr": 92,