Improvements

Mathieu Taillefumier · Mathieu Taillefumier · commit 270ba42ca492 · 2026-02-18T10:42:02.000+01:00
- QUDA_FERMIONIC_FORCES and QUDA_EXPERIMENTAL are always on
- Removed KOJAK instrumentation
- Added a custom Qphix find package file because the original one is broken
diff --git a/.github/workflows/qphix-build.yaml b/.github/workflows/qphix-build.yaml
@@ -172,9 +172,10 @@ jobs:
            -DTM_USE_OMP=ON \
            -DTM_USE_LEMON=ON \
            -DTM_USE_QPHIX=ON \
-           -DCMAKE_CXXFLAGS="-O2 -mtune=haswell -march=haswell -mavx2 -mfma -DOMPI_SKIP_MPICXX -fopenmp" \
-           -DCMAKE_CFLAGS="-O2 -mtune=haswell -march=haswell -mavx2 -mfma -DOMPI_SKIP_MPICXX -fopenmp" \
+           -DCMAKE_CXX_FLAGS="-O2 -mtune=haswell -march=haswell -mavx2 -mfma -DOMPI_SKIP_MPICXX -fopenmp" \
+           -DCMAKE_C_FLAGS="-O2 -mtune=haswell -march=haswell -mavx2 -mfma -DOMPI_SKIP_MPICXX -fopenmp" \
            -DQPHIX_DIR="${{github.workspace}}/qphix/build/install_dir" \
+           -DQMP_DIR="${{github.workspace}}/qmp/build/install_dir" \
             ..
           make -j > config.log
 
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -123,12 +123,6 @@ cmake_dependent_option(TM_USE_LEMON "Use the lemon io library" OFF "TM_USE_MPI"
                        ON)
 
 # GPU dependent options
-cmake_dependent_option(TM_USE_QUDA_EXPERIMENTAL "Enable QUDA support" ON
-                       "TM_USE_QUDA" OFF)
-cmake_dependent_option(
-  TM_QUDA_FERMIONIC_FORCES "Enable support for fermionic forces using QUDA" ON
-  "TM_USE_QUDA" OFF)
-
 cmake_dependent_option(TM_USE_NVHPC "Enable Nvidia HPC toolkit" OFF
                        "TM_USE_CUDA" OFF)
 
@@ -253,15 +247,7 @@ if(TM_USE_HIP OR QUDA_TARGET_HIP)
 endif()
 
 if(TM_USE_QPHIX)
-  find_package(QPhiX REQUIRED CONFIG)
-  message("${QPhiX_LIBRARIES}")
-  if(NOT TARGET tmlqcd::qphix)
-    add_library(tmlqcd::qphix INTERFACE IMPORTED)
-    set_target_properties(tmlqcd::qphix PROPERTIES INTERFACE_LINK_LIBRARIES
-                                                   "${QPhiX_LIBRARIES}")
-    set_target_properties(tmlqcd::qphix PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
-                                                   "${QPhiX_INCLUDE_DIRS}")
-  endif()
+  find_package(Qphix REQUIRED)
 endif()
 
 # check for fftw3 (rely on pkgconfig).
diff --git a/cmake/FindQphix.cmake b/cmake/FindQphix.cmake
@@ -0,0 +1,39 @@
+find_library(TM_QMP_LIBS NAMES qmp PATH_SUFFIXES "lib" "lib64")
+find_library(TM_QPHIX_LIBS_CODEGEN NAMES "qphix_codegen" PATH_SUFFIXES "lib" "lib64")
+find_library(TM_QPHIX_LIBS_SOLVER NAMES "qphix_solver" PATH_SUFFIXES "lib" "lib64")
+
+message("${QMP_DIR}")
+
+find_path(
+  TM_QMP_INCLUDE_DIRS
+  NAMES qmp.h
+  PATH_SUFFIXES "include"
+  PATHS "${QMP_DIR}")
+
+find_path(
+  TM_QPHIX_INCLUDE_DIRS
+  NAMES qphix_config.h
+  PATH_SUFFIXES "qphix"
+  PATHS "${QPHIX_DIR}")
+find_path(
+  TM_QPHIX_CODEGEN_INCLUDE_DIRS
+  NAMES qpx_utils.h
+  PATH_SUFFIXES "qphix_codegen"
+  PATHS "${QPHIX_DIR}")
+
+message("${TM_QMP_INCLUDE_DIRS} ${TM_QPHIX_INCLUDE_DIRS} ${TM_QMP_LIBS} ${TM_QPHIX_LIBS_CODEGEN} ${TM_QPHIX_LIBS_SOLVER}")
+
+find_package_handle_standard_args(
+  Qphix DEFAULT_MSG TM_QPHIX_LIBS_CODEGEN TM_QPHIX_LIBS_SOLVER TM_QPHIX_INCLUDE_DIRS TM_QMP_LIBS TM_QMP_INCLUDE_DIRS TM_QPHIX_CODEGEN_INCLUDE_DIRS)
+
+if(TM_QMP_LIBS
+    AND TM_QPHIX_INCLUDE_DIRS
+    AND NOT TARGET tmlqcd::qphix)
+  add_library(tmlqcd::qphix INTERFACE IMPORTED)
+  set_target_properties(tmlqcd::qphix PROPERTIES INTERFACE_LINK_LIBRARIES
+    "${TM_QPHIX_LIBS_CODEGEN};${TM_QPHIX_LIBS_SOLVER};${TM_QMP_LIBS}")
+  set_target_properties(tmlqcd::qphix PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+    "${TM_QPHIX_INCLUDE_DIRS}/..;${TM_QPHIX_INCLUDE_DIRS};${TM_QPHIX_CODEGEN_INCLUDE_DIRS}")
+endif()
+
+mark_as_advanced(TM_QPHIX_LIBRARIES TM_QPHIX_INCLUDE_DIRS TM_QMP_LIBS TM_QMP_INCLUDE_DIRS)
diff --git a/cmake/tmlqcd_config_internal.h.in b/cmake/tmlqcd_config_internal.h.in
@@ -111,12 +111,6 @@
 /* Using QUDA GPU */
 #cmakedefine TM_USE_QUDA 
 
-/* Using experimental QUDA version */
-#cmakedefine TM_QUDA_EXPERIMENTAL
-
-/* Using QUDA fermionic forces */
-#cmakedefine TM_QUDA_FERMIONIC_FORCES
-
 /* Using DDalphaAMG */
 #cmakedefine TM_USE_DDalphaAMG
 
diff --git a/src/bin/deriv_mg_tune.c b/src/bin/deriv_mg_tune.c
@@ -98,11 +98,6 @@ int main(int argc, char *argv[]) {
 
   init_critical_globals(TM_PROGRAM_DERIV_MG_TUNE);
 
-#ifdef TM_KOJAK_INST
-#pragma pomp inst init
-#pragma pomp inst begin(main)
-#endif
-
   verbose = 1;
   g_use_clover_flag = 0;
 
@@ -367,9 +362,6 @@ int main(int argc, char *argv[]) {
 #endif
 
   return (0);
-#ifdef TM_KOJAK_INST
-#pragma pomp inst end(main)
-#endif
 }
 
 static void usage(const tm_ExitCode_t exit_code) {
diff --git a/src/bin/hmc_tm.c b/src/bin/hmc_tm.c
@@ -113,11 +113,6 @@ int main(int argc, char *argv[]) {
 
   init_critical_globals(TM_PROGRAM_HMC_TM);
 
-#ifdef TM_KOJAK_INST
-#pragma pomp inst init
-#pragma pomp inst begin(main)
-#endif
-
   strcpy(gauge_filename, "conf.save");
   strcpy(nstore_filename, "nstore_counter");
   strcpy(tmp_filename, ".conf.tmp");
@@ -591,9 +586,6 @@ int main(int argc, char *argv[]) {
 #endif
 
   return (0);
-#ifdef TM_KOJAK_INST
-#pragma pomp inst end(main)
-#endif
 }
 
 static void usage(const tm_ExitCode_t exit_code) {
diff --git a/src/bin/invert.c b/src/bin/invert.c
@@ -114,11 +114,6 @@ int main(int argc, char *argv[]) {
 
   init_critical_globals(TM_PROGRAM_INVERT);
 
-#ifdef TM_KOJAK_INST
-#pragma pomp inst init
-#pragma pomp inst begin(main)
-#endif
-
   DUM_DERI = 8;
   DUM_MATRIX = DUM_DERI + 5;
   NO_OF_SPINORFIELDS = DUM_MATRIX + 4;
@@ -457,9 +452,6 @@ int main(int argc, char *argv[]) {
   MPI_Finalize();
 #endif
   return (0);
-#ifdef TM_KOJAK_INST
-#pragma pomp inst end(main)
-#endif
 }
 
 static void usage(tm_ExitCode_t exit_code) {
diff --git a/src/bin/offline_measurement.c b/src/bin/offline_measurement.c
@@ -83,11 +83,6 @@ int main(int argc, char *argv[]) {
 
   init_critical_globals(TM_PROGRAM_OFFLINE_MEASUREMENT);
 
-#ifdef TM_KOJAK_INST
-#pragma pomp inst init
-#pragma pomp inst begin(main)
-#endif
-
   DUM_DERI = 8;
   DUM_MATRIX = DUM_DERI + 5;
   NO_OF_SPINORFIELDS = DUM_MATRIX + 3;
@@ -306,10 +301,6 @@ int main(int argc, char *argv[]) {
   MPI_Finalize();
 #endif
   return (0);
-
-#ifdef TM_KOJAK_INST
-#pragma pomp inst end(main)
-#endif
 }
 
 static void usage(const tm_ExitCode_t exit_code) {
diff --git a/src/lib/deriv_Sb.c b/src/lib/deriv_Sb.c
@@ -83,10 +83,6 @@ void deriv_Sb(const int ieo, spinor* const l, spinor* const k, hamiltonian_field
 
 #ifdef TM_USE_OMP
 #undef static
-#endif
-
-#ifdef TM_KOJAK_INST
-#pragma pomp inst begin(derivSb)
 #endif
 
     if (ieo == 0) {
@@ -292,7 +288,4 @@ void deriv_Sb(const int ieo, spinor* const l, spinor* const k, hamiltonian_field
   } /* OpenMP closing brace */
 #endif
   tm_stopwatch_pop(&g_timers, 0, 1, "");
-#ifdef TM_KOJAK_INST
-#pragma pomp inst end(derivSb)
-#endif
 }
diff --git a/src/lib/deriv_Sb_D_psi.c b/src/lib/deriv_Sb_D_psi.c
@@ -61,10 +61,6 @@ void deriv_Sb_D_psi(spinor* const l, spinor* const k, hamiltonian_field_t* const
 
 #ifdef TM_USE_OMP
 #undef static
-#endif
-
-#ifdef TM_KOJAK_INST
-#pragma pomp inst begin(derivSb)
 #endif
 
     /************** loop over all lattice sites ****************/
@@ -225,9 +221,6 @@ void deriv_Sb_D_psi(spinor* const l, spinor* const k, hamiltonian_field_t* const
 
       /****************** end of loop ************************/
     }
-#ifdef TM_KOJAK_INST
-#pragma pomp inst end(derivSb)
-#endif
 
 #ifdef TM_USE_OMP
   } /*OpenMP closing brace */
diff --git a/src/lib/get_rectangle_staples.c b/src/lib/get_rectangle_staples.c
@@ -34,9 +34,6 @@ void get_rectangle_staples_general(su3 *const v, const int x, const int mu,
                                    const su3 *const *const gf) {
   su3 ALIGN tmp1, tmp2;
   const su3 *a, *b, *c, *d, *e;
-#ifdef TM_KOJAK_INST
-#pragma pomp inst begin(rectstaples)
-#endif
   _su3_zero((*v));
   for (int nu = 0; nu < 4; nu++) {
     if (mu != nu) {
@@ -178,7 +175,4 @@ void get_rectangle_staples_general(su3 *const v, const int x, const int mu,
       _su3_times_su3_acc((*v), tmp2, tmp1);
     }
   }
-#ifdef TM_KOJAK_INST
-#pragma pomp inst end(rectstaples)
-#endif
 }
diff --git a/src/lib/get_staples.c b/src/lib/get_staples.c
@@ -35,10 +35,6 @@ void get_staples(su3* const staple, const int x, const int mu, const su3** in_ga
   su3 ALIGN st;
   const su3 *w1, *w2, *w3;
 
-#ifdef TM_KOJAK_INST
-#pragma pomp inst begin(staples)
-#endif
-
   _su3_zero(*staple);
   for (int k = 0; k < 4; k++) {
     if (k != mu) {
@@ -61,9 +57,6 @@ void get_staples(su3* const staple, const int x, const int mu, const su3** in_ga
       _su3d_times_su3_acc(*staple, *w1, st);
     }
   }
-#ifdef TM_KOJAK_INST
-#pragma pomp inst end(staples)
-#endif
 }
 
 void get_spacelike_staples(su3* const staple, const int x, const int mu,
@@ -72,10 +65,6 @@ void get_spacelike_staples(su3* const staple, const int x, const int mu,
   su3 ALIGN st;
   const su3 *w1, *w2, *w3;
 
-#ifdef TM_KOJAK_INST
-#pragma pomp inst begin(staples)
-#endif
-
   _su3_zero(*staple);
   for (int k = 1; k < 4; k++) {
     if (k != mu) {
@@ -98,9 +87,6 @@ void get_spacelike_staples(su3* const staple, const int x, const int mu,
       _su3d_times_su3_acc(*staple, *w1, st);
     }
   }
-#ifdef TM_KOJAK_INST
-#pragma pomp inst end(staples)
-#endif
 }
 
 void get_timelike_staples(su3* const staple, const int x, const int mu,
@@ -109,10 +95,6 @@ void get_timelike_staples(su3* const staple, const int x, const int mu,
   su3 ALIGN st;
   const su3 *w1, *w2, *w3;
 
-#ifdef TM_KOJAK_INST
-#pragma pomp inst begin(staples)
-#endif
-
   _su3_zero(*staple);
   int k = 0;
   if (k != mu) {
@@ -134,7 +116,4 @@ void get_timelike_staples(su3* const staple, const int x, const int mu,
     /* v = v + w1^d * st */
     _su3d_times_su3_acc(*staple, *w1, st);
   }
-#ifdef TM_KOJAK_INST
-#pragma pomp inst end(staples)
-#endif
 }
diff --git a/src/lib/operator/Hopping_Matrix_nocom.c b/src/lib/operator/Hopping_Matrix_nocom.c
@@ -48,8 +48,5 @@
 
 #define Hopping_Matrix Hopping_Matrix_nocom
 #define _NO_COMM 1
-#ifdef TM_KOJAK_INST
-#undef TM_KOJAK_INST
-#endif
 
 #include "Hopping_Matrix.c"
diff --git a/src/lib/operator/halfspinor_body.c b/src/lib/operator/halfspinor_body.c
@@ -30,10 +30,6 @@ halfspinor* restrict* phi ALIGN;
 halfspinor32* restrict* phi32 ALIGN;
 _declare_hregs();
 
-#ifdef TM_KOJAK_INST
-#pragma pomp inst begin(hoppingmatrix)
-#endif
-
 #ifndef TM_USE_OMP
 s = k;
 _prefetch_spinor(s);
@@ -320,6 +316,3 @@ if (g_sloppy_precision == 1 && g_sloppy_precision_flag == 1) {
 #endif
   }
 }
-#ifdef TM_KOJAK_INST
-#pragma pomp inst end(hoppingmatrix)
-#endif
diff --git a/src/lib/quda_interface.c b/src/lib/quda_interface.c
@@ -2059,9 +2059,7 @@ void _setQudaMultigridParam(QudaMultigridParam *mg_param) {
 
     // this is needed after QUDA commit
     // https://github.com/lattice/quda/commit/7903288629f0fcc474989fec5a1393ecc17a4b42
-#ifdef TM_QUDA_EXPERIMENTAL
     mg_param->n_vec_batch[level] = 1;
-#endif
 
     // set the MG EigSolver parameters, almost equivalent to
     // setEigParam from QUDA's multigrid_invert_test, except
@@ -3031,7 +3029,6 @@ void quda_mg_tune_params(void *spinorOut, void *spinorIn, const int max_iter) {
   free(tunable_params);
 }
 
-#ifdef TM_QUDA_FERMIONIC_FORCES
 void compute_cloverdet_derivative_quda(monomial *const mnl, hamiltonian_field_t *const hf,
                                        spinor *const X_o, spinor *const phi, int detratio) {
   tm_stopwatch_push(&g_timers, __func__, "");
@@ -3131,23 +3128,6 @@ void compute_ndcloverrat_derivative_quda(monomial *const mnl, hamiltonian_field_
 
   tm_stopwatch_pop(&g_timers, 0, 1, "TM_QUDA");
 }
-#else
-void compute_cloverdet_derivative_quda(monomial *const mnl, hamiltonian_field_t *const hf,
-                                       spinor *const X_o, spinor *const phi, int detratio) {
-  tm_debug_printf(0, 0,
-                  "Error:   UseExternalLibrary = quda requires that tmLQCD is compiled with "
-                  "--enable-quda_fermionic=yes\n");
-  exit(1);
-}
-void compute_ndcloverrat_derivative_quda(monomial *const mnl, hamiltonian_field_t *const hf,
-                                         spinor **const Qup, spinor **const Qdn,
-                                         solver_params_t *solver_params, int detratio) {
-  tm_debug_printf(0, 0,
-                  "Error:   UseExternalLibrary = quda requires that tmLQCD is compiled with "
-                  "--enable-quda_fermionic=yes\n");
-  exit(1);
-}
-#endif
 
 void compute_WFlow_quda(const double eps, const double tmax, const int traj, FILE *outfile) {
   tm_stopwatch_push(&g_timers, __func__, "");
diff --git a/src/lib/test/check_overlap.c b/src/lib/test/check_overlap.c
@@ -105,11 +105,6 @@ int main(int argc, char *argv[]) {
   char *gaugecksum = NULL;
   double plaquette_energy;
 
-#ifdef TM_KOJAK_INST
-#pragma pomp inst init
-#pragma pomp inst begin(main)
-#endif
-
 #ifdef TM_USE_LEMON
   MPI_File fh;
   LemonWriter *lemonWriter;
@@ -389,7 +384,4 @@ int main(int argc, char *argv[]) {
     free_chi_dn_spinor_field();
   }
   return (0);
-#ifdef TM_KOJAK_INST
-#pragma pomp inst end(main)
-#endif
 }
diff --git a/src/lib/update_gauge.c b/src/lib/update_gauge.c
diff --git a/src/lib/xchange/xchange_2fields.c b/src/lib/xchange/xchange_2fields.c
diff --git a/src/lib/xchange/xchange_halffield.c b/src/lib/xchange/xchange_halffield.c
diff --git a/src/lib/xchange/xchange_lexicfield.c b/src/lib/xchange/xchange_lexicfield.c
