Improves the CI tests for reporting (#479)

sergiorg-hpc · Blanco Alonso Jorge · web-flow · commit 58306a00f998 · 2021-01-27T17:28:13.000+01:00
* Added reference files for better report comparison.
* Renamed cmake variables to increase readability.
* Enabled libsonata tests in reports.

Co-authored-by: Blanco Alonso Jorge &lt;jblanco@bbpv2.epfl.ch&gt;
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -204,22 +204,27 @@ if(CORENRN_ENABLE_REPORTING)
   find_package(reportinglib)
   find_package(sonata)
   find_program(H5DUMP_EXECUTABLE h5dump)
-  if(NOT reportinglib_FOUND AND NOT sonata_FOUND)
-    message(SEND_ERROR "Neither reportinglib nor SONATA libraries were found")
-  elseif(reportinglib_FOUND AND sonata_FOUND)
-    add_definitions("-DENABLE_BIN_REPORTS")
-    add_definitions("-DENABLE_SONATA_REPORTS")
-  elseif(reportinglib_FOUND)
+
+  if(reportinglib_FOUND)
     add_definitions("-DENABLE_BIN_REPORTS")
-  else() # sonata_FOUND
+    set(ENABLE_BIN_REPORTS_TESTS ON)
+  else()
+    set(reportinglib_INCLUDE_DIR "")
+    set(reportinglib_LIBRARY "")
+  endif()
+  if(sonata_FOUND)
     if(TARGET sonata::sonata_report)
       add_definitions("-DENABLE_SONATA_REPORTS")
-      set(reportinglib_INCLUDE_DIR "")
-      set(reportinglib_LIBRARY "")
+      set(ENABLE_SONATA_REPORTS_TESTS ON)
     else()
       message(SEND_ERROR "SONATA library was found but without reporting support")
     endif()
   endif()
+
+  if(NOT reportinglib_FOUND AND NOT sonata_FOUND)
+    message(SEND_ERROR "Neither reportinglib nor SONATA libraries were found")
+  endif()
+
   include_directories(${reportinglib_INCLUDE_DIR})
   include_directories(${sonatareport_INCLUDE_DIR})
 endif()
diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt
@@ -101,6 +101,7 @@ if(CORENRN_ENABLE_REPORTING)
     configure_file(reportinglib/${TEST_NAME}.conf.in ${SIM_NAME}/${TEST_NAME}.conf @ONLY)
     configure_file(reportinglib/reporting_test.sh.in ${SIM_NAME}/reporting_test.sh @ONLY)
     configure_file(reportinglib/${TEST_NAME}.check.in ${SIM_NAME}/${TEST_NAME}.check @ONLY)
+    file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/reportinglib/test_ref.out" DESTINATION "${SIM_NAME}/")
     add_test(
       NAME ${SIM_NAME}
       COMMAND "/bin/sh" ${CMAKE_CURRENT_BINARY_DIR}/${SIM_NAME}/reporting_test.sh
diff --git a/tests/integration/integration_test.sh.in b/tests/integration/integration_test.sh.in
@@ -1,6 +1,7 @@
 #! /bin/sh
 
 export OMP_NUM_THREADS=1
+export LIBSONATA_ZERO_BASED_GIDS=true
 
 # Run the executable
 SRUN_EXTRA=
@@ -23,18 +24,18 @@ fi
 cd @CMAKE_CURRENT_BINARY_DIR@/@SIM_NAME@
 
 # We convert spikes to out.dat format
-reports=@ENABLE_SONATA_REPORTS@
+reports=@ENABLE_SONATA_REPORTS_TESTS@
 if [ "$reports" = "ON" ]
 then
-  data=$(@H5DUMP_EXECUTABLE@ -d /spikes/timestamps -d /spikes/node_ids -y -O out.h5 | tr "," "\n")
-  echo $data | awk '{n=NF/2; for (i=1;i<=n;i++) print $i "\t" $(n+i) }' >> out_SONATA.dat
+  data=$(@H5DUMP_EXECUTABLE@ -d /spikes/All/timestamps -d /spikes/All/node_ids -y -O out.h5 | sed 's/"ms"//g;s/,/\n/g')
+  echo $data | awk '{n=NF/2; for (i=1;i<=n;i++) print $i "\t" $(n+i) }' > out_SONATA.dat
 
   if [ ! -f out_SONATA.dat ]
   then
     echo "[ERROR] No SONATA output files. Test failed!" >&2
     exit 1
   fi
-  diff -w out_SONATA.dat @CMAKE_CURRENT_SOURCE_DIR@/@SIM_NAME@/out.dat.ref > diff_SONATA.dat 2>&1
+  diff -w out_SONATA.dat out.dat.ref > diff_SONATA.dat 2>&1
   if [ -s diff_SONATA.dat ]
   then
     echo "[ERROR] SONATA Results are different, check the file diff_SONATA.dat. Test failed!" >&2
@@ -48,7 +49,7 @@ then
   exit 1
 fi
 
-diff -w out.dat @CMAKE_BINARY_DIR@/tests/integration/@SIM_NAME@/out.dat.ref > diff.dat 2>&1
+diff -w out.dat out.dat.ref > diff.dat 2>&1
 
 if [ -s diff.dat ]
 then
diff --git a/tests/integration/negative_integration_test.sh.in b/tests/integration/negative_integration_test.sh.in
@@ -2,6 +2,7 @@
 
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:@CMAKE_BINARY_DIR@/lib
 export OMP_NUM_THREADS=1
+export LIBSONATA_ZERO_BASED_GIDS=true
 
 # Run the executable
 #SRUN_EXTRA="--propagate=CORE"
@@ -25,11 +26,11 @@ fi
 cd @CMAKE_CURRENT_BINARY_DIR@/@SIM_NAME@
 
 # We convert spikes to out.dat format
-reports=@ENABLE_SONATA_REPORTS@
+reports=@ENABLE_SONATA_REPORTS_TESTS@
 if [ "$reports" = "ON" ]
 then
-  data=$(@H5DUMP_EXECUTABLE@ -d /spikes/timestamps -d /spikes/node_ids -y -O spikes | tr "," "\n")
-  echo $data | awk '{n=NF/2; for (i=1;i<=n;i++) print $i "\t" $(n+i) }' >> out_SONATA.dat
+  data=$(@H5DUMP_EXECUTABLE@ -d /spikes/All/timestamps -d /spikes/All/node_ids -y -O spikes | sed 's/"ms"//g;s/,/\n/g')
+  echo $data | awk '{n=NF/2; for (i=1;i<=n;i++) print $i "\t" $(n+i) }' > out_SONATA.dat
 
   if [ ! -f out_SONATA.dat ]
   then
diff --git a/tests/integration/reportinglib/1.check.in b/tests/integration/reportinglib/1.check.in
@@ -1,27 +1,22 @@
-#! /bin/sh
-
-#NOTE: binary diff is not a good idea as it dont reflect what people use to read reports
+#!/bin/sh
 
 OK=0
 FAILED=1
-sonata_reports=@ENABLE_SONATA_REPORTS@
-bin_reports=@ENABLE_BIN_REPORTS@
+sonata_reports=@ENABLE_SONATA_REPORTS_TESTS@
+bin_reports=@ENABLE_BIN_REPORTS_TESTS@
+test_ref=@CMAKE_CURRENT_BINARY_DIR@/@SIM_NAME@/test_ref.out
 
 if [ "$bin_reports" = "ON" ]
 then
   if [ -f test_1.bbp ]
   then
-     #checking something with @reportinglib_BIN_DIR@/dump test_1.bbp
-     # for ascii files that will be using grep for instance.
-     nb_recorded=$(@reportinglib_somaDump@ test_1.bbp 1| wc -l)
-     # 101 because a white space
-     if [ $nb_recorded -eq 100 ]; then
-       exit $OK
-     else
-       echo "[ERROR] The number of steps recorded by ReportingLib doesn't match.
-             Found $nb_recorded, expected 100. Test failed!" >&2
-       exit $FAILED
-     fi
+    somaDump_diff=$(@reportinglib_somaDump@ test_1.bbp 1 | sed 's/ //g' | diff $test_ref -)
+    
+    if [ $? -ne 0 ]
+    then
+      echo -e "[ERROR] The report output generated by Reportinglib differs!\n$somaDump_diff" >&2
+      exit $FAILED
+    fi
   else
      echo "[ERROR] Expected ReportingLib soma file 'test_1.bbp' is missing. Test failed!" >&2
      exit $FAILED
@@ -32,19 +27,18 @@ if [ "$sonata_reports" = "ON" ]
 then
   if [ -f test_2.h5 ]
   then
-     #checking something with @reportinglib_BIN_DIR@/dump test_1.bbp
-     # for ascii files that will be using grep for instance.
-     nb_recorded_sonata=$(@H5DUMP_EXECUTABLE@ -d /report/data -y -O test_2.h5 | wc -l)
-     # 101 because a white space
-     if [ $nb_recorded_sonata -eq 101 ]; then
-       exit $OK
-     else
-       echo "[ERROR] The number of steps recorded by SONATA doesn't match.
-            Found $nb_recorded, expected 101. Test failed!" >&2
-       exit $FAILED
-     fi
+    h5dump_diff=$(@H5DUMP_EXECUTABLE@ -d /report/All/data -y -O test_2.h5 | sed '1d;$d;s/,//g;s/ //g' | diff $test_ref -)
+    
+    if [ $? -ne 0 ]
+    then
+      echo -e "[ERROR] The report output generated by Libsonata differs!\n$h5dump_diff" >&2
+      exit $FAILED
+    fi
   else
      echo "[ERROR] Expected SONATA soma file 'test_2.h5' doesn't exist. Test failed!" >&2
      exit $FAILED
   fi
 fi
+
+# If we reach this point, all tests were successful
+exit $OK
diff --git a/tests/integration/reportinglib/reporting_test.sh.in b/tests/integration/reportinglib/reporting_test.sh.in
@@ -3,6 +3,8 @@
 set -e -o pipefail
 
 export OMP_NUM_THREADS=1
+export LIBSONATA_ZERO_BASED_GIDS=true
+
 @SRUN_PREFIX@ @CMAKE_BINARY_DIR@/bin/@CMAKE_SYSTEM_PROCESSOR@/special-core --mpi --read-config @CMAKE_CURRENT_BINARY_DIR@/@SIM_NAME@/@TEST_NAME@.conf
 chmod +x @CMAKE_CURRENT_BINARY_DIR@/@SIM_NAME@/@TEST_NAME@.check
 exit `@CMAKE_CURRENT_BINARY_DIR@/@SIM_NAME@/@TEST_NAME@.check`
diff --git a/tests/integration/reportinglib/test_ref.out b/tests/integration/reportinglib/test_ref.out
@@ -0,0 +1,100 @@
+-65
+-64.9973
+-64.9951
+-64.9932
+-64.9916
+-64.9902
+-64.9889
+-64.9877
+-64.9867
+-64.9858
+-64.985
+-64.9842
+-64.9836
+-64.9829
+-64.9824
+-64.9819
+-64.9815
+-64.9811
+-64.9807
+-64.9804
+-64.9802
+-64.9799
+-64.9797
+-64.9796
+-64.9794
+-64.9793
+-64.9792
+-64.9791
+-64.979
+-64.979
+-64.979
+-64.979
+-64.979
+-64.979
+-64.979
+-64.9791
+-64.9791
+-64.7371
+-63.6264
+-62.1068
+-60.4682
+-58.847
+-57.2905
+-55.7913
+-54.3056
+-52.7594
+-51.044
+-48.9961
+-46.3491
+-42.6233
+-36.8741
+-27.1665
+-10.1852
+13.977
+31.4561
+36.143
+35.2487
+32.4239
+28.6338
+24.2472
+19.4933
+14.5405
+9.51339
+4.50006
+-0.440951
+-5.27461
+-9.98373
+-14.5648
+-19.0258
+-23.3868
+-27.6838
+-31.9759
+-36.353
+-40.9401
+-45.8855
+-51.303
+-57.1176
+-62.8313
+-67.5469
+-70.6416
+-72.2969
+-73.0829
+-73.4434
+-73.6102
+-73.6866
+-73.7171
+-73.7212
+-73.7082
+-73.6828
+-73.6479
+-73.6053
+-73.5561
+-73.5012
+-73.4414
+-73.3771
+-73.3089
+-73.237
+-73.1618
+-73.0836
+-73.0025

-Original file line number
+Diff line change
 set -e -o pipefail
 export OMP_NUM_THREADS=1
 +export LIBSONATA_ZERO_BASED_GIDS=true
++
 @SRUN_PREFIX@ @CMAKE_BINARY_DIR@/bin/@CMAKE_SYSTEM_PROCESSOR@/special-core --mpi --read-config @CMAKE_CURRENT_BINARY_DIR@/@SIM_NAME@/@[email protected]
 chmod +x @CMAKE_CURRENT_BINARY_DIR@/@SIM_NAME@/@[email protected]
 exit `@CMAKE_CURRENT_BINARY_DIR@/@SIM_NAME@/@[email protected]`