Validation mode logging implementation

AndrewBMadison · AndrewBMadison · commit 8f1b4a8fd781 · 2025-02-27T18:41:20.000-08:00
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -36,6 +36,11 @@ if(NOT DEFINED TARGET_ARCH)
         set(TARGET_ARCH 37)
 endif()
 
+#CONDITIONAL FLAG to turn on the validation mode
+if(NOT VALIDATION_MODE)
+        set(VALIDATION_MODE NO)
+endif()
+
 #Creates the Graphitti project with the correct languages, depending on if using GPU or not
 #If using CUDA, also verify the CUDA package and set the required CUDA variables
 if(ENABLE_CUDA)	
@@ -102,7 +107,7 @@ set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -pg")
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_DEBUG}")
     if(ENABLE_CUDA)
-        set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -G")   
+        set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -g -G")   
     endif()
 elseif(CMAKE_BUILD_TYPE STREQUAL "Release")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_RELEASE}")
@@ -150,6 +155,11 @@ if(PERFORMANCE_METRICS)
         add_definitions(-DPERFORMANCE_METRICS)
 endif()
 
+if(VALIDATION_MODE)
+        message("-- Setting VALIDATION_MODE: ON")
+        add_definitions(-DVALIDATION_MODE)
+endif()
+
 #HDF5 Support, finds HDF5 package for C and C++ and links the hdf5 libraries to the executable     \
 #                  later in the file.
 find_package(HDF5 COMPONENTS C CXX)
@@ -528,4 +538,5 @@ unset(PERFORMANCE_METRICS CACHE)
 unset(GPROF CACHE)
 unset(CMAKE_BUILD_TYPE CACHE)
 unset(NVTX_LIBRARY CACHE)
-unset(TARGET_ARCH CACHE)
+unset(TARGET_ARCH CACHE)
+unset(VALIDATION_MODE CACHE)
diff --git a/Simulator/Core/GPUModel.cpp b/Simulator/Core/GPUModel.cpp
@@ -13,7 +13,10 @@
 #include "AllVertices.h"
 #include "Connections.h"
 #include "Global.h"
-
+#ifdef VALIDATION_MODE
+   #include "AllIFNeurons.h"
+   #include "OperationManager.h"
+#endif
 #ifdef PERFORMANCE_METRICS
 float g_time;
 cudaEvent_t start, stop;
@@ -144,19 +147,21 @@ void GPUModel::advance()
    AllVertices &neurons = layout_->getVertices();
    AllEdges &synapses = connections_->getEdges();
 
-   //#ifdef VALIDATION_MODE
-   // int verts = Simulator::getInstance().getTotalVertices();
-   // std::vector<float> randNoise_h(verts);
+#ifdef VALIDATION_MODE
+   int verts = Simulator::getInstance().getTotalVertices();
+   std::vector<float> randNoise_h(verts);
+   for (int i = verts - 1; i >= 0; i--) {
+      randNoise_h[i] = (*noiseRNG)();
+   }
+   //static int testNumbers = 0;
    // for (int i = 0; i < verts; i++) {
-   //    randNoise_h[i] = noiseRNG->rand();
+   //    outFile << "index: " << i << " " << randNoise_h[i] << endl;
    // }
-   // cudaMemcpy(randNoise_d, randNoise_h.data(), verts * sizeof(float), cudaMemcpyHostToDevice);
-   //#endif // VALIDATION_MODE
-
-   //#else VALIDATION_MODE
+   cudaMemcpy(randNoise_d, randNoise_h.data(), verts * sizeof(float), cudaMemcpyHostToDevice);
+#else
    normalMTGPU(randNoise_d);
-   //#endif
-
+#endif
+//LOG4CPLUS_DEBUG(vertexLogger_, "Index: " << index << " Vm: " << Vm);
 #ifdef PERFORMANCE_METRICS
    cudaLapTime(t_gpu_rndGeneration);
    cudaStartTimer();
@@ -167,7 +172,43 @@ void GPUModel::advance()
    dynamic_cast<AllSpikingNeurons &>(neurons).advanceVertices(connections_->getEdges(),
                                                               allVerticesDevice_, allEdgesDevice_,
                                                               randNoise_d, synapseIndexMapDevice_);
+#ifdef VALIDATION_MODE
+   //(AllIFNeuronsDeviceProperties *)allVerticesDevice,
+   log4cplus::Logger vertexLogger_ = log4cplus::Logger::getInstance(LOG4CPLUS_TEXT("vertex"));
+   std::vector<float> sp_h(verts);
+   std::vector<float> vm_h(verts);
+   std::vector<float> Inoise_h(verts);
+   HANDLE_ERROR(cudaMemcpy(sp_h.data(), allVerticesDevice_->summationPoints_, verts * sizeof(float),
+                           cudaMemcpyDeviceToHost));
+   HANDLE_ERROR(
+      cudaMemcpy(sp_h.data(), randNoise_d, verts * sizeof(float), cudaMemcpyDeviceToHost));
+   HANDLE_ERROR(cudaMemcpy(vm_h.data(), ((AllIFNeuronsDeviceProperties *)(allVerticesDevice_))->Vm_,
+                           verts * sizeof(float), cudaMemcpyDeviceToHost));
+   HANDLE_ERROR(cudaMemcpy(Inoise_h.data(),
+                           ((AllIFNeuronsDeviceProperties *)(allVerticesDevice_))->Inoise_,
+                           verts * sizeof(float), cudaMemcpyDeviceToHost));
+   HANDLE_ERROR(cudaMemcpy(sp_h.data(), allVerticesDevice_->spValidation_, verts * sizeof(float),
+                           cudaMemcpyDeviceToHost));
 
+   for (int i = 0; i < verts; i++) {
+      LOG4CPLUS_DEBUG(vertexLogger_, "CUDA advance Index[ "
+                                        << i << "] :: Noise = " << randNoise_h[i]
+                                        << "\tVm: " << vm_h[i] << endl
+                                        << "\tsp = " << sp_h[i] << endl
+                                        << "\tInoise = " << Inoise_h[i] << endl);
+   }
+#endif
+//LOG4CPLUS_DEBUG(vertexLogger_, "ADVANCE NEURON LIF[" << index << "] :: Noise = " << noise);
+//LOG4CPLUS_DEBUG(vertexLogger_, "Index: " << index << " Vm: " << Vm);
+// LOG4CPLUS_DEBUG(vertexLogger_, "NEURON[" << index << "] {" << endl
+//                                          << "\tVm = " << Vm << endl
+//                                          << "\tVthresh = " << Vthresh << endl
+//                                          << "\tsummationPoint = " << summationPoint << endl
+//                                          << "\tI0 = " << I0 << endl
+//                                          << "\tInoise = " << Inoise << endl
+//                                          << "\tC1 = " << C1 << endl
+//                                          << "\tC2 = " << C2 << endl
+//                                          << "}" << endl);
 #ifdef PERFORMANCE_METRICS
    cudaLapTime(t_gpu_advanceNeurons);
    cudaStartTimer();
diff --git a/Simulator/Core/GPUModel.h b/Simulator/Core/GPUModel.h
@@ -38,6 +38,11 @@
 #include "AllSpikingNeurons.h"
 #include "AllSpikingSynapses.h"
 
+#ifdef VALIDATION_MODE
+   #include <fstream>
+   #include <iostream>
+#endif   // VALIDATION_MODE
+
 #ifdef __CUDACC__
    #include "Book.h"
 #endif
diff --git a/Simulator/Core/Simulator.cpp b/Simulator/Core/Simulator.cpp
@@ -31,6 +31,7 @@ Simulator::Simulator()
 
    consoleLogger_ = log4cplus::Logger::getInstance(LOG4CPLUS_TEXT("console"));
    fileLogger_ = log4cplus::Logger::getInstance(LOG4CPLUS_TEXT("file"));
+   fileLogger_.setLogLevel(log4cplus::DEBUG_LOG_LEVEL);
    edgeLogger_ = log4cplus::Logger::getInstance(LOG4CPLUS_TEXT("edge"));
    workbenchLogger_ = log4cplus::Logger::getInstance(LOG4CPLUS_TEXT("workbench"));
 
diff --git a/Simulator/Vertices/AllVertices.cpp b/Simulator/Vertices/AllVertices.cpp
@@ -25,6 +25,7 @@ AllVertices::AllVertices() : size_(0)
    // Get a copy of the file and vertex logger to use log4cplus macros to print to debug files
    fileLogger_ = log4cplus::Logger::getInstance(LOG4CPLUS_TEXT("file"));
    vertexLogger_ = log4cplus::Logger::getInstance(LOG4CPLUS_TEXT("vertex"));
+   vertexLogger_.setLogLevel(log4cplus::DEBUG_LOG_LEVEL);
 }
 
 ///  Setup the internal structure of the class (allocate memories).
diff --git a/Simulator/Vertices/AllVertices.h b/Simulator/Vertices/AllVertices.h
@@ -155,6 +155,9 @@ struct AllVerticesDeviceProperties {
    ///  On the next advance cycle, vertices add the values stored in their corresponding
    ///  summation points to their Vm and resets the summation points to zero
    BGFLOAT *summationPoints_;
+   #ifdef VALIDATION_MODE
+   BGFLOAT *spValidation_;
+   #endif
 };
 #endif   // defined(USE_GPU)
 
diff --git a/Simulator/Vertices/Neuro/AllIFNeurons_d.cpp b/Simulator/Vertices/Neuro/AllIFNeurons_d.cpp
@@ -50,6 +50,9 @@ void AllIFNeurons::allocDeviceStruct(AllIFNeuronsDeviceProperties &allVerticesDe
    HANDLE_ERROR(
       cudaMalloc((void **)&allVerticesDevice.numStepsInRefractoryPeriod_, count * sizeof(int)));
    HANDLE_ERROR(cudaMalloc((void **)&allVerticesDevice.summationPoints_, count * sizeof(BGFLOAT)));
+#ifdef VALIDATION_MODE
+   HANDLE_ERROR(cudaMalloc((void **)&allVerticesDevice.spValidation_, count * sizeof(BGFLOAT)));
+#endif
    HANDLE_ERROR(cudaMalloc((void **)&allVerticesDevice.spikeHistory_, count * sizeof(uint64_t *)));
 
    uint64_t *pSpikeHistory[count];
@@ -107,6 +110,9 @@ void AllIFNeurons::deleteDeviceStruct(AllIFNeuronsDeviceProperties &allVerticesD
    HANDLE_ERROR(cudaFree(allVerticesDevice.hasFired_));
    HANDLE_ERROR(cudaFree(allVerticesDevice.numStepsInRefractoryPeriod_));
    HANDLE_ERROR(cudaFree(allVerticesDevice.summationPoints_));
+#ifdef VALIDATION_MODE
+   HANDLE_ERROR(cudaFree(allVerticesDevice.spValidation_));
+#endif
    HANDLE_ERROR(cudaFree(allVerticesDevice.spikeHistory_));
 }
 
diff --git a/Simulator/Vertices/Neuro/AllIZHNeurons.cpp b/Simulator/Vertices/Neuro/AllIZHNeurons.cpp
@@ -224,17 +224,27 @@ void AllIZHNeurons::advanceNeuron(int index)
    BGFLOAT &u = this->u_[index];
 
    if (nStepsInRefr > 0) {
-      // is neuron refractory?
+   // is neuron refractory?
+   #ifdef VALIDATION_MODE
+      BGFLOAT noise = (*noiseRNG)();
+      LOG4CPLUS_DEBUG(vertexLogger_, "REFRACTORY NEURON IZH[" << index << "] :: Noise = " << noise);
+   #endif
       --nStepsInRefr;
    } else if (Vm >= Vthresh) {
-      // should it fire?
+   // should it fire?
+   #ifdef VALIDATION_MODE
+      BGFLOAT noise = (*noiseRNG)();
+      LOG4CPLUS_DEBUG(vertexLogger_, "FIRE NEURON IZH[" << index << "] :: Noise = " << noise);
+   #endif
       fire(index);
    } else {
       summationPoint += I0;   // add IO
       // add noise
       BGFLOAT noise = (*noiseRNG)();
-      // Happens really often, causes drastic slow down
-      // DEBUG_MID(cout << "ADVANCE NEURON[" << index << "] :: noise = " << noise << endl;)
+   // Happens really often, causes drastic slow down
+   #ifdef VALIDATION_MODE
+      LOG4CPLUS_DEBUG(vertexLogger_, "ADVANCE NEURON IZH[" << index << "] :: Noise = " << noise);
+   #endif
       summationPoint += noise * Inoise;   // add noise
 
       BGFLOAT Vint = Vm * 1000;
diff --git a/Simulator/Vertices/Neuro/AllLIFNeurons.cpp b/Simulator/Vertices/Neuro/AllLIFNeurons.cpp
@@ -33,33 +33,46 @@ void AllLIFNeurons::advanceNeuron(int index)
    int &nStepsInRefr = this->numStepsInRefractoryPeriod_[index];
 
    if (nStepsInRefr > 0) {
-      // is neuron refractory?
+   // is neuron refractory?
+   #ifdef VALIDATION_MODE
+      BGFLOAT noise = (*noiseRNG)();
+      LOG4CPLUS_DEBUG(vertexLogger_, "neuron refractory LIF[" << index << "] :: Noise = " << noise);
+   #endif
       --nStepsInRefr;
    } else if (Vm >= Vthresh) {
-      // should it fire?
+   // should it fire?
+   #ifdef VALIDATION_MODE
+      BGFLOAT noise = (*noiseRNG)();
+      LOG4CPLUS_DEBUG(vertexLogger_, "FIRE NEURON LIF[" << index << "] :: Noise = " << noise);
+   #endif
       fire(index);
    } else {
       summationPoint += I0;   // add IO
       // add noise
       BGFLOAT noise = (*noiseRNG)();
-      //LOG4CPLUS_DEBUG(vertexLogger_, "ADVANCE NEURON[" << index << "] :: Noise = " << noise);
+   #ifdef VALIDATION_MODE
+      LOG4CPLUS_DEBUG(vertexLogger_, "ADVANCE NEURON LIF[" << index << "] :: Noise = " << noise);
+   #endif
       summationPoint += noise * Inoise;     // add noise
       Vm = C1 * Vm + C2 * summationPoint;   // decay Vm and add inputs
    }
-   // clear synaptic input for next time step
-   summationPoint = 0;
 
    // Causes a huge slowdown since it's printed so frequently
-   //    LOG4CPLUS_DEBUG(vertexLogger_, "Index: " << index << " Vm: " << Vm);
-   //    LOG4CPLUS_DEBUG(vertexLogger_, "NEURON[" << index << "] {" << endl
-   //                   << "\tVm = " << Vm << endl
-   //                   << "\tVthresh = " << Vthresh << endl
-   //                   << "\tsummationPoint = " << summationPoint << endl
-   //                   << "\tI0 = " << I0 << endl
-   //                   << "\tInoise = " << Inoise << endl
-   //                   << "\tC1 = " << C1 << endl
-   //                   << "\tC2 = " << C2 << endl
-   //                   << "}" << endl);
+   #ifdef VALIDATION_MODE
+   LOG4CPLUS_DEBUG(vertexLogger_, "Index: " << index << " Vm: " << Vm);
+   LOG4CPLUS_DEBUG(vertexLogger_, "NEURON[" << index << "] {" << endl
+                                            << "\tVm = " << Vm << endl
+                                            << "\tVthresh = " << Vthresh << endl
+                                            << "\tsummationPoint = " << summationPoint << endl
+                                            << "\tI0 = " << I0 << endl
+                                            << "\tInoise = " << Inoise << endl
+                                            << "\tC1 = " << C1 << endl
+                                            << "\tC2 = " << C2 << endl
+                                            << "}" << endl);
+   #endif
+
+   // clear synaptic input for next time step
+   summationPoint = 0;
 }
 
 ///  Fire the selected Neuron and calculate the result.
diff --git a/Simulator/Vertices/Neuro/AllLIFNeurons.h b/Simulator/Vertices/Neuro/AllLIFNeurons.h
@@ -113,7 +113,7 @@ class AllLIFNeurons : public AllIFNeurons {
    ///  @param  edgeIndexMapDevice  GPU address of the EdgeIndexMap on device memory.
    virtual void advanceVertices(AllEdges &synapses, void *allVerticesDevice, void *allEdgesDevice,
                                 float randNoise[], EdgeIndexMapDevice *edgeIndexMapDevice) override;
-#else   // !defined(USE_GPU)
+#else    // !defined(USE_GPU)
 protected:
    ///  Helper for #advanceNeuron. Updates state of a single neuron.
    ///
@@ -124,7 +124,6 @@ class AllLIFNeurons : public AllIFNeurons {
    ///
    ///  @param  index Index of the neuron to fire.
    virtual void fire(int index);
-
 #endif   // defined(USE_GPU)
 };
 
diff --git a/Simulator/Vertices/Neuro/AllLIFNeurons_d.cpp b/Simulator/Vertices/Neuro/AllLIFNeurons_d.cpp
@@ -181,7 +181,9 @@ __global__ void advanceLIFNeuronsDevice(int totalVertices, int maxEdges, int max
       vm = allVerticesDevice->C1_[idx] * r_vm
            + allVerticesDevice->C2_[idx] * (r_sp);   // decay Vm and add inputs
    }
-
+#ifdef VALIDATION_MODE
+   allVerticesDevice->spValidation_[idx] = r_sp;
+#endif
    // clear synaptic input for next time step
    sp = 0;
 }

Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,7 @@ AllVertices::AllVertices() : size_(0)`
`25`	`25`	`// Get a copy of the file and vertex logger to use log4cplus macros to print to debug files`
`26`	`26`	`fileLogger_ = log4cplus::Logger::getInstance(LOG4CPLUS_TEXT("file"));`
`27`	`27`	`vertexLogger_ = log4cplus::Logger::getInstance(LOG4CPLUS_TEXT("vertex"));`
	`28`	`+ vertexLogger_.setLogLevel(log4cplus::DEBUG_LOG_LEVEL);`
`28`	`29`	`}`
`29`	`30`
`30`	`31`	`/// Setup the internal structure of the class (allocate memories).`