Merge pull request #702 from UWB-Biocomputing/issue-697-update-serializer-class

d-kamath · web-flow · commit 095537e169bb · 2024-09-16T09:50:44.000-07:00
Cleanup the Serialization support
diff --git a/Simulator/Connections/Connections.cpp b/Simulator/Connections/Connections.cpp
@@ -103,31 +103,3 @@ void Connections::updateSynapsesWeights()
 {
 }
 #endif   // !USE_GPU
-
-///  Creates synapses from synapse weights saved in the serialization file.
-void Connections::createSynapsesFromWeights()
-{
-   int numVertices = Simulator::getInstance().getTotalVertices();
-   Layout &layout = Simulator::getInstance().getModel().getLayout();
-   AllVertices &vertices = layout.getVertices();
-
-   // for each neuron
-   for (int i = 0; i < numVertices; i++) {
-      // for each synapse in the vertex
-      for (BGSIZE synapseIndex = 0; synapseIndex < Simulator::getInstance().getMaxEdgesPerVertex();
-           synapseIndex++) {
-         BGSIZE iEdg = Simulator::getInstance().getMaxEdgesPerVertex() * i + synapseIndex;
-         // if the synapse weight is not zero (which means there is a connection), create the synapse
-         if (edges_->W_[iEdg] != 0.0) {
-            BGFLOAT theW = edges_->W_[iEdg];
-            int srcVertex = edges_->sourceVertexIndex_[iEdg];
-            int destVertex = edges_->destVertexIndex_[iEdg];
-            edgeType type = layout.edgType(srcVertex, destVertex);
-            edges_->edgeCounts_[i]++;
-            edges_->createEdge(iEdg, srcVertex, destVertex, Simulator::getInstance().getDeltaT(),
-                               type);
-            edges_->W_[iEdg] = theW;
-         }
-      }
-   }
-}
diff --git a/Simulator/Connections/Connections.h b/Simulator/Connections/Connections.h
@@ -73,9 +73,6 @@ class Connections {
    ///  @return true if successful, false otherwise.
    virtual bool updateConnections(AllVertices &vertices);
 
-   ///  Creates synapses from synapse weights saved in the serialization file.
-   void createSynapsesFromWeights();
-
    ///  Cereal serialization method
    template <class Archive> void serialize(Archive &archive, std::uint32_t const version);
 
diff --git a/Simulator/Core/Core.cpp b/Simulator/Core/Core.cpp
@@ -184,7 +184,7 @@ int Core::runSimulation(string executableName, string cmdLineArguments)
       LOG4CPLUS_TRACE(consoleLogger, "Deserializing state from file.");
 
       // Deserialization
-      if (!serializer.deserializeSynapses()) {
+      if (!serializer.deserialize()) {
          LOG4CPLUS_FATAL(consoleLogger, "Failed while deserializing objects");
          return -1;
       }
@@ -214,7 +214,7 @@ int Core::runSimulation(string executableName, string cmdLineArguments)
    // Serializes internal state for the current simulation
    if (!simulator.getSerializationFileName().empty()) {
       LOG4CPLUS_TRACE(consoleLogger, "Serializing current state");
-      serializer.serializeSynapses();
+      serializer.serialize();
    }
 
    // Tell simulation to clean-up and run any post-simulation logic.
diff --git a/Simulator/Core/Serializer.cpp b/Simulator/Core/Serializer.cpp
@@ -3,31 +3,43 @@
  *
  * @ingroup Simulator/Core
  * 
- * @brief Handles implementation details of serialization and deserialization of synapses.
+ * @brief Provides serialization and deserialization functionality using the Cereal library.
  * 
- *  Serialize and Deserialize synapse weights, source vertices, destination vertices,
- *  maxEdgesPerVertex, totalVertices and radii.
+ * This class handles the serialization and deserialization of all member variables 
+ * in the Connections, Layout, Edges, Vertices, and associated helper classes such as
+ * EdgeIndexMap, Model, RecordableBase, RecordableVector, Matrix, RNG and EventBuffer.
+ * Note that Recorder class is not serialized or deserialized.
+ * 
+ * The serialization and deserialization process typically begins with the Model class,
+ * which internally calls the serialization of the Connections and Layout classes.
+ * Connections, in turn, handle the serialization of Edges, while Layout handles 
+ * the serialization of Vertices. This ensures a comprehensive serialization of 
+ * the entire simulation structure.
+ * 
+ * @note As of September 2024, serialization support is currently available
+ * for CPU-based Neuron simulations. While GPU-based Neuron serialization is functional, 
+ * the output result files differ, and this is being addressed in [Issue #701].
+ * Serialization support for NG911 will be extended in the future [Issue #700].
+ *  
  */
 
 #include "Serializer.h"
 #include "ConnGrowth.h"
-#include "Connections.h"
 #include "GPUModel.h"
-#include "Simulator.h"
 #include <fstream>
 
-// Displays Graphitti as top most element instead of the default Cereal
-// CEREAL_XML_STRING_VALUE should be placed before defining cereal archives library
+// About CEREAL_XML_STRING_VALUE
+// 1. Displays Graphitti as top most element instead of the default Cereal in the serialized xml file.
+// 2. It should be placed before defining cereal archives library
 #define CEREAL_XML_STRING_VALUE "Graphitti"
 #include <cereal/archives/binary.hpp>
 #include <cereal/archives/xml.hpp>
 
-///  Deserializes synapse weights, source vertices, destination vertices,
-///  maxEdgesPerVertex, totalVertices.
-///  if running a connGrowth model and radii is in serialization file, deserializes radii as well
+/// Deserializes all member variables of the
+/// Connections, Layout, Edges, Vertices, and associated helper classes.
 ///
 ///  @returns    true if successful, false otherwise.
-bool Serializer::deserializeSynapses()
+bool Serializer::deserialize()
 {
    Simulator &simulator = Simulator::getInstance();
 
@@ -47,59 +59,24 @@ bool Serializer::deserializeSynapses()
    cereal::XMLInputArchive archive(memory_in);
    //cereal::BinaryInputArchive archive(memory_in);
 
-   Connections &connections = simulator.getModel().getConnections();
-   //Layout &layout = simulator.getModel().getLayout();
-
-   // Deserializes synapse weights along with each synapse's source vertex and destination vertex
-   // Uses "try catch" to catch any cereal exception
-   // try {
-   //    archive(dynamic_cast<AllEdges &>(connections.getEdges()));
-   // } catch (cereal::Exception e) {
-   //    cerr << e.what() << endl
-   //         << "Failed to deserialize synapse weights, source vertices, and/or destination vertices."
-   //         << endl;
-   //    return false;
-   // }
-
-   // Creates synapses from weight
-   //    connections->createSynapsesFromWeights(simulator.getTotalVertices(), layout.get(),
-   //                                           (layout->getVertices()), (connections->getEdges()));
-
-
-   // Uses "try catch" to catch any cereal exception
-   try {
-      archive(connections);
-   } catch (cereal::Exception e) {
-      cerr << e.what() << endl << "Failed to deserialize radii." << endl;
+   if (!processArchive(archive, simulator)) {
+      cerr << "Failed to deserialize" << endl;
       return false;
    }
 
-   //Creates synapses from weight
-   // connections.createSynapsesFromWeights();
-
-
-#if defined(USE_GPU)
-   // Copies CPU Synapse data to GPU after deserialization, if we're doing
-   // a GPU-based simulation.
-   simulator.copyCPUSynapseToGPU();
-#endif   // USE_GPU
-
-   // Creates synapse index map (includes copy CPU index map to GPU)
-   // connections.createEdgeIndexMap();
 
 #if defined(USE_GPU)
    GPUModel &gpuModel = static_cast<GPUModel &>(simulator.getModel());
-   gpuModel.copySynapseIndexMapHostToDevice(connections.getEdgeIndexMap(),
+   gpuModel.copySynapseIndexMapHostToDevice(simulator.getModel().getConnections().getEdgeIndexMap(),
                                             simulator.getTotalVertices());
 #endif   // USE_GPU
 
    return true;
 }
 
-///  Serializes synapse weights, source vertices, destination vertices,
-///  maxEdgesPerVertex, totalVertices.
-///  if running a connGrowth model serializes radii as well.
-void Serializer::serializeSynapses()
+/// Serializes all member variables of the
+/// Connections, Layout, Edges, Vertices, and associated helper classes.
+void Serializer::serialize()
 {
    Simulator &simulator = Simulator::getInstance();
 
@@ -108,29 +85,27 @@ void Serializer::serializeSynapses()
    ofstream memory_out(simulator.getSerializationFileName().c_str());
    cout << "Please find the serialized file in " << simulator.getSerializationFileName().c_str();
 
-   // Options parameter are optional which sets
-   // 1. Sets the Preceision of floating point number to 30
-   // 2. Keeps indedntaion in XML file,
-   // 3. Displays output type of element values in XML eg. float
-   // 4. Displays if the size of a node in XML is dynamic or not.
-   cereal::XMLOutputArchive archive(memory_out, cereal::XMLOutputArchive::Options()
-                                                   .precision(30)
-                                                   .indent(true)
-                                                   .outputType(false)
-                                                   .sizeAttributes(true));
+   cereal::XMLOutputArchive archive(memory_out);
    //ofstream memory_out (simInfo->memOutputFileName.c_str(), std::ios::binary);
    //cereal::BinaryOutputArchive archive(memory_out);
 
-#if defined(USE_GPU)
-   // Copies GPU Synapse props data to CPU for serialization
-   simulator.copyGPUSynapseToCPU();
-#endif   // USE_GPU
-
-   Model &model = simulator.getModel();
-
-   // Serializes synapse weights along with each synapse's source vertex and destination vertex
-   // archive(
-   // cereal::make_nvp("AllEdges", dynamic_cast<AllEdges &>(model.getConnections().getEdges())));
+   if (!processArchive(archive, simulator)) {
+      cerr << "Failed to serialize" << endl;
+   }
+}
 
-   archive(cereal::make_nvp("Connections", model.getConnections()));
+template <typename Archive> bool Serializer::processArchive(Archive &archive, Simulator &simulator)
+{
+   try {
+      // Starts the serialization/deserialization process from the Model class.
+      // Note that the Model object gets sliced, and only
+      // the `serialize` function of the base Model class is called.
+      archive(simulator.getModel());
+      // Serialize/Deserialize required global variables
+      archive(initRNG, noiseRNG, g_simulationStep);
+   } catch (cereal::Exception e) {
+      cerr << e.what() << endl;
+      return false;
+   }
+   return true;
 }
diff --git a/Simulator/Core/Serializer.h b/Simulator/Core/Serializer.h
@@ -3,26 +3,43 @@
  *
  * @ingroup Simulator/Core
  * 
- * @brief Handles implementation of serialization and deserialization of synapses.
+ * @brief Provides serialization and deserialization functionality using the Cereal library.
  * 
- *  Serializes and Deserializes synapse weights, source vertices, destination vertices,
- *  maxEdgesPerVertex, totalVertices and radii.
+ * This class handles the serialization and deserialization of all member variables 
+ * in the Connections, Layout, Edges, Vertices, and associated helper classes such as
+ * EdgeIndexMap, Model, RecordableBase, RecordableVector, Matrix, RNG and EventBuffer.
+ * Note that Recorder class is not serialized or deserialized.
+ * 
+ * The serialization and deserialization process typically begins with the Model class,
+ * which internally calls the serialization of the Connections and Layout classes.
+ * Connections, in turn, handle the serialization of Edges, while Layout handles 
+ * the serialization of Vertices. This ensures a comprehensive serialization of 
+ * the entire simulation structure.
+ * 
+ * @note As of September 2024, serialization support is currently available 
+ * for CPU-based Neuron simulations. While GPU-based Neuron serialization is functional, 
+ * the output result files differ, and this is being addressed in [Issue #701].
+ * Serialization support for NG911 will be extended in the future [Issue #700].
+ *  
  */
+
 #pragma once
+#include "Simulator.h"
 
 class Serializer {
 public:
    Serializer() = default;
 
-   ///  Serializes synapse weights, source vertices, destination vertices,
-   ///  maxEdgesPerVertex, totalVertices.
-   ///  if running a connGrowth model serializes radii as well
-   void serializeSynapses();
+   /// Serializes all member variables of the
+   /// Connections, Layout, Edges, Vertices, and associated helper classes.
+   void serialize();
 
-   ///  Deserializes synapse weights, source vertices, destination vertices,
-   ///  maxEdgesPerVertex, totalVertices.
-   ///  if running a connGrowth model and radii is in serialization file, deserializes radii as well
+   /// Deserializes all member variables of the
+   /// Connections, Layout, Edges, Vertices, and associated helper classes.
    ///
-   ///  @returns    true if successful, false otherwise.
-   bool deserializeSynapses();
+   /// @returns true if deserialization is successful; false otherwise.
+   bool deserialize();
+
+private:
+   template <typename Archive> static bool processArchive(Archive &archive, Simulator &simulator);
 };
diff --git a/Simulator/Core/Simulator.cpp b/Simulator/Core/Simulator.cpp
@@ -120,21 +120,6 @@ void Simulator::printParameters() const
                       << "\tInitializer RNG Seed: " << initRngSeed_ << endl);
 }
 
-// Code from STDPFix branch, doesn't do anything
-/// Copy GPU Synapse data to CPU.
-void Simulator::copyGPUSynapseToCPU()
-{
-   // ToDo: Delete this method and implement using OperationManager
-   // model->copyGPUSynapseToCPUModel();
-}
-
-/// Copy CPU Synapse data to GPU.
-void Simulator::copyCPUSynapseToGPU()
-{
-   // ToDo: Delete this method and implement using OperationManager
-   // model->copyCPUSynapseToGPUModel();
-}
-
 /// Resets all of the maps. Releases and re-allocates memory for each map,
 /// clearing them as necessary.
 void Simulator::reset()
@@ -153,16 +138,16 @@ void Simulator::reset()
 void Simulator::simulate()
 {
    // Main simulation loop - execute maxGrowthSteps
-   for (int currentEpoch = 1; currentEpoch <= numEpochs_; currentEpoch++) {
-      LOG4CPLUS_TRACE(consoleLogger_, "Performing epoch number: " << currentEpoch);
+   for (currentEpoch_ = 1; currentEpoch_ <= numEpochs_; currentEpoch_++) {
+      LOG4CPLUS_TRACE(consoleLogger_, "Performing epoch number: " << currentEpoch_);
       LOG4CPLUS_TRACE(fileLogger_, "Begin network state:");
-      currentEpoch_ = currentEpoch;
+
 #ifdef PERFORMANCE_METRICS
       // Start timer for advance
       short_timer.start();
 #endif
       // Advance simulation to next growth cycle
-      advanceEpoch(currentEpoch);
+      advanceEpoch(currentEpoch_);
 #ifdef PERFORMANCE_METRICS
       // Time to advance
       t_host_advance += short_timer.lap() / 1000000.0;
@@ -208,16 +193,26 @@ void Simulator::advanceEpoch(int currentEpoch) const
    while (g_simulationStep < endStep) {
       // Output status once every 1% of total simulation time
       if (count % onePercent == 0) {
-         LOG4CPLUS_TRACE(consoleLogger_,
-                         "Epoch: " << currentEpoch << "/" << numEpochs_
-                                   << " simulating time: " << g_simulationStep * deltaT_ << "/"
-                                   << (epochDuration_ * numEpochs_) - 1);
-         LOG4CPLUS_TRACE(workbenchLogger_,
-                         "Epoch: " << currentEpoch << "/" << numEpochs_
-                                   << " simulating time: " << g_simulationStep * deltaT_ << "/"
-                                   << (epochDuration_ * numEpochs_) - 1);
+         // Simulation time is calculated independently to handle deserialization scenarios.
+         // If g_simulationStep is set to a value greater than totalDuration after deserialization,
+         // a modulo operation is applied to ensure the displayed simulation time is accurate.
+         // If totalDuration is zero, the entire simulation time is determined by g_simulationStep * deltaT_.
+         uint64_t totalDuration = (epochDuration_ * numEpochs_) - 1;
+         auto simulatedTime
+            = (totalDuration == 0 ? (g_simulationStep * deltaT_)
+                                  : (uint64_t)(g_simulationStep * deltaT_) % totalDuration);
+
+         LOG4CPLUS_TRACE(consoleLogger_, "Epoch: " << currentEpoch << "/" << numEpochs_
+                                                   << " simulating time: " << simulatedTime << "/"
+                                                   << totalDuration);
+
+         LOG4CPLUS_TRACE(workbenchLogger_, "Epoch: " << currentEpoch << "/" << numEpochs_
+                                                     << " simulating time: " << simulatedTime << "/"
+                                                     << totalDuration);
+
          count = 0;
       }
+
       count++;
       // input stimulus
       /***** S_INPUT NOT IN REPO YET *******/
diff --git a/Simulator/Core/Simulator.h b/Simulator/Core/Simulator.h
@@ -37,12 +37,6 @@ class Simulator {
 
    void printParameters() const;   /// Prints loaded parameters to logging file.
 
-   // Copied over from STDPFix
-   void copyGPUSynapseToCPU();   /// Copy GPU Synapse data to CPU.
-
-   // Copied over from STDPFix
-   void copyCPUSynapseToGPU();   /// Copy CPU Synapse data to GPU.
-
    void reset();   /// Reset simulation objects.
 
    void simulate();
@@ -126,8 +120,6 @@ class Simulator {
    Simulator();   /// Constructor is private to keep a singleton instance of this
                   /// class.
 
-   int totalNeurons_;   /// Count of neurons in the simulation
-
    int currentEpoch_;   /// Current epoch step
 
    int numEpochs_;   /// Number of simulator epochs
diff --git a/Simulator/Edges/AllEdges.h b/Simulator/Edges/AllEdges.h
@@ -15,7 +15,6 @@
 #include <vector>
 // cereal
 #include "cereal/types/vector.hpp"
-#include <cereal/types/polymorphic.hpp>
 
 class AllVertices;
 
diff --git a/Simulator/Layouts/Layout.h b/Simulator/Layouts/Layout.h
diff --git a/Simulator/Layouts/Neuro/LayoutNeuro.h b/Simulator/Layouts/Neuro/LayoutNeuro.h
