Merge pull request #683 from UWB-Biocomputing/issue-658-add-compileHistoriesforUpdatedTypeDYNAMICForHdf5Recorder

Hsu0227 · web-flow · commit ed263626f877 · 2024-08-04T18:36:59.000-07:00
Implement compileHistories() for UpdatedType:: DYNAMI for Hdf5 Recorder
diff --git a/Simulator/Core/Model.cpp b/Simulator/Core/Model.cpp
@@ -67,7 +67,7 @@ Model::Model()
 void Model::saveResults()
 {
    if (recorder_ != nullptr) {
-      recorder_->saveSimData(layout_->getVertices());
+      recorder_->saveSimData();
    }
 }
 
@@ -146,7 +146,7 @@ void Model::updateHistory()
       LOG4CPLUS_INFO(fileLogger_, "ERROR: Recorder class is null.");
    }
    if (recorder_ != nullptr) {
-      recorder_->compileHistories(layout_->getVertices());
+      recorder_->compileHistories();
    }
 }
 
diff --git a/Simulator/Recorders/Hdf5Recorder.cpp b/Simulator/Recorders/Hdf5Recorder.cpp
@@ -104,11 +104,12 @@ void Hdf5Recorder::term()
 }
 
 // create the dataset for constant variable and store the data to dataset
-void Hdf5Recorder::saveSimData(const AllVertices &neurons)
+void Hdf5Recorder::saveSimData()
 {
    // Initialize datasets for constant variables
    for (auto &variableInfo : variableTable_) {
-      if (variableInfo.variableType_ == UpdatedType::CONSTANT) {
+      if (variableInfo.variableLocation_.getNumElements() > 0
+          && variableInfo.variableType_ == UpdatedType::CONSTANT) {
          // Define dimensions for the constant dataset
          hsize_t constantDims[1]
             = {static_cast<hsize_t>(variableInfo.variableLocation_.getNumElements())};
@@ -122,6 +123,97 @@ void Hdf5Recorder::saveSimData(const AllVertices &neurons)
    }
 }
 
+// Processes and updates HDF5 datasets for variables marked as DYNAMIC
+void Hdf5Recorder::compileHistories()
+{
+   // Define the maximum chunk size for datasets to optimize storage and access
+   // This defines the maximum number of elements per chunk
+   // If the dataset size exceeds this value, HDF5 will create multiple chunks to store the data
+   const hsize_t max_chunk_size = 1024;
+
+   // Iterate over each variableInfo object in the variable table
+   for (auto &variableInfo : variableTable_) {
+      // Process only variables marked as DYNAMIC, which change over time
+      if (variableInfo.variableType_ == UpdatedType::DYNAMIC) {
+         // Check if there are elements to process
+         if (variableInfo.variableLocation_.getNumElements() > 0) {
+            // Determine the new dimensions for the dataset based on the number of elements
+            hsize_t newDims[1]
+               = {static_cast<hsize_t>(variableInfo.variableLocation_.getNumElements())};
+
+            // Create the dataset if it does not already exist
+            if (!H5Iis_valid(variableInfo.hdf5DataSet_.getId())) {
+               // Create initial dataspace with unlimited maximum dimensions
+               hsize_t maxDims[1] = {H5S_UNLIMITED};
+               DataSpace initialSpace(1, newDims, maxDims);
+
+               // Create dataset creation properties and set chunk size for efficiency
+               DSetCreatPropList cparms;
+               hsize_t chunk_dims[1] = {std::min(newDims[0], max_chunk_size)};
+               cparms.setChunk(1, chunk_dims);
+
+               // Create the dataset with specified name, datatype, initial space, and properties
+               variableInfo.hdf5DataSet_ = resultOut_->createDataSet(
+                  variableInfo.variableName_, variableInfo.hdf5Datatype_, initialSpace, cparms);
+
+               // Write the initial data to the dataset
+               variableInfo.captureData();
+            } else {
+               // Handle the case where the dataset already exists
+
+               // Get the current dimensions of the dataset
+               DataSpace currentSpace = variableInfo.hdf5DataSet_.getSpace();
+               hsize_t currentDims[1];
+               currentSpace.getSimpleExtentDims(currentDims, nullptr);
+
+               // Calculate the new dimensions after appending the new data
+               hsize_t appendDims[1] = {currentDims[0] + newDims[0]};
+               variableInfo.hdf5DataSet_.extend(appendDims);
+
+               // Select the hyperslab in the extended portion of the dataset
+               DataSpace fileSpace = variableInfo.hdf5DataSet_.getSpace();
+               hsize_t offset[1] = {currentDims[0]};
+               fileSpace.selectHyperslab(H5S_SELECT_SET, newDims, offset);
+
+               // Create a memory dataspace for the new data
+               DataSpace memSpace(1, newDims);
+
+               // Prepare the data buffer and write the new data to the dataset
+               if (variableInfo.hdf5Datatype_ == PredType::NATIVE_FLOAT) {
+                  vector<float> dataBuffer(variableInfo.variableLocation_.getNumElements());
+                  for (size_t i = 0; i < variableInfo.variableLocation_.getNumElements(); ++i) {
+                     dataBuffer[i] = get<float>(variableInfo.variableLocation_.getElement(i));
+                  }
+                  variableInfo.hdf5DataSet_.write(dataBuffer.data(), variableInfo.hdf5Datatype_,
+                                                  memSpace, fileSpace);
+               } else if (variableInfo.hdf5Datatype_ == PredType::NATIVE_INT) {
+                  vector<int> dataBuffer(variableInfo.variableLocation_.getNumElements());
+                  for (size_t i = 0; i < variableInfo.variableLocation_.getNumElements(); ++i) {
+                     dataBuffer[i] = get<int>(variableInfo.variableLocation_.getElement(i));
+                  }
+                  variableInfo.hdf5DataSet_.write(dataBuffer.data(), variableInfo.hdf5Datatype_,
+                                                  memSpace, fileSpace);
+               } else if (variableInfo.hdf5Datatype_ == PredType::NATIVE_UINT64) {
+                  vector<uint64_t> dataBuffer(variableInfo.variableLocation_.getNumElements());
+                  for (size_t i = 0; i < variableInfo.variableLocation_.getNumElements(); ++i) {
+                     dataBuffer[i] = get<uint64_t>(variableInfo.variableLocation_.getElement(i));
+                  }
+                  variableInfo.hdf5DataSet_.write(dataBuffer.data(), variableInfo.hdf5Datatype_,
+                                                  memSpace, fileSpace);
+               } else {
+                  // Throw an exception if the data type is unsupported
+                  throw runtime_error("Unsupported data type for variable: "
+                                      + variableInfo.variableName_);
+               }
+            }
+         }
+      }
+
+      // Call startNewEpoch() to prepare for new data input
+      variableInfo.variableLocation_.startNewEpoch();
+   }
+}
+
 /// Receives a recorded variable entity from the variable owner class
 /// used when the return type from recordable variable is supported by Recorder
 /**
diff --git a/Simulator/Recorders/Hdf5Recorder.h b/Simulator/Recorders/Hdf5Recorder.h
@@ -58,15 +58,13 @@ class Hdf5Recorder : public Recorder {
    /// Terminate process
    virtual void term() override;
 
-   // TODO: No parameters needed (AllVertices &vertices)
+
    /// Compile/capture variable history information in every epoch
-   virtual void compileHistories(AllVertices &neurons) override
-   {
-   }
+   virtual void compileHistories() override;
+
 
-   // TODO: No parameters needed (AllVertices &vertices)
    /// Writes simulation results to an output destination.
-   virtual void saveSimData(const AllVertices &neurons) override;
+   virtual void saveSimData() override;
 
    /// Prints out all parameters to logging file.
    /// Registered to OperationManager as Operation::printParameters
@@ -202,29 +200,6 @@ class Hdf5Recorder : public Recorder {
 
    // Member variables for HDF5 datasets
    H5File *resultOut_;
-   /*DataSet dataSetXloc_;
-   DataSet dataSetYloc_;
-   DataSet dataSetNeuronTypes_;
-   DataSet dataSetNeuronThresh_;
-   DataSet dataSetStarterNeurons_;
-   DataSet dataSetTsim_;
-   DataSet dataSetSimulationEndTime_;
-   DataSet dataSetProbedNeurons_;
-   DataSet dataSetSpikesHist_;
-   DataSet dataSetSpikesProbedNeurons_;
-
-   // HDF5 dataset names
-   const H5std_string nameSpikesHist = "spikesHistory";
-   const H5std_string nameXloc = "xloc";
-   const H5std_string nameYloc = "yloc";
-   const H5std_string nameNeuronTypes = "neuronTypes";
-   const H5std_string nameNeuronThresh = "neuronThresh";
-   const H5std_string nameStarterNeurons = "starterNeurons";
-   const H5std_string nameTsim = "Tsim";
-   const H5std_string nameSimulationEndTime = "simulationEndTime";
-   const H5std_string nameSpikesProbedNeurons = "spikesProbedNeurons";
-   const H5std_string nameAttrPNUnit = "attrPNUint";
-   const H5std_string nameProbedNeurons = "probedNeurons";*/
 
    // Keep track of where we are in incrementally writing spikes
    vector<hsize_t> offsetSpikesProbedNeurons_;
diff --git a/Simulator/Recorders/NG911/Xml911Recorder.cpp b/Simulator/Recorders/NG911/Xml911Recorder.cpp
@@ -33,14 +33,14 @@ void Xml911Recorder::getValues()
 /// Compile history information in every epoch
 ///
 /// @param[in] vertices   The entire list of vertices.
-void Xml911Recorder::compileHistories(AllVertices &vertices)
+void Xml911Recorder::compileHistories()
 {
 }
 
 /// Writes simulation results to an output destination.
 ///
 /// @param  vertices the Vertex list to search from.
-void Xml911Recorder::saveSimData(const AllVertices &vertices)
+void Xml911Recorder::saveSimData()
 {
    auto &conns = Simulator::getInstance().getModel().getConnections();
    Connections911 &conns911 = dynamic_cast<Connections911 &>(conns);
diff --git a/Simulator/Recorders/NG911/Xml911Recorder.h b/Simulator/Recorders/NG911/Xml911Recorder.h
@@ -40,12 +40,12 @@ class Xml911Recorder : public XmlRecorder {
    /// Compile history information in every epoch
    ///
    /// @param[in] vertices   The entire list of vertices.
-   virtual void compileHistories(AllVertices &vertices) override;
+   virtual void compileHistories() override;
 
    /// Writes simulation results to an output destination.
    ///
    /// @param  vertices the Vertex list to search from.
-   virtual void saveSimData(const AllVertices &vertices) override;
+   virtual void saveSimData() override;
 
    ///  Prints out all parameters to logging file.
    ///  Registered to OperationManager as Operation::printParameters
diff --git a/Simulator/Recorders/Recorder.h b/Simulator/Recorders/Recorder.h
@@ -52,13 +52,13 @@ class Recorder {
    /// Terminate process
    virtual void term() = 0;
 
-   // TODO: No parameters needed (AllVertices &vertices)
+
    /// Compile/capture variable history information in every epoch
-   virtual void compileHistories(AllVertices &vertices) = 0;
+   virtual void compileHistories() = 0;
+
 
-   // TODO: No parameters needed (AllVertices &vertices)
    /// Writes simulation results to an output destination.
-   virtual void saveSimData(const AllVertices &vertices) = 0;
+   virtual void saveSimData() = 0;
 
    /// Prints loaded parameters to logging file.
    virtual void printParameters() = 0;
diff --git a/Simulator/Recorders/XmlRecorder.cpp b/Simulator/Recorders/XmlRecorder.cpp
@@ -73,7 +73,7 @@ void XmlRecorder::term()
 
 // TODO : @param[in] vertices will be removed eventually after HDF5Recorder implementing
 /// Compile history information in every epoch
-void XmlRecorder::compileHistories(AllVertices &vertices)
+void XmlRecorder::compileHistories()
 {
    //capture data information in each epoch
    for (int rowIndex = 0; rowIndex < variableTable_.size(); rowIndex++) {
@@ -86,7 +86,7 @@ void XmlRecorder::compileHistories(AllVertices &vertices)
 
 // TODO : @param[in] vertices will be removed eventually after HDF5Recorder implementing
 /// Writes simulation results to an output destination.
-void XmlRecorder::saveSimData(const AllVertices &vertices)
+void XmlRecorder::saveSimData()
 {
    // Write XML header information:
    string header = "<?xml version=\"1.0\" standalone=\"no\"?>\n";
diff --git a/Simulator/Recorders/XmlRecorder.h b/Simulator/Recorders/XmlRecorder.h
@@ -60,13 +60,13 @@ class XmlRecorder : public Recorder {
    /// Terminate process
    virtual void term() override;
 
-   // TODO: No parameters needed (AllVertices &vertices)
+
    /// Compile/capture variable history information in every epoch
-   virtual void compileHistories(AllVertices &vertices) override;
+   virtual void compileHistories() override;
+
 
-   // TODO: No parameters needed (AllVertices &vertices)
    /// Writes simulation results to an output destination.
-   virtual void saveSimData(const AllVertices &vertices) override;
+   virtual void saveSimData() override;
 
    ///  Prints out all parameters to logging file.
    ///  Registered to OperationManager as Operation::printParameters
diff --git a/Testing/UnitTesting/Hdf5RecorderTests.cpp b/Testing/UnitTesting/Hdf5RecorderTests.cpp
@@ -86,11 +86,8 @@ TEST(Hdf5RecorderTest, SaveSimDataTest)
    // Register the variable with Hdf5Recorder
    recorder.registerVariable("test_var1", eventBuffer, Recorder::UpdatedType::CONSTANT);
 
-   // Create a unique_ptr to an empty AllVertices object
-   unique_ptr<AllVertices> vertices = 0;
-
    // Call saveSimData() to write the data to the file
-   recorder.saveSimData(*vertices);
+   recorder.saveSimData();
 
    // Open the HDF5 file and read back the data
    H5File file(outputFile, H5F_ACC_RDONLY);
@@ -112,4 +109,55 @@ TEST(Hdf5RecorderTest, SaveSimDataTest)
    }
 }
 
+// Define the test case for compiling histories
+TEST(Hdf5RecorderTest, CompileHistoriesTest)
+{
+   // Define a temporary file path for testing
+   std::string outputFile
+      = "../Testing/UnitTesting/TestOutput/Hdf5test_output_compile_histories.h5";
+
+   // Create an instance of Hdf5Recorder
+   Hdf5Recorder recorder(outputFile);
+   recorder.init();
+
+   // Create and configure variables for testing
+   EventBuffer eventBufferInt(5);   // Example with int type
+
+   // Register the variable with Hdf5Recorder as DYNAMIC
+   recorder.registerVariable("test_var_int", eventBufferInt, Recorder::UpdatedType::DYNAMIC);
+
+
+   // Call compileHistories() multiple times to simulate multiple epochs
+   for (int epoch = 0; epoch < 3; ++epoch) {
+      // Clear and insert new events to simulate new data each epoch
+      eventBufferInt.clear();
+      eventBufferInt.insertEvent(1 * (epoch + 1));
+      eventBufferInt.insertEvent(2 * (epoch + 1));
+      eventBufferInt.insertEvent(3 * (epoch + 1));
+      eventBufferInt.insertEvent(4 * (epoch + 1));
+      eventBufferInt.insertEvent(5 * (epoch + 1));
+
+      recorder.compileHistories();
+   }
+
+   // Open the HDF5 file and read back the data
+   H5File file(outputFile, H5F_ACC_RDONLY);
+   DataSet dataset = file.openDataSet("test_var_int");
+   DataSpace dataspace = dataset.getSpace();
+
+   hsize_t num_elements;
+   dataspace.getSimpleExtentDims(&num_elements, nullptr);
+
+   std::vector<int> dataBuffer(num_elements);
+   // Read the data into the buffer
+   dataset.read(dataBuffer.data(), PredType::NATIVE_INT);
+
+   // Verify the data matches the expected values (repeated 3 times)
+   std::vector<int> expectedData = {1, 2, 3, 4, 5, 2, 4, 6, 8, 10, 3, 6, 9, 12, 15};
+   ASSERT_EQ(expectedData.size(), dataBuffer.size());
+   for (size_t i = 0; i < expectedData.size(); ++i) {
+      EXPECT_EQ(expectedData[i], dataBuffer[i]);
+   }
+}
+
 #endif   // HDF5
diff --git a/Testing/UnitTesting/XmlRecorderTests.cpp b/Testing/UnitTesting/XmlRecorderTests.cpp
@@ -146,7 +146,7 @@ TEST(XmlRecorderTest, CompileHistoriesTest)
    buffer0.insertEvent(2);
 
    // Call the compileHistories method
-   recorderTest_->compileHistories(*vertices.get());
+   recorderTest_->compileHistories();
    vector<std::variant<uint64_t, bool, int, BGFLOAT>> history = recorderTest_->getHistory(0);
 
    // Verify the events compiled hisotry
@@ -214,9 +214,9 @@ TEST(XmlRecorderTest, SaveSimDataTest)
    buffer.insertEvent(3);
 
    // Call the compileHistories method
-   recorderTest_->compileHistories(*vertices.get());
+   recorderTest_->compileHistories();
    // Call the saveSimData() function
-   recorderTest_->saveSimData(*vertices.get());
+   recorderTest_->saveSimData();
 
    // Open the test_output.xml file and read its content
    std::ifstream inputFile("../Testing/UnitTesting/TestOutput/test_output.xml");

Original file line number	Diff line number	Diff line change
`@@ -67,7 +67,7 @@ Model::Model()`
`67`	`67`	`void Model::saveResults()`
`68`	`68`	`{`
`69`	`69`	`if (recorder_ != nullptr) {`
`70`		`- recorder_->saveSimData(layout_->getVertices());`
	`70`	`+ recorder_->saveSimData();`
`71`	`71`	`}`
`72`	`72`	`}`
`73`	`73`
`@@ -146,7 +146,7 @@ void Model::updateHistory()`
`146`	`146`	`LOG4CPLUS_INFO(fileLogger_, "ERROR: Recorder class is null.");`
`147`	`147`	`}`
`148`	`148`	`if (recorder_ != nullptr) {`
`149`		`- recorder_->compileHistories(layout_->getVertices());`
	`149`	`+ recorder_->compileHistories();`
`150`	`150`	`}`
`151`	`151`	`}`
`152`	`152`
Original file line number	Diff line number	Diff line change
`@@ -33,14 +33,14 @@ void Xml911Recorder::getValues()`
`33`	`33`	`/// Compile history information in every epoch`
`34`	`34`	`///`
`35`	`35`	`/// @param[in] vertices The entire list of vertices.`
`36`		`-void Xml911Recorder::compileHistories(AllVertices &vertices)`
	`36`	`+void Xml911Recorder::compileHistories()`
`37`	`37`	`{`
`38`	`38`	`}`
`39`	`39`
`40`	`40`	`/// Writes simulation results to an output destination.`
`41`	`41`	`///`
`42`	`42`	`/// @param vertices the Vertex list to search from.`
`43`		`-void Xml911Recorder::saveSimData(const AllVertices &vertices)`
	`43`	`+void Xml911Recorder::saveSimData()`
`44`	`44`	`{`
`45`	`45`	`auto &conns = Simulator::getInstance().getModel().getConnections();`
`46`	`46`	`Connections911 &conns911 = dynamic_cast<Connections911 &>(conns);`