Skip to content

Commit dbc404b

Browse files
Hsu0227Hsu0227
committed
Implement compileHistories() for UpdatedType:: DYNAMI for Hdf5 Recorder
1 parent 0c4c8f4 commit dbc404b

File tree

3 files changed

+144
-27
lines changed

3 files changed

+144
-27
lines changed

‎Simulator/Recorders/Hdf5Recorder.cpp‎

Lines changed: 90 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,8 @@ void Hdf5Recorder::saveSimData(const AllVertices &neurons)
108108
{
109109
// Initialize datasets for constant variables
110110
for (auto &variableInfo : variableTable_) {
111-
if (variableInfo.variableType_ == UpdatedType::CONSTANT) {
111+
if (variableInfo.variableLocation_.getNumElements() > 0
112+
&& variableInfo.variableType_ == UpdatedType::CONSTANT) {
112113
// Define dimensions for the constant dataset
113114
hsize_t constantDims[1]
114115
= {static_cast<hsize_t>(variableInfo.variableLocation_.getNumElements())};
@@ -122,6 +123,94 @@ void Hdf5Recorder::saveSimData(const AllVertices &neurons)
122123
}
123124
}
124125

126+
// Processes and updates HDF5 datasets for variables marked as DYNAMIC
127+
void Hdf5Recorder::compileHistories(AllVertices &vertices)
128+
{
129+
// Define the maximum chunk size for datasets to optimize storage and access
130+
const hsize_t max_chunk_size = 1024;
131+
132+
// Iterate over each variableInfo object in the variable table
133+
for (auto &variableInfo : variableTable_) {
134+
// Process only variables marked as DYNAMIC, which change over time
135+
if (variableInfo.variableType_ == UpdatedType::DYNAMIC) {
136+
// Check if there are elements to process
137+
if (variableInfo.variableLocation_.getNumElements() > 0) {
138+
// Determine the new dimensions for the dataset based on the number of elements
139+
hsize_t newDims[1]
140+
= {static_cast<hsize_t>(variableInfo.variableLocation_.getNumElements())};
141+
142+
// Create the dataset if it does not already exist
143+
if (!H5Iis_valid(variableInfo.hdf5DataSet_.getId())) {
144+
// Create initial dataspace with unlimited maximum dimensions
145+
hsize_t maxDims[1] = {H5S_UNLIMITED};
146+
DataSpace initialSpace(1, newDims, maxDims);
147+
148+
// Create dataset creation properties and set chunk size for efficiency
149+
DSetCreatPropList cparms;
150+
hsize_t chunk_dims[1] = {std::min(newDims[0], max_chunk_size)};
151+
cparms.setChunk(1, chunk_dims);
152+
153+
// Create the dataset with specified name, datatype, initial space, and properties
154+
variableInfo.hdf5DataSet_ = resultOut_->createDataSet(
155+
variableInfo.variableName_, variableInfo.hdf5Datatype_, initialSpace, cparms);
156+
157+
// Write the initial data to the dataset
158+
variableInfo.captureData();
159+
} else {
160+
// Handle the case where the dataset already exists
161+
162+
// Get the current dimensions of the dataset
163+
DataSpace currentSpace = variableInfo.hdf5DataSet_.getSpace();
164+
hsize_t currentDims[1];
165+
currentSpace.getSimpleExtentDims(currentDims, nullptr);
166+
167+
// Calculate the new dimensions after appending the new data
168+
hsize_t appendDims[1] = {currentDims[0] + newDims[0]};
169+
variableInfo.hdf5DataSet_.extend(appendDims);
170+
171+
// Select the hyperslab in the extended portion of the dataset
172+
DataSpace fileSpace = variableInfo.hdf5DataSet_.getSpace();
173+
hsize_t offset[1] = {currentDims[0]};
174+
fileSpace.selectHyperslab(H5S_SELECT_SET, newDims, offset);
175+
176+
// Create a memory dataspace for the new data
177+
DataSpace memSpace(1, newDims);
178+
179+
// Prepare the data buffer and write the new data to the dataset
180+
if (variableInfo.hdf5Datatype_ == PredType::NATIVE_FLOAT) {
181+
std::vector<float> dataBuffer(variableInfo.variableLocation_.getNumElements());
182+
for (size_t i = 0; i < variableInfo.variableLocation_.getNumElements(); ++i) {
183+
dataBuffer[i] = get<float>(variableInfo.variableLocation_.getElement(i));
184+
}
185+
variableInfo.hdf5DataSet_.write(dataBuffer.data(), variableInfo.hdf5Datatype_,
186+
memSpace, fileSpace);
187+
} else if (variableInfo.hdf5Datatype_ == PredType::NATIVE_INT) {
188+
std::vector<int> dataBuffer(variableInfo.variableLocation_.getNumElements());
189+
for (size_t i = 0; i < variableInfo.variableLocation_.getNumElements(); ++i) {
190+
dataBuffer[i] = get<int>(variableInfo.variableLocation_.getElement(i));
191+
}
192+
variableInfo.hdf5DataSet_.write(dataBuffer.data(), variableInfo.hdf5Datatype_,
193+
memSpace, fileSpace);
194+
} else if (variableInfo.hdf5Datatype_ == PredType::NATIVE_UINT64) {
195+
std::vector<uint64_t> dataBuffer(variableInfo.variableLocation_.getNumElements());
196+
for (size_t i = 0; i < variableInfo.variableLocation_.getNumElements(); ++i) {
197+
dataBuffer[i] = get<uint64_t>(variableInfo.variableLocation_.getElement(i));
198+
}
199+
variableInfo.hdf5DataSet_.write(dataBuffer.data(), variableInfo.hdf5Datatype_,
200+
memSpace, fileSpace);
201+
} else {
202+
// Throw an exception if the data type is unsupported
203+
throw std::runtime_error("Unsupported data type");
204+
}
205+
}
206+
}
207+
}
208+
209+
// Call startNewEpoch() to prepare for new data input
210+
variableInfo.variableLocation_.startNewEpoch();
211+
}
212+
}
213+
125214
/// Receives a recorded variable entity from the variable owner class
126215
/// used when the return type from recordable variable is supported by Recorder
127216
/**

‎Simulator/Recorders/Hdf5Recorder.h‎

Lines changed: 1 addition & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,7 @@ class Hdf5Recorder : public Recorder {
6060

6161
// TODO: No parameters needed (AllVertices &vertices)
6262
/// Compile/capture variable history information in every epoch
63-
virtual void compileHistories(AllVertices &neurons) override
64-
{
65-
}
63+
virtual void compileHistories(AllVertices &neurons) override;
6664

6765
// TODO: No parameters needed (AllVertices &vertices)
6866
/// Writes simulation results to an output destination.
@@ -202,29 +200,6 @@ class Hdf5Recorder : public Recorder {
202200

203201
// Member variables for HDF5 datasets
204202
H5File *resultOut_;
205-
/*DataSet dataSetXloc_;
206-
DataSet dataSetYloc_;
207-
DataSet dataSetNeuronTypes_;
208-
DataSet dataSetNeuronThresh_;
209-
DataSet dataSetStarterNeurons_;
210-
DataSet dataSetTsim_;
211-
DataSet dataSetSimulationEndTime_;
212-
DataSet dataSetProbedNeurons_;
213-
DataSet dataSetSpikesHist_;
214-
DataSet dataSetSpikesProbedNeurons_;
215-
216-
// HDF5 dataset names
217-
const H5std_string nameSpikesHist = "spikesHistory";
218-
const H5std_string nameXloc = "xloc";
219-
const H5std_string nameYloc = "yloc";
220-
const H5std_string nameNeuronTypes = "neuronTypes";
221-
const H5std_string nameNeuronThresh = "neuronThresh";
222-
const H5std_string nameStarterNeurons = "starterNeurons";
223-
const H5std_string nameTsim = "Tsim";
224-
const H5std_string nameSimulationEndTime = "simulationEndTime";
225-
const H5std_string nameSpikesProbedNeurons = "spikesProbedNeurons";
226-
const H5std_string nameAttrPNUnit = "attrPNUint";
227-
const H5std_string nameProbedNeurons = "probedNeurons";*/
228203

229204
// Keep track of where we are in incrementally writing spikes
230205
vector<hsize_t> offsetSpikesProbedNeurons_;

‎Testing/UnitTesting/Hdf5RecorderTests.cpp‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,4 +112,57 @@ TEST(Hdf5RecorderTest, SaveSimDataTest)
112112
}
113113
}
114114

115+
// Define the test case for compiling histories
116+
TEST(Hdf5RecorderTest, CompileHistoriesTest)
117+
{
118+
// Define a temporary file path for testing
119+
std::string outputFile
120+
= "../Testing/UnitTesting/TestOutput/Hdf5test_output_compile_histories.h5";
121+
122+
// Create an instance of Hdf5Recorder
123+
Hdf5Recorder recorder(outputFile);
124+
recorder.init();
125+
126+
// Create and configure variables for testing
127+
EventBuffer eventBufferInt(5); // Example with int type
128+
129+
// Register the variable with Hdf5Recorder as DYNAMIC
130+
recorder.registerVariable("test_var_int", eventBufferInt, Recorder::UpdatedType::DYNAMIC);
131+
132+
// Create a unique_ptr to an empty AllVertices object
133+
unique_ptr<AllVertices> vertices = 0;
134+
135+
// Call compileHistories() multiple times to simulate multiple epochs
136+
for (int epoch = 0; epoch < 3; ++epoch) {
137+
// Clear and insert new events to simulate new data each epoch
138+
eventBufferInt.clear();
139+
eventBufferInt.insertEvent(1 * (epoch + 1));
140+
eventBufferInt.insertEvent(2 * (epoch + 1));
141+
eventBufferInt.insertEvent(3 * (epoch + 1));
142+
eventBufferInt.insertEvent(4 * (epoch + 1));
143+
eventBufferInt.insertEvent(5 * (epoch + 1));
144+
145+
recorder.compileHistories(*vertices);
146+
}
147+
148+
// Open the HDF5 file and read back the data
149+
H5File file(outputFile, H5F_ACC_RDONLY);
150+
DataSet dataset = file.openDataSet("test_var_int");
151+
DataSpace dataspace = dataset.getSpace();
152+
153+
hsize_t num_elements;
154+
dataspace.getSimpleExtentDims(&num_elements, nullptr);
155+
156+
std::vector<int> dataBuffer(num_elements);
157+
// Read the data into the buffer
158+
dataset.read(dataBuffer.data(), PredType::NATIVE_INT);
159+
160+
// Verify the data matches the expected values (repeated 3 times)
161+
std::vector<int> expectedData = {1, 2, 3, 4, 5, 2, 4, 6, 8, 10, 3, 6, 9, 12, 15};
162+
ASSERT_EQ(expectedData.size(), dataBuffer.size());
163+
for (size_t i = 0; i < expectedData.size(); ++i) {
164+
EXPECT_EQ(expectedData[i], dataBuffer[i]);
165+
}
166+
}
167+
115168
#endif // HDF5

0 commit comments

Comments
 (0)