Skip to content

Copy weights from GPU #796

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: HaripriyaDevelopment
Choose a base branch
from
Draft

Copy weights from GPU #796

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3edba91
Broke up CudaSource into multiple librariers and made the libraries s…
AndrewBMadison Jan 17, 2025
7d7836f
Revert cuda compilation parameters to support kepler architecture
AndrewBMadison Jan 20, 2025
2915062
Merge pull request #767 from UWB-Biocomputing/issue-753-CMakeLists-fixes
AndrewBMadison Jan 25, 2025
1fda261
Copy back weights from GPU each epoch
Feb 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 21 additions & 14 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ cmake_minimum_required(VERSION 3.12)
#
#"YES" / GPU choice only available if CUDA library is installed and the GPU is CUDA capable.
############################################################################################

if(NOT ENABLE_CUDA)
set(ENABLE_CUDA NO)
endif()
Expand Down Expand Up @@ -161,18 +162,24 @@ include_directories(

if(ENABLE_CUDA)

set(cuda_sources Simulator/Edges/Neuro/AllSynapsesDeviceFuncs_d.cpp
Simulator/Vertices/Neuro/AllVerticesDeviceFuncs_d.cpp
Simulator/Edges/Neuro/AllSpikingSynapses_d.cpp
Simulator/Edges/Neuro/AllSTDPSynapses_d.cpp
Simulator/Edges/Neuro/AllDSSynapses_d.cpp
Simulator/Edges/Neuro/AllDynamicSTDPSynapses_d.cpp
Simulator/Vertices/Neuro/AllLIFNeurons_d.cpp
Simulator/Vertices/Neuro/AllIZHNeurons_d.cpp
Simulator/Vertices/Neuro/AllIFNeurons_d.cpp
Simulator/Vertices/Neuro/AllSpikingNeurons_d.cpp
set(cuda_ConnectionsSources
Simulator/Connections/Neuro/ConnGrowth_d.cpp )
set_source_files_properties(${cuda_sources} PROPERTIES LANGUAGE CUDA)
set_source_files_properties(${cuda_ConnectionsSources} PROPERTIES LANGUAGE CUDA)

set(cuda_EdgesSources Simulator/Edges/Neuro/AllSynapsesDeviceFuncs_d.cpp
Simulator/Edges/Neuro/AllSpikingSynapses_d.cpp
Simulator/Edges/Neuro/AllSTDPSynapses_d.cpp
Simulator/Edges/Neuro/AllDSSynapses_d.cpp
Simulator/Edges/Neuro/AllDynamicSTDPSynapses_d.cpp)
set_source_files_properties(${cuda_EdgesSources} PROPERTIES LANGUAGE CUDA)

set(cuda_VerticesSources
Simulator/Vertices/Neuro/AllVerticesDeviceFuncs_d.cpp
Simulator/Vertices/Neuro/AllLIFNeurons_d.cpp
Simulator/Vertices/Neuro/AllIZHNeurons_d.cpp
Simulator/Vertices/Neuro/AllIFNeurons_d.cpp
Simulator/Vertices/Neuro/AllSpikingNeurons_d.cpp)
set_source_files_properties(${cuda_VerticesSources} PROPERTIES LANGUAGE CUDA)
endif()

#Collect source files and create libraries
Expand All @@ -184,7 +191,7 @@ endif()
file(GLOB_RECURSE Connections_Source Simulator/Connections/*.cpp Simulator/Connections/*.h)

if(ENABLE_CUDA)
add_library(Connections STATIC ${Connections_Source} ${cuda_sources})
add_library(Connections STATIC ${Connections_Source} ${cuda_ConnectionsSources})
else()
list(REMOVE_ITEM Connections_Source "${CMAKE_CURRENT_SOURCE_DIR}/Simulator/Connections/Neuro/ConnGrowth_d.cpp")
add_library(Connections STATIC ${Connections_Source})
Expand All @@ -195,7 +202,7 @@ endif()
file(GLOB_RECURSE Vertices_Source Simulator/Vertices/*.cpp Simulator/Vertices/*.h)

if(ENABLE_CUDA)
add_library(Vertices SHARED ${Vertices_Source} ${cuda_sources})
add_library(Vertices STATIC ${Vertices_Source} ${cuda_VerticesSources})
else()
list(REMOVE_ITEM Vertices_Source "${CMAKE_CURRENT_SOURCE_DIR}/Simulator/Vertices/Neuro/AllVerticesDeviceFuncs.h")
list(REMOVE_ITEM Vertices_Source "${CMAKE_CURRENT_SOURCE_DIR}/Simulator/Vertices/Neuro/AllVerticesDeviceFuncs_d.cpp")
Expand All @@ -212,7 +219,7 @@ endif()
# Create Edges library
file(GLOB_RECURSE Edges_Source Simulator/Edges/*.cpp Simulator/Edges/*.h)
if(ENABLE_CUDA)
add_library(Edges SHARED ${Edges_Source} ${cuda_sources})
add_library(Edges STATIC ${Edges_Source} ${cuda_EdgesSources})
else()
list(REMOVE_ITEM Edges_Source "${CMAKE_CURRENT_SOURCE_DIR}/Simulator/Edges/Neuro/AllSynapsesDeviceFuncs.h")
list(REMOVE_ITEM Edges_Source "${CMAKE_CURRENT_SOURCE_DIR}/Simulator/Edges/Neuro/AllSynapsesDeviceFuncs_d.cpp")
Expand Down
3 changes: 3 additions & 0 deletions Simulator/Connections/Neuro/ConnGrowth_d.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,4 +78,7 @@ void ConnGrowth::updateSynapsesWeights(int numVertices, AllVertices &vertices, A
synapses.copyDeviceEdgeCountsToHost(allEdgesDevice);
// copy device synapse summation coordinate to host memory
synapses.copyDeviceEdgeSumIdxToHost(allEdgesDevice);
synapses.copyDeviceEdgeWeightsToHost(allEdgesDevice);

synapses.Setup_xml_to_file();
}
4 changes: 3 additions & 1 deletion Simulator/Edges/AllEdges.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,9 @@ class AllEdges {
///
/// @param allEdgesDevice GPU address of the allEdges struct on device memory.
virtual void copyDeviceEdgeSumIdxToHost(void *allEdgesDevice) = 0;

virtual void copyDeviceEdgeWeightsToHost(void *allEdgesDevice) = 0;
virtual void Setup_xml_to_file() = 0;
//virtual std::string vectorToXML(const std::vector<float> &matrix, int rows, int cols, const std::string &name) = 0 ;
/// Advance all the Synapses in the simulation.
/// Update the state of all edges for a time step.
///
Expand Down
3 changes: 3 additions & 0 deletions Simulator/Edges/NG911/All911Edges.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ class All911Edges : public AllEdges {
virtual void copyEdgeDeviceToHost(void *allEdgesDevice) {};
virtual void copyDeviceEdgeCountsToHost(void *allEdgesDevice) {};
virtual void copyDeviceEdgeSumIdxToHost(void *allEdgesDevice) {};
virtual void copyDeviceEdgeWeightsToHost(void *allEdgesDevice) {};
virtual void Setup_xml_to_file() {};
//virtual std::string vectorToXML(const std::vector<float> &matrix, int rows, int cols, const std::string &name){};
virtual void advanceEdges(void *allEdgesDevice, void *allVerticesDevice,
void *edgeIndexMapDevice) {};
virtual void setAdvanceEdgesDeviceParams() {};
Expand Down
4 changes: 3 additions & 1 deletion Simulator/Edges/Neuro/AllSpikingSynapses.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,9 @@ class AllSpikingSynapses : public AllNeuroEdges {
///
/// @param allEdgesDevice GPU address of the allEdges struct on device memory.
virtual void copyDeviceEdgeSumIdxToHost(void *allEdgesDevice) override;

virtual void copyDeviceEdgeWeightsToHost(void *allEdgesDevice) ;
virtual void Setup_xml_to_file() ;
//virtual std::string vectorToXML(const std::vector<float> &matrix, int rows, int cols, const std::string &name) ;
/// Advance all the Synapses in the simulation.
/// Update the state of all synapses for a time step.
///
Expand Down
54 changes: 54 additions & 0 deletions Simulator/Edges/Neuro/AllSpikingSynapses_d.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,60 @@ void AllSpikingSynapses::copyDeviceEdgeCountsToHost(void *allEdgesDevice)
//allEdges.countVertices_ = 0;
}

//
void AllSpikingSynapses::copyDeviceEdgeWeightsToHost(void *allEdgesDevice)
{
AllSpikingSynapsesDeviceProperties allEdgesDeviceProps;
int numVertices = Simulator::getInstance().getTotalVertices();
BGSIZE maxTotalSynapses = Simulator::getInstance().getMaxEdgesPerVertex() * numVertices;
int vertexCount = Simulator::getInstance().getTotalVertices();
HANDLE_ERROR(cudaMemcpy(&allEdgesDeviceProps, allEdgesDevice,
sizeof(AllSpikingSynapsesDeviceProperties), cudaMemcpyDeviceToHost));
// std::cout << "size: " << vertexCount * vertexCount * sizeof(BGFLOAT) << std::endl;
// std::cout << "W_.data(): " << W_.data() << std::endl;
// std::cout << "allEdgesDeviceProps.W_: " << allEdgesDeviceProps.W_ << std::endl;
HANDLE_ERROR(cudaMemcpy(W_.data(), allEdgesDeviceProps.W_,
maxTotalSynapses * sizeof(BGFLOAT), cudaMemcpyDeviceToHost));
}
// Set countVertices_ to 0 to avoid illegal memory deallocation
// at AllSpikingSynapses deconstructor.
//allEdges.countVertices_ = 0;

std::string vectorToXML(const std::vector<BGFLOAT>& matrix, int rows, int cols, const std::string& name) {
std::ostringstream os;
os << "<Matrix name=\"" << name << "\" rows=\"" << rows << "\" columns=\"" << cols << "\">\n";

int index = 0;
std::for_each(matrix.begin(), matrix.end(), [&](BGFLOAT value) mutable {
if (index % cols == 0) os << " "; // new row
os << value << " ";
if (++index % cols == 0) os << "\n"; // New line after every row
});

os << "</Matrix>";
return os.str();
}

void AllSpikingSynapses::Setup_xml_to_file() {
const std::string filename = "/DATA/Weights.xml"; // Hardcoded filename
int vertexCount = Simulator::getInstance().getTotalVertices();

std::ofstream outFile(filename);
if (!outFile) {
std::cerr << "Error: Unable to open file " << filename << std::endl;
std::cerr << "Error details: " << strerror(errno) << std::endl;
return;
}

std::string xmlContent = vectorToXML(W_, vertexCount, vertexCount, "WeightMatrix");
Copy link
Contributor

@varndorfer varndorfer Mar 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The size of the weights matrix is vertexCount * Simulator::getInstance().getMaxEdgesPerVertex();. See AllSynapsesDeviceFuncs_d.cpp lines 568 and 584

Copy link
Contributor

@varndorfer varndorfer Mar 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is to support having multiple synapses between two neurons but I don't know if that's actually implemented anywhere in the code

outFile << xmlContent;
outFile.close();

std::cout << "Success: XML written to " << filename << std::endl;
}



/// Get summationCoord and in_use in AllEdges struct on device memory.
///
/// @param allEdgesDevice GPU address of the AllSpikingSynapsesDeviceProperties struct
Expand Down
Loading