diff --git a/Simulator/Connections/Connections.cpp b/Simulator/Connections/Connections.cpp index a8a63fc0e..48966de19 100644 --- a/Simulator/Connections/Connections.cpp +++ b/Simulator/Connections/Connections.cpp @@ -81,9 +81,8 @@ void Connections::createEdgeIndexMap() /// Update the connections status in every epoch. /// -/// @param vertices The vertex list to search from. /// @return true if successful, false otherwise. -bool Connections::updateConnections(AllVertices &vertices) +bool Connections::updateConnections() { return false; } diff --git a/Simulator/Connections/Connections.h b/Simulator/Connections/Connections.h index d8bcc8596..adfe57c64 100644 --- a/Simulator/Connections/Connections.h +++ b/Simulator/Connections/Connections.h @@ -67,9 +67,8 @@ class Connections { /// Update the connections status in every epoch. /// - /// @param vertices The vertex list to search from. /// @return true if successful, false otherwise. - virtual bool updateConnections(AllVertices &vertices); + virtual bool updateConnections(); /// Cereal serialization method template void serialize(Archive &archive); diff --git a/Simulator/Connections/NG911/Connections911.cpp b/Simulator/Connections/NG911/Connections911.cpp index 7d5c499e3..d319c1ec5 100644 --- a/Simulator/Connections/NG911/Connections911.cpp +++ b/Simulator/Connections/NG911/Connections911.cpp @@ -63,7 +63,7 @@ void Connections911::printParameters() const #if !defined(USE_GPU) /// Update the connections status in every epoch. -bool Connections911::updateConnections(AllVertices &vertices) +bool Connections911::updateConnections() { // Only run on the first epoch if (Simulator::getInstance().getCurrentStep() != 1) { @@ -73,6 +73,7 @@ bool Connections911::updateConnections(AllVertices &vertices) // Record old type map int numVertices = Simulator::getInstance().getTotalVertices(); Layout &layout = Simulator::getInstance().getModel().getLayout(); + AllVertices &vertices = layout.getVertices(); oldTypeMap_ = layout.vertexTypeMap_; // Erase PSAPs diff --git a/Simulator/Connections/NG911/Connections911.h b/Simulator/Connections/NG911/Connections911.h index 25d38c63e..85284ce67 100644 --- a/Simulator/Connections/NG911/Connections911.h +++ b/Simulator/Connections/NG911/Connections911.h @@ -72,9 +72,8 @@ class Connections911 : public Connections { /// Update the connections status in every epoch. /// Uses the parent definition for USE_GPU /// - /// @param vertices The Vertex list to search from. /// @return true if successful, false otherwise. - virtual bool updateConnections(AllVertices &vertices) override; + virtual bool updateConnections() override; /// Finds the outgoing edge from the given vertex to the Responder closest to /// the emergency call location diff --git a/Simulator/Connections/Neuro/ConnGrowth.cpp b/Simulator/Connections/Neuro/ConnGrowth.cpp index beaad1660..41cc2dcd6 100644 --- a/Simulator/Connections/Neuro/ConnGrowth.cpp +++ b/Simulator/Connections/Neuro/ConnGrowth.cpp @@ -126,10 +126,12 @@ void ConnGrowth::printParameters() const /// Update the connections status in every epoch. /// -/// @param vertices The vertex list to search from. /// @return true if successful, false otherwise. -bool ConnGrowth::updateConnections(AllVertices &vertices) +bool ConnGrowth::updateConnections() { + Layout &layout = Simulator::getInstance().getModel().getLayout(); + AllVertices &vertices = layout.getVertices(); + // Update Connections data updateConns(vertices); diff --git a/Simulator/Connections/Neuro/ConnGrowth.h b/Simulator/Connections/Neuro/ConnGrowth.h index e6a88078c..0a6abb41f 100644 --- a/Simulator/Connections/Neuro/ConnGrowth.h +++ b/Simulator/Connections/Neuro/ConnGrowth.h @@ -103,9 +103,8 @@ class ConnGrowth : public Connections { /// Update the connections status in every epoch. /// - /// @param vertices The vertex list to search from. /// @return true if successful, false otherwise. - virtual bool updateConnections(AllVertices &vertices) override; + virtual bool updateConnections() override; /// Cereal serialization method template void serialize(Archive &archive); diff --git a/Simulator/Connections/Neuro/ConnGrowth_d.cpp b/Simulator/Connections/Neuro/ConnGrowth_d.cpp index 148834fea..1854bfe83 100644 --- a/Simulator/Connections/Neuro/ConnGrowth_d.cpp +++ b/Simulator/Connections/Neuro/ConnGrowth_d.cpp @@ -77,6 +77,12 @@ void ConnGrowth::updateEdgesWeights(int numVertices, AllVertices &vertices, AllE // copy device synapse count to host memory edges.copyDeviceEdgeCountsToHost(allEdgesDevice); - // copy device synapse summation coordinate to host memory - dynamic_cast(edges).copyDeviceEdgeSumIdxToHost(allEdgesDevice); + + // copy device synapse summation coordinate and weights to host memory + AllSpikingSynapses &synapses = dynamic_cast(edges); + synapses.copyDeviceEdgeSumIdxToHost(allEdgesDevice); + synapses.copyDeviceEdgeWeightsToHost(allEdgesDevice); + + // output weight matrix every epoch + synapses.outputWeights(simulator.getCurrentStep()); } diff --git a/Simulator/Connections/Neuro/ConnStatic.cpp b/Simulator/Connections/Neuro/ConnStatic.cpp index 7d168d199..ae966a45f 100644 --- a/Simulator/Connections/Neuro/ConnStatic.cpp +++ b/Simulator/Connections/Neuro/ConnStatic.cpp @@ -89,3 +89,14 @@ void ConnStatic::loadParameters() void ConnStatic::printParameters() const { } + +/// Output the weights matrix after every epoch. +/// +/// @return true if successful, false otherwise. +bool ConnStatic::updateConnections() +{ + AllNeuroEdges &synapses = dynamic_cast(*edges_); + synapses.outputWeights(Simulator::getInstance().getCurrentStep()); + + return true; +} diff --git a/Simulator/Connections/Neuro/ConnStatic.h b/Simulator/Connections/Neuro/ConnStatic.h index 47256709c..35014e75b 100644 --- a/Simulator/Connections/Neuro/ConnStatic.h +++ b/Simulator/Connections/Neuro/ConnStatic.h @@ -80,6 +80,11 @@ class ConnStatic : public Connections { return destVertexIndexCurrentEpoch_; } + /// Output the weights matrix after every epoch. + /// + /// @return true if successful, false otherwise. + virtual bool updateConnections() override; + /// Cereal serialization method template void serialize(Archive &archive); diff --git a/Simulator/Core/CPUModel.cpp b/Simulator/Core/CPUModel.cpp index 1e335fca6..f9388e96a 100644 --- a/Simulator/Core/CPUModel.cpp +++ b/Simulator/Core/CPUModel.cpp @@ -35,7 +35,7 @@ void CPUModel::advance() void CPUModel::updateConnections() { // Update Connections data - if (connections_->updateConnections(layout_->getVertices())) { + if (connections_->updateConnections()) { connections_->updateEdgesWeights(); // create edge inverse map connections_->createEdgeIndexMap(); diff --git a/Simulator/Core/GPUModel.cpp b/Simulator/Core/GPUModel.cpp index 5aaf15571..c652e28ba 100644 --- a/Simulator/Core/GPUModel.cpp +++ b/Simulator/Core/GPUModel.cpp @@ -237,7 +237,7 @@ void GPUModel::updateConnections() vertices.copyFromDevice(allVerticesDevice_); // Update Connections data - if (connections_->updateConnections(vertices)) { + if (connections_->updateConnections()) { connections_->updateEdgesWeights(Simulator::getInstance().getTotalVertices(), vertices, edges, allVerticesDevice_, allEdgesDevice_, getLayout()); // create edge index map diff --git a/Simulator/Edges/Neuro/AllNeuroEdges.h b/Simulator/Edges/Neuro/AllNeuroEdges.h index e8f0bdfb3..86810f6e4 100644 --- a/Simulator/Edges/Neuro/AllNeuroEdges.h +++ b/Simulator/Edges/Neuro/AllNeuroEdges.h @@ -84,6 +84,9 @@ class AllNeuroEdges : public AllEdges { /// Cereal serialization method template void serialize(Archive &archive); + /// Output weights and srcIndex to xml + virtual void outputWeights(int epochNum) = 0; + protected: /// Setup the internal structure of the class (allocate memories and initialize them). /// diff --git a/Simulator/Edges/Neuro/AllSpikingSynapses.cpp b/Simulator/Edges/Neuro/AllSpikingSynapses.cpp index 0c7b54bb3..23328177a 100644 --- a/Simulator/Edges/Neuro/AllSpikingSynapses.cpp +++ b/Simulator/Edges/Neuro/AllSpikingSynapses.cpp @@ -333,3 +333,61 @@ void AllSpikingSynapses::printSynapsesProps() const } } } + +string vectorToXML(const vector &matrix, int rows, int cols, const string &name) +{ + ostringstream os; + os << "<" << name << " rows=\"" << rows << "\" columns=\"" << cols << "\">\n"; + + int index = 0; + for_each(matrix.begin(), matrix.end(), [&](BGFLOAT value) mutable { + os << " " << value << "\n"; + index++; + }); + + os << "\n"; + return os.str(); +} + +string vectorToXML(const vector &matrix, int rows, int cols, const string &name) +{ + ostringstream os; + os << "<" << name << " rows=\"" << rows << "\" columns=\"" << cols << "\">\n"; + + int index = 0; + for_each(matrix.begin(), matrix.end(), [&](int value) mutable { + os << " " << value << "\n"; + + index++; + }); + + os << "\n"; + return os.str(); +} + +void AllSpikingSynapses::outputWeights(int epochNum) +{ + const std::string filename = "./Output/Results/weights-epoch-" + std::to_string(epochNum) + + ".xml"; // Hardcoded filename + int vertexCount = Simulator::getInstance().getTotalVertices(); + + ofstream outFile(filename); + if (!outFile) { + cerr << "Error: Unable to open file " << filename << endl; + cerr << "Error details: " << strerror(errno) << endl; + return; + } + + int maxEdges = Simulator::getInstance().getMaxEdgesPerVertex(); + + string wContent = vectorToXML(W_, vertexCount, maxEdges, "WeightMatrix"); + string srcContent = vectorToXML(sourceVertexIndex_, vertexCount, maxEdges, "SourceVertexIndex"); + + outFile << "\n"; + outFile << wContent; + outFile << srcContent; + outFile << ""; + outFile.close(); + + cout << "Weights matrix output to: " << filename << endl; +} diff --git a/Simulator/Edges/Neuro/AllSpikingSynapses.h b/Simulator/Edges/Neuro/AllSpikingSynapses.h index 0cd04821b..001a91cdf 100644 --- a/Simulator/Edges/Neuro/AllSpikingSynapses.h +++ b/Simulator/Edges/Neuro/AllSpikingSynapses.h @@ -87,6 +87,9 @@ class AllSpikingSynapses : public AllNeuroEdges { /// Cereal serialization method template void serialize(Archive &archive); + /// Output weights and srcIndex to xml + virtual void outputWeights(int epochNum); + protected: /// Setup the internal structure of the class (allocate memories and initialize them). /// @@ -196,6 +199,12 @@ class AllSpikingSynapses : public AllNeuroEdges { /// @param allEdgesDevice GPU address of the allEdges struct on device memory. void copyDeviceEdgeSumIdxToHost(void *allEdgesDevice); + /// Get weights matrix in AllEdges struct on device memory. + /// + /// @param allEdgesDevice GPU address of the AllSpikingSynapsesDeviceProperties struct + /// on device memory. + virtual void copyDeviceEdgeWeightsToHost(void *allEdgesDevice); + protected: /// Allocate GPU memories to store all synapses' states, /// and copy them from host to GPU memory. diff --git a/Simulator/Edges/Neuro/AllSpikingSynapses_d.cpp b/Simulator/Edges/Neuro/AllSpikingSynapses_d.cpp index 94364e4a2..f426292b3 100644 --- a/Simulator/Edges/Neuro/AllSpikingSynapses_d.cpp +++ b/Simulator/Edges/Neuro/AllSpikingSynapses_d.cpp @@ -273,6 +273,28 @@ void AllSpikingSynapses::copyDeviceEdgeCountsToHost(void *allEdgesDevice) //allEdges.countVertices_ = 0; } +/// Get weights matrix in AllEdges struct on device memory. +/// +/// @param allEdgesDevice GPU address of the AllSpikingSynapsesDeviceProperties struct +/// on device memory. +void AllSpikingSynapses::copyDeviceEdgeWeightsToHost(void *allEdgesDevice) +{ + AllSpikingSynapsesDeviceProperties allEdgesDeviceProps; + + int numVertices = Simulator::getInstance().getTotalVertices(); + BGSIZE maxTotalSynapses = Simulator::getInstance().getMaxEdgesPerVertex() * numVertices; + + HANDLE_ERROR(cudaMemcpy(&allEdgesDeviceProps, allEdgesDevice, + sizeof(AllSpikingSynapsesDeviceProperties), cudaMemcpyDeviceToHost)); + + // std::cout << "size: " << vertexCount * vertexCount * sizeof(BGFLOAT) << std::endl; + // std::cout << "W_.data(): " << W_.data() << std::endl; + // std::cout << "allEdgesDeviceProps.W_: " << allEdgesDeviceProps.W_ << std::endl; + + HANDLE_ERROR(cudaMemcpy(W_.data(), allEdgesDeviceProps.W_, maxTotalSynapses * sizeof(BGFLOAT), + cudaMemcpyDeviceToHost)); +} + /// Get summationCoord and in_use in AllEdges struct on device memory. /// /// @param allEdgesDevice GPU address of the AllSpikingSynapsesDeviceProperties struct diff --git a/Tools/XMLToGraphML/README.md b/Tools/XMLToGraphML/README.md new file mode 100644 index 000000000..b36f0e0b8 --- /dev/null +++ b/Tools/XMLToGraphML/README.md @@ -0,0 +1,17 @@ +# XML to GraphML Converter + +## Overview +This script was created to parse the weight matrix output from neuro simulations and generate a graphml file for input into subsequent simulations. + +## `getGraphEdges` +* Must be run with Python3.9 +* Takes a weight matrix xml file and existing graphml file as input +* Adds the edge weights to the graphml file + +## Contributors +- Vanessa Arndorfer + + + + + diff --git a/Tools/XMLToGraphML/getGraphEdges.py b/Tools/XMLToGraphML/getGraphEdges.py new file mode 100644 index 000000000..57d8cb12c --- /dev/null +++ b/Tools/XMLToGraphML/getGraphEdges.py @@ -0,0 +1,124 @@ +''' +GETGRAPHEDGES Generate a graphml file from the weights matrix simulation output + +This function reads the Graphitti weight matrix output from the +growth simulation and reformats it to graphml for input into the +STDP simulation. + +Input: +weights_file - Weight matrix output from Graphitti. The entire path can be used; for example + '/CSSDIV/research/biocomputing/data/2025/tR_1.0--fE_0.90_10000/weights-epoch-1.xml' +graphml_file - graphml file to add the edges. This is typically the same file + used as input to the Graphitti simulation. + +Output: + - weight_graph.graphml + +Author: Vanessa Arndorfer (vanessa.arndorfer@gmail.com) +Last updated: 07/01/2025 +''' + +import numpy as np +import os +import pandas as pd +import sys +import xml.etree.ElementTree as ET + + +def xmlToNumpy(node, rows, cols): + print("Converting xml to matrix for: " + node.tag) + + m = np.zeros(shape=(rows, cols)) + r = 0 + c = 0 + for child in node: + m[r][c] = float(child.text) + c += 1 + + # new row + if c == cols: + r += 1 + c = 0 + + return m + + +def getWeightMatrix(file_name, src_root, weights_root): + print("Building weights matrix for: " + file_name) + + tree = ET.parse(file_name) + root = tree.getroot() + + idNum = 0 + srcIdx = root.find(src_root) + weights = root.find(weights_root) + + rows = 10000 + cols = 200 + + srcIdx_np = xmlToNumpy(srcIdx, rows, cols) + weights_np = xmlToNumpy(weights, rows, cols) + + print("Converting matrices into square format...") + edge_weights = np.zeros(shape=(rows, rows)) + + weight_count = 0 + + for r in range(weights_np.shape[0]): + for c in range(weights_np.shape[1]): + if weights_np[r][c] != 0: + weight_count += 1 + src = int(srcIdx_np[r][c]) + edge_weights[src][r] = weights_np[r][c] + + print("Total weighted edges: " + str(weight_count)) + + return edge_weights + + +def getEdgeGraphML(edge_weights, graphml_file, output_dir): + print("Adding edges to the graph file: " + graphml_file) + + # register xml namespaces + ET.register_namespace('', "http://graphml.graphdrawing.org/xmlns") + ET.register_namespace('xsi', "http://www.w3.org/2001/XMLSchema-instance") + ET.register_namespace('xsi:schemaLocation', "http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd") + + tree = ET.parse(graphml_file) + root = tree.getroot() + + # define weight attribute for edges + weight_key = ET.Element("key", attrib={"id":"weight", "for":"edge", "attr.name":"weight", "attr.type":"double"}) + root.append(weight_key) + + idNum = 0 + graph = root.find('./{http://graphml.graphdrawing.org/xmlns}graph') + + for r in range(edge_weights.shape[0]): + for c in range(edge_weights.shape[1]): + if edge_weights[r][c] != 0: + edgeId = "e" + str(idNum) + edge = ET.Element("edge", attrib={"id":edgeId, "source":str(r), "target":str(c)}) + data = ET.SubElement(edge, "data", attrib={"key": "weight"}) + data.text = str(edge_weights[r][c]) + graph.append(edge) + idNum += 1 + + print(str(idNum) + " edges created") + ET.indent(tree, space="\t", level=0) + tree.write(output_dir + "/weight_graph.graphml", encoding="utf-8", xml_declaration=True) + + +if __name__ == "__main__": + # execution format: python3.9 ./getGraphEdges.py weights_file graphml_file + # example: python3.9 ./getGraphEdges.py /CSSDIV/research/biocomputing/data/2025/tR_1.0--fE_0.90_10000/weights-epoch-25.xml configfiles/graphs/fE_0.90_10000.graphml + weights_file = sys.argv[1] + graphml_file = sys.argv[2] + + ht = os.path.split(weights_file) + output_dir = ht[0] + print("Output dir: " + output_dir) + + df = getWeightMatrix(weights_file, "SourceVertexIndex", "WeightMatrix") + getEdgeGraphML(df, graphml_file, output_dir) + \ No newline at end of file