Fix to edge attribute reading

CMakeLists.txt

@@ -15,7 +15,7 @@ endif()
 
 include(${PROJECT_SOURCE_DIR}/cmake/neuroh5_utils.cmake)
 
-set(NEUROH5_VERSION 0.1.17)
+set(NEUROH5_VERSION 0.1.18)
 
 cmake_policy(SET CMP0074 NEW) # enables use of HDF5_ROOT variable
 

setup.py

@@ -141,7 +141,7 @@ def build_extensions(self):
     name="NeuroH5",
     package_dir={"": "python"},
     packages=["neuroh5"],
-    version="0.1.17",
+    version="0.1.18",
     maintainer="Ivan Raikov",
     maintainer_email="ivan.g.raikov@gmail.com",
     description="A parallel HDF5-based library for storage and processing of large-scale graphs and neural cell model attributes.",

src/graph/edge_attributes.cc

@@ -15,6 +15,7 @@
 #include "path_names.hh"
 #include "serialize_data.hh"
 #include "read_template.hh"
+#include "mpi_debug.hh"
 #include "throw_assert.hh"
 
 #include <iostream>
@@ -379,6 +380,10 @@ namespace neuroh5
                        "read_edge_attributes: error in H5Pset_dxpl_mpio");
         }
 #endif
+      mpi::MPI_DEBUG(comm, "read_edge_attributes: reading attributes for ", 
+                     src_pop_name, " -> ", dst_pop_name, 
+                     " namespace ", name_space, " attribute ", attr_name, 
+                     " edge_base: ", edge_base, " edge count: ", edge_count);
 
       string dset_path = hdf5::edge_attribute_path(src_pop_name, dst_pop_name, name_space, attr_name);
       ierr = hdf5::exists_dataset (file, dset_path.c_str());

src/graph/read_projection.cc

@@ -123,6 +123,11 @@ namespace neuroh5
                    "read_projection: error in append_edge_map");
       local_num_nodes = prj_edge_map.size();
 
+      mpi::MPI_DEBUG(comm, "read_projection: local_num_nodes = ", 
+                     local_num_nodes, 
+                     " local_prj_num_edges = ", local_prj_num_edges,
+                     " edge_count = ", edge_count);
+
       // ensure that all edges in the projection have been read and
       // appended to edge_list
       throw_assert(local_prj_num_edges == edge_count,

src/graph/scatter_read_projection.cc

@@ -124,11 +124,12 @@ namespace neuroh5
               hsize_t local_read_blocks;
 
               mpi::MPI_DEBUG(io_comm, "scatter_read_projection: reading projection ", src_pop_name, " -> ", dst_pop_name);
-              throw_assert_nomsg(hdf5::read_projection_datasets(io_comm, file_name, src_pop_name, dst_pop_name,
-                                                                block_base, edge_base,
-                                                                dst_blk_ptr, dst_idx, dst_ptr, src_idx,
-                                                                total_num_edges, total_read_blocks, local_read_blocks,
-                                                                offset, numitems * size) >= 0);
+              throw_assert(hdf5::read_projection_datasets(io_comm, file_name, src_pop_name, dst_pop_name,
+                                                          block_base, edge_base,
+                                                          dst_blk_ptr, dst_idx, dst_ptr, src_idx,
+                                                          total_num_edges, total_read_blocks, local_read_blocks,
+                                                          offset, numitems * size) >= 0,
+                           "error in read_projection_datasets");
 
               mpi::MPI_DEBUG(io_comm, "scatter_read_projection: validating projection ", src_pop_name, " -> ", dst_pop_name);
               // validate the edges
@@ -153,9 +154,10 @@ namespace neuroh5
 
 
               // append to the edge map
-              throw_assert_nomsg(data::append_rank_edge_map(rank, size, dst_start, src_start, dst_blk_ptr, dst_idx, dst_ptr, src_idx,
-                                                            attr_namespaces, edge_attr_map, node_rank_map, num_edges, prj_rank_edge_map,
-                                                            edge_map_type) >= 0);
+              throw_assert(data::append_rank_edge_map(rank, size, dst_start, src_start, dst_blk_ptr, dst_idx, dst_ptr, src_idx,
+                                                      attr_namespaces, edge_attr_map, node_rank_map, num_edges, prj_rank_edge_map,
+                                                      edge_map_type) >= 0,
+                           "error in append_rank_edge_map");
 
               mpi::MPI_DEBUG(io_comm, "scatter_read_projection: read ", num_edges,
                              " edges from projection ", src_pop_name, " -> ", dst_pop_name);
@@ -180,7 +182,13 @@ namespace neuroh5
 
 
           MPI_Comm_free(&io_comm);
-          throw_assert_nomsg(MPI_Bcast(&total_read_blocks, 1, MPI_SIZE_T, io_rank_root, all_comm) == MPI_SUCCESS);
+          MPI_Request bcast_req;
+          throw_assert(MPI_Ibcast(&total_read_blocks, 1, MPI_SIZE_T, io_rank_root, all_comm,
+                                  &bcast_req) == MPI_SUCCESS,
+                       "error in MPI_Ibcast");
+          throw_assert(MPI_Wait(&bcast_req, MPI_STATUS_IGNORE) == MPI_SUCCESS,
+                       "error in MPI_Wait");
+
           throw_assert_nomsg(mpi::alltoallv_vector<char>(all_comm, MPI_CHAR, sendcounts, sdispls, sendbuf,
                                                          recvcounts, rdispls, recvbuf) >= 0);
         }
@@ -203,18 +211,31 @@ namespace neuroh5
                 data::serialize_data(edge_attr_names, sendbuf);
                 sendbuf_size = sendbuf.size();
               }
+
+            MPI_Request bcast_req;
 
             throw_assert_nomsg(MPI_Barrier(all_comm) == MPI_SUCCESS);
-            throw_assert_nomsg(MPI_Bcast(&sendbuf_size, 1, MPI_UINT32_T, 0, all_comm) == MPI_SUCCESS);
+            throw_assert(MPI_Ibcast(&sendbuf_size, 1, MPI_UINT32_T, 0, all_comm,
+                                    &bcast_req) == MPI_SUCCESS,
+                         "error in MPI_Ibcast");
+            throw_assert(MPI_Wait(&bcast_req, MPI_STATUS_IGNORE) == MPI_SUCCESS,
+                         "error in MPI_Wait");
             sendbuf.resize(sendbuf_size);
-            throw_assert_nomsg(MPI_Bcast(&sendbuf[0], sendbuf_size, MPI_CHAR, 0, all_comm) == MPI_SUCCESS);
+            throw_assert(MPI_Ibcast(&sendbuf[0], sendbuf_size, MPI_CHAR, 0, all_comm,
+                                    &bcast_req) == MPI_SUCCESS,
+                         "error in MPI_Ibcast");
+            throw_assert(MPI_Wait(&bcast_req, MPI_STATUS_IGNORE) == MPI_SUCCESS,
+                         "error in MPI_Wait");
+
+            mpi::MPI_DEBUG(all_comm, "scatter_read_projection: sendbuf size is ", sendbuf_size);
 
             if (rank != 0)
               {
                 data::deserialize_data(sendbuf, edge_attr_names);
               }
             edge_attr_names_vector.push_back(edge_attr_names);
 
+            mpi::MPI_DEBUG(all_comm, "scatter_read_projection: deserialized edge attr names");
           }
       }
 

src/hdf5/create_file_toplevel.cc

@@ -1,4 +1,5 @@
 
+#include <mpi.h>
 #include <hdf5.h>
 
 #include <string>

src/hdf5/file_access.cc

@@ -1,4 +1,4 @@
-
+#include <mpi.h>
 #include <hdf5.h>
 #include <string>
 #include <vector>