1. refactor MPI_Wrapper and Cereal_Func

Author: PeizeLin

include/Comm/global/Cereal_Func.h

@@ -18,6 +18,8 @@ class Cereal_Func
 {
   public:
 
+	inline Cereal_Func();
+
 	// every 2^exponent_align char concatenate to 1 word
 	inline std::size_t align_stringstream(std::stringstream &ss);
 

include/Comm/global/Cereal_Func.hpp

@@ -8,6 +8,7 @@
 #include "Cereal_Func.h"
 #include "Cereal_Types.h"
 #include "MPI_Wrapper.h"
+#include "Global_Func.h"
 
 #include <cereal/archives/binary.hpp>
 #include <sstream>
@@ -20,6 +21,23 @@
 namespace Comm
 {
 
+inline Cereal_Func::Cereal_Func()
+{
+  #if MPI_VERSION>=4
+	using int_type = MPI_Count;
+  #else
+	using int_type = int;
+  #endif
+
+	// assuming MPI communication is less than max(1TB, memory availablle now) for most case,
+	// so initialize here to avoid thread conflict in the future.
+	const std::size_t TB = std::size_t(1)<<12;
+	const std::size_t memory_max = std::max(TB, Global_Func::memory_available());
+	const std::size_t times = std::ceil( double(memory_max) / double(std::numeric_limits<int_type>::max()) );
+	const std::size_t exponent_align = std::ceil( std::log(times) / std::log(2) );
+	this->char_contiguous.resize(exponent_align);
+}	
+
 // every 2^exponent_align char concatenate to 1 word
 	// <<exponent_align means *2^exponent_align
 	// >>exponent_align means /2^exponent_align
@@ -34,7 +52,7 @@ inline std::size_t Cereal_Func::align_stringstream(std::stringstream &ss)
 	const std::size_t size_old = ss.str().size();				// Inefficient, should be optimized
 	const std::size_t times = std::ceil( double(size_old) / double(std::numeric_limits<int_type>::max()) );
 	const std::size_t exponent_align = std::ceil( std::log(times) / std::log(2) );
-	this->char_contiguous(exponent_align);
+	this->char_contiguous.resize(exponent_align);
 	constexpr char c0 = 0;
 	const std::size_t size_align = 1<<exponent_align;
 	if(size_old%size_align)
@@ -47,6 +65,7 @@ inline std::size_t Cereal_Func::align_stringstream(std::stringstream &ss)
 // Send str
 inline void Cereal_Func::mpi_send(const std::string &str, const std::size_t exponent_align, const int rank_recv, const int tag, const MPI_Comm &mpi_comm)
 {
+	this->char_contiguous.resize(exponent_align);
   #if MPI_VERSION>=4
 	MPI_CHECK( MPI_Send_c( str.c_str(), str.size()>>exponent_align, this->char_contiguous(exponent_align), rank_recv, tag, mpi_comm ) );
   #else
@@ -73,6 +92,7 @@ void Cereal_Func::mpi_send(const int rank_recv, const int tag, const MPI_Comm &m
 // Isend str
 inline void Cereal_Func::mpi_isend(const std::string &str, const std::size_t exponent_align, const int rank_recv, const int tag, const MPI_Comm &mpi_comm, MPI_Request &request)
 {
+	this->char_contiguous.resize(exponent_align);
   #if MPI_VERSION>=4
 	MPI_CHECK( MPI_Isend_c( str.c_str(), str.size()>>exponent_align, this->char_contiguous(exponent_align), rank_recv, tag, mpi_comm, &request ) );
   #else
@@ -102,6 +122,7 @@ inline std::vector<char> Cereal_Func::mpi_recv(const MPI_Comm &mpi_comm, MPI_Sta
 {
 	for(std::size_t exponent_align=0; ; ++exponent_align)
 	{
+		this->char_contiguous.resize(exponent_align);
 		const MPI_Datatype mpi_type = this->char_contiguous(exponent_align);
 	  #if MPI_VERSION>=4
 		MPI_Count size;		MPI_CHECK( MPI_Get_count_c( &status, mpi_type, &size ) );
@@ -146,6 +167,7 @@ inline std::vector<char> Cereal_Func::mpi_mrecv(MPI_Message &message_recv, const
 {
 	for(std::size_t exponent_align=0; ; ++exponent_align)
 	{
+		this->char_contiguous.resize(exponent_align);
 		const MPI_Datatype mpi_type = this->char_contiguous(exponent_align);
 	  #if MPI_VERSION>=4
 		MPI_Count size;		MPI_CHECK( MPI_Get_count_c( &status, mpi_type, &size ) );

include/Comm/global/Global_Func.hpp

@@ -22,14 +22,17 @@ namespace Comm
 
 namespace Global_Func
 {
+#if defined(__MACH__)
 	inline std::size_t memory_available()
 	{
-#if defined(__MACH__)
 		// HACK: always assume half of the system memory free. Only for build and test on macOS
-		std::size_t pages = sysconf(_SC_PHYS_PAGES);
-		std::size_t page_size = sysconf(_SC_PAGE_SIZE);
+		const std::size_t pages = sysconf(_SC_PHYS_PAGES);
+		const std::size_t page_size = sysconf(_SC_PAGE_SIZE);
 		return pages * page_size / 2;
+	}
 #else
+	inline std::size_t memory_available()
+	{
 		constexpr std::size_t kB_to_B = 1024;
 		std::ifstream ifs("/proc/meminfo");
 		int num = 0;

include/Comm/global/MPI_Wrapper.h

@@ -6,6 +6,7 @@
 #pragma once
 
 #include <mpi.h>
+#include <omp.h>
 #include <stdexcept>
 #include <string>
 #include <vector>
@@ -53,8 +54,13 @@ namespace MPI_Wrapper
 	class MPI_Type_Contiguous_Pool
 	{
 	  public:
-		MPI_Datatype operator()(const std::size_t exponent)
+		MPI_Datatype operator()(const std::size_t exponent)	const
 		{
+			return this->type_pool.at(exponent);
+		}
+		void resize(const std::size_t exponent)
+		{
+			#pragma omp critical(MPI_Type_Contiguous_Pool)
 			if(this->type_pool.size()<exponent+1)
 			{
 				const std::size_t size_old = this->type_pool.size();
@@ -65,7 +71,6 @@ namespace MPI_Wrapper
 					MPI_CHECK( MPI_Type_commit( &this->type_pool[ie] ) );
 				}
 			}
-			return this->type_pool[exponent];
 		}
 		MPI_Type_Contiguous_Pool(const MPI_Datatype &type_base_in)
 		{