Gh 147 - Serialization Format V6 - Serialized v9.0.0

* GH-147 - Serialization V6 - Serialized version 9.0.0
- changed the format so that the humans are separated from the nodes to reduce memory usage
- simulation, nodes, and human collections are explicitly managed
- each object can have its own compression type
* Update tests & sync'd serialization helper tests with emod-api

Author: Bridenbecker

Eradication/Node.cpp

@@ -2697,7 +2697,11 @@ namespace Kernel
 
         if ( node.serializationFlags.test( SerializationFlags::Population ) )
         {
-            ar.labelElement("individualHumans"   ) & node.individualHumans;
+            // --------------------------------------------------------
+            // --- handling humans within the SerializedPopulation code
+            // --------------------------------------------------------
+            //ar.labelElement("individualHumans"   ) & node.individualHumans;
+
             ar.labelElement("home_individual_ids") & node.home_individual_ids;
         }
 

Eradication/Node.h

@@ -29,11 +29,21 @@ namespace Kernel
     class SimulationConfig;
     class RANDOMBASE;
     struct INodeEventContext;
-    //typedef 
     class  NodeEventContextHost;
     struct ISimulation;
     struct IMigrationInfoFactory;
+    class Node;
+    struct IIndividualHuman;
+}
+
+namespace SerializedState
+{
+    void AddHumans( Kernel::Node* pNode,
+                    const std::vector<Kernel::IIndividualHuman*>& rHumanCollection );
+}
 
+namespace Kernel
+{
     class IDMAPI Node : public INodeContext, public JsonConfigurable
     {
         GET_SCHEMA_STATIC_WRAPPER(Node)
@@ -46,6 +56,9 @@ namespace Kernel
         friend class ::ReportVector;
         friend class ::DemographicsReport;
 
+        friend void SerializedState::AddHumans( Kernel::Node* pNode,
+                                                const std::vector<Kernel::IIndividualHuman*>& rHumanCollection );
+
     public:
         static Node *CreateNode(ISimulationContext *_parent_sim, ExternalNodeId_t externalNodeId, suids::suid node_suid);
 

Eradication/SerializationParameters.cpp

@@ -16,6 +16,18 @@ END_QUERY_INTERFACE_BODY(SerializationParameters)
 
 SerializationParameters* SerializationParameters::p_instance = nullptr;
 
+// --------------------------------------------------------------------------------
+// --- I'm setting this to 2000 because:
+// ---    A) A lot of scenarios have a population of 1000 +/1 a few.
+// ---       These scenarios don't usually have memory issues, so we want
+// ---       to only have one human colleciton per node.
+// ---    B) Python Post Processing - In a test scenario with three nodes of
+// ---       10K, 11K, 12K people, it took 60 minutes vs 24 minutes in a test that
+// ---       replaced the genomes.  The old version used 22GB of memory.  The new
+// ---       version with a max of 1000 peaked at 9.2GB.  When we switched to 2000,
+// ---       we got a peak of 11.3 GB and took 73 minutes.  Hmm.
+// --------------------------------------------------------------------------------
+#define DEFAULT_MAX_HUMANS_PER_COLLECTION (2000)
 
 SerializationParameters::SerializationParameters()
     : m_serializationReadMask( 0 )
@@ -29,6 +41,7 @@ SerializationParameters::SerializationParameters()
     , m_serialization_time_steps()
     , m_serialization_times()
     , m_supportedFlags( SerializationBitMask_t{}.set( SerializationFlags::LarvalHabitats ) )
+    , m_MaxHumansPerCollection( DEFAULT_MAX_HUMANS_PER_COLLECTION )
 { }
 
 
@@ -65,6 +78,11 @@ std::string SerializationParameters::GetSerializedPopulationFilename() const
     return population_filename;
 }
 
+int32_t SerializationParameters::GetMaxHumansPerCollection() const
+{
+    return m_MaxHumansPerCollection;
+}
+
 std::deque<int32_t> SerializationParameters::GetSerializedTimeSteps( int32_t steps ) const
 {
     float start_time = GET_CONFIGURABLE( SimulationConfig )->starttime;
@@ -228,6 +246,16 @@ bool SerializationParameters::Configure( const Configuration * inputJson )
     initConfig( "Serialization_Precision", m_serialization_precision, inputJson, MetadataDescriptor::Enum( "Serialization_Precision", Serialization_Precision_DESC_TEXT, MDD_ENUM_ARGS( SerializationPrecision ) ), "Serialized_Population_Writing_Type", "TIMESTEP,TIME" );
     initConfigTypeMap( "Serialization_Mask_Node_Write", ( uint32_t* )&m_serializationWriteMask, SerializationMask_Node_Write_DESC_TEXT, 0, UINT32_MAX, 0, "Serialized_Population_Writing_Type","TIME,TIMESTEP" );
 
+    if( JsonConfigurable::_dryrun || inputJson->Exist( "Serialization_Max_Humans_Per_Collection" ) )
+    {
+        initConfigTypeMap( "Serialization_Max_Humans_Per_Collection",
+                           &m_MaxHumansPerCollection,
+                           Serialization_Max_Humans_Per_Collection_DESC_TEXT,
+                           5, INT32_MAX, DEFAULT_MAX_HUMANS_PER_COLLECTION,
+                           "Serialized_Population_Writing_Type",
+                           "TIME,TIMESTEP" );
+    }
+
     // Read serialized population
     initConfig( "Serialized_Population_Reading_Type", m_serialization_read_type, inputJson, MetadataDescriptor::Enum( "Serialized_Population_Reading_Type", Serialized_Population_Reading_Type_DESC_TEXT, MDD_ENUM_ARGS( SerializationTypeRead ) ) );
     initConfigTypeMap( "Serialization_Mask_Node_Read", ( uint32_t* )&m_serializationReadMask, SerializationMask_Node_Read_DESC_TEXT, 0, UINT32_MAX, 0, "Serialized_Population_Reading_Type", "READ" );

Eradication/SerializationParameters.h

@@ -53,6 +53,7 @@ GET_SCHEMA_STATIC_WRAPPER( SerializationParameters )
     bool GetCreateRngFromSerializedData() const;
     std::string GetSerializedPopulationPath() const;
     std::string GetSerializedPopulationFilename() const;
+    int32_t GetMaxHumansPerCollection() const;
 
 private:
     SerializationParameters( const SerializationParameters& ) = delete;
@@ -72,4 +73,5 @@ GET_SCHEMA_STATIC_WRAPPER( SerializationParameters )
     std::vector<float> m_serialization_times;
     static SerializationParameters* p_instance;
     SerializationBitMask_t m_supportedFlags;
+    int32_t m_MaxHumansPerCollection;
 };