Merge pull request #741 from htm-community/pickled_network

Fix for pickling a Network

Author: dkeeney

bindings/py/cpp_src/bindings/engine/py_Engine.cpp

@@ -267,6 +267,21 @@ namespace htm_ext
 
                 r.setInputData(name, s);
             });
+            
+        py_Region.def(py::pickle(
+            [](const Region& self) {
+                std::stringstream ss;
+                  self.save(ss);
+                return py::bytes(ss.str());
+            },
+            // Note: a de-serialized Region will need to be reattached to a Network
+            //           before it could be used.  See Network::addRegion( Region*)
+            [](const py::bytes& s) {
+                std::istringstream ss(s);
+                Region self;
+                self.load(ss);
+                return self;
+        }));
 
 
         py_Region.def("getParameterInt32", &Region::getParameterInt32)
@@ -407,6 +422,11 @@ namespace htm_ext
                     , py::arg("name")
                     , py::arg("nodeType" )
                     , py::arg("nodeParams"));
+        py_Network.def("addRegion", (Region_Ptr_t (htm::Network::*)(
+                    Region_Ptr_t&))
+                    &htm::Network::addRegion,
+                    "add region for deserialization."
+                    , py::arg("region"));
 
         py_Network.def("getRegions", &htm::Network::getRegions)
             .def("getRegion",          &htm::Network::getRegion)
@@ -427,6 +447,18 @@ namespace htm_ext
             .def("saveToFile",      &htm::Network::saveToFile, py::arg("file"), py::arg("fmt") = SerializableFormat::BINARY)
             .def("loadFromFile",    &htm::Network::loadFromFile, py::arg("file"), py::arg("fmt") = SerializableFormat::BINARY);
 
+        py_Network.def(py::pickle(
+            [](const Network& self) {
+                std::stringstream ss;
+                self.save(ss);
+                return py::bytes(ss.str());
+            },
+            [](const py::bytes& s) {
+                std::istringstream ss(s);
+                Network self;
+                self.load(ss);
+                return self;  
+        }));
 
         py_Network.def("link", &htm::Network::link
             , "Defines a link between regions"

bindings/py/cpp_src/plugin/PyBindRegion.cpp

@@ -304,30 +304,39 @@ namespace py = pybind11;
         // 1. serialize main state using pickle
         // 2. call class method to serialize external state
 
-        // 1. Serialize main state of the Python module
-				//    We want this to end up in the open stream obtained from bundle.
-				//    a. We first pickle the python into a temporary file.
-				//    b. copy the file into our open stream.
-
-				std::string tmp_pickle = "pickle.tmp";
-		    py::tuple args = py::make_tuple(tmp_pickle, "wb");
-		    auto f = py::module::import("__builtin__").attr("file")(*args);
+        //    Serialize main state of the Python module
+				//    We want this to end up in a string that we pass back to Cereal.
+				//    a. We first pickle the python into an in-memory byte stream.
+				//    b. We then convert that to a Base64 std::string that is returned.
+        //
+        //  Basicly, we are executing the following Python code:
+        //    import io
+        //    import base64
+        //    import pickle
+        //    f = io.BytesIO()
+        //    pickle.dump(node, f, 3)
+        //    b = f.getvalue()
+        //    content = str(base64.b64encode(b))
+        //    f.close()
+        
+		    py::tuple args;
+		    auto f = py::module::import("io").attr("BytesIO")();
 
 #if PY_MAJOR_VERSION >= 3
 		    auto pickle = py::module::import("pickle");
+		    args = py::make_tuple(node_, f, 3);   // use type 3 protocol
 #else
 		    auto pickle = py::module::import("cPickle");
-#endif
 		    args = py::make_tuple(node_, f, 2);   // use type 2 protocol
+#endif
 		    pickle.attr("dump")(*args);
-		    pickle.attr("close")();
-
-				// copy the pickle into the out string
-				std::ifstream pfile(tmp_pickle.c_str(), std::ios::binary);
-				std::string content((std::istreambuf_iterator<char>(pfile)),
-				                     std::istreambuf_iterator<char>());
-				pfile.close();
-		 		Path::remove(tmp_pickle);
+
+				// copy the pickle stream into the content as a base64 encoded utf8 string
+        py::bytes b = f.attr("getvalue")();
+        args = py::make_tuple(b);
+		    std::string content = py::str(py::module::import("base64").attr("b64encode")(*args));
+       
+		    f.attr("close")();
 		    return content;
     }
     std::string PyBindRegion::extraSerialize() const
@@ -354,31 +363,34 @@ namespace py = pybind11;
 		void PyBindRegion::pickleDeserialize(std::string p) {
         // 1. deserialize main state using pickle
         // 2. call class method to deserialize external state
-
-				std::ofstream des;
-				std::string tmp_pickle = "pickle.tmp";
-
-
-			  std::ofstream pfile(tmp_pickle.c_str(), std::ios::binary);
-				pfile.write(p.c_str(), p.size());
-				pfile.close();
-
-
-		// Tell Python to un-pickle using what is now in the pickle.tmp file.
-        py::args args = py::make_tuple(tmp_pickle, "rb");
-        auto f = py::module::import("__builtin__").attr("file")(*args);
+        //
+		    // Tell Python to un-pickle using what is in the string p.
+        // but first we need to convert the base64 string into bytes.
+        //
+        // Basically we are executing the following Python code:
+        //   import base64
+        //   import io
+        //   import pickle
+        //   b = base64.b64decode(bytes(p))
+        //   f = io.BytesIO(b)
+        //   node = pickle.load(f)
+        //   f.close()
+        
+        py::args args;
+        args = py::make_tuple(py::bytes(p));
+        py::bytes b = py::module::import("base64").attr("b64decode")(*args);
+        args = py::make_tuple(b);
+		    auto f = py::module::import("io").attr("BytesIO")(*args);
 
 #if PY_MAJOR_VERSION >= 3
         auto pickle = py::module::import("pickle");
 #else
         auto pickle = py::module::import("cPickle");
 #endif
+        args = py::make_tuple(f);
+        node_ = pickle.attr("load")(*args);
 
-        args = py::make_tuple(node_, f);
-        pickle.attr("load")(*args);
-
-        pickle.attr("close")();
-				Path::remove(tmp_pickle);
+        f.attr("close")();
 		}
 
 		void PyBindRegion::extraDeserialize(std::string e) {

bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp

@@ -103,6 +103,10 @@ namespace htm
         {
             throw Exception(__FILE__, __LINE__, e.what());
         }
+        catch (const py::cast_error& e)
+        {
+            throw Exception(__FILE__, __LINE__, e.what());
+        }
         catch (htm::Exception & e)
         {
             throw htm::Exception(e);
@@ -127,6 +131,10 @@ namespace htm
         {
             throw Exception(__FILE__, __LINE__, e.what());
         }
+        catch (const py::cast_error& e)
+        {
+            throw Exception(__FILE__, __LINE__, e.what());
+        }
         catch (htm::Exception & e)
         {
             throw htm::Exception(e);

bindings/py/tests/regions/__init__.py

@@ -0,0 +1,16 @@
+# ----------------------------------------------------------------------
+# HTM Community Edition of NuPIC
+# Copyright (C) 2015, Numenta, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Affero Public License for more details.
+#
+# You should have received a copy of the GNU Affero Public License
+# along with this program.  If not, see http://www.gnu.org/licenses.
+# ----------------------------------------------------------------------

bindings/py/tests/regions/network_test.py

@@ -19,7 +19,11 @@
 import unittest
 import pytest
 import numpy as np
+import sys
+import os
+import pickle
 
+    
 from htm.bindings.regions.PyRegion import PyRegion
 from htm.bindings.sdr import SDR
 import htm.bindings.engine_internal as engine
@@ -96,11 +100,12 @@ def setUp(self):
     engine.Network.cleanup()
     engine.Network.registerPyRegion(LinkRegion.__module__, LinkRegion.__name__)
 
-  @pytest.mark.skip(reason="pickle support needs work...another PR")
   def testSerializationWithPyRegion(self):
     """Test  (de)serialization of network containing a python region"""
     engine.Network.registerPyRegion(__name__,
                                     SerializationTestPyRegion.__name__)
+                                    
+    file_path = "SerializationTest.stream"
     try:
       srcNet = engine.Network()
       srcNet.addRegion(SerializationTestPyRegion.__name__,
@@ -111,12 +116,20 @@ def testSerializationWithPyRegion(self):
                        }))
 
       # Serialize
-      srcNet.saveToFile("SerializationTest.stream")
+      # Note: This will do the following:
+      #    - Call network.saveToFile(), in C++. this opens the file.
+      #    - that calls network.save(stream) 
+      #    - that will use Cereal to serialize the Network object.
+      #    - that will serialize the Region object.
+      #    - that will serialize PyBindRegion object because this is a python Region.
+      #    - that will use pickle to serialize SerializationTestPyRegion in
+      #                serialization_test_py_region.py into Base64.
+      srcNet.saveToFile(file_path, engine.SerializableFormat.BINARY)
 
 
       # Deserialize
       destNet = engine.Network()
-      destNet.loadFromFile("SerializationTest.stream")
+      destNet.loadFromFile(file_path)
 
       destRegion = destNet.getRegion(SerializationTestPyRegion.__name__)
 
@@ -125,6 +138,8 @@ def testSerializationWithPyRegion(self):
 
     finally:
       engine.Network.unregisterPyRegion(SerializationTestPyRegion.__name__)
+      if os.path.isfile(file_path):
+        os.unlink("SerializationTest.stream")
 
 
   def testSimpleTwoRegionNetworkIntrospection(self):
@@ -174,7 +189,6 @@ def testNetworkLinkTypeValidation(self):
     network.link("from", "to", "", "", "UInt32", "Real32")
 
 
-  @pytest.mark.skip(reason="parameter types don't match.")
   def testParameters(self):
 
     n = engine.Network()
@@ -331,4 +345,35 @@ def testExecuteCommand2(self):
     result = r.executeCommand("HelloWorld", 42, lst)
     self.assertTrue(result == "Hello World says: arg1=42 arg2=['list arg', 86]")
 
+  def testNetworkPickle(self):
+    """
+    Test region pickling/unpickling.
+    """
+    network = engine.Network()
+    r_from = network.addRegion("from", "py.LinkRegion", "")
+    r_to = network.addRegion("to", "py.LinkRegion", "")
+    cnt = r_from.getOutputElementCount("UInt32")
+    self.assertEqual(5, cnt)
+
+    network.link("from", "to", "", "", "UInt32", "UInt32")
+    network.link("from", "to", "", "", "Real32", "Real32")
+    network.link("from", "to", "", "", "Real32", "UInt32")
+    network.link("from", "to", "", "", "UInt32", "Real32")
+    network.initialize()
+    
+    if sys.version_info[0] >= 3:
+      proto = 3
+    else:
+      proto = 2
+
+    # Simple test: make sure that dumping / loading works...
+    pickledNetwork = pickle.dumps(network, proto)
+    network2 = pickle.loads(pickledNetwork)
+    
+    s1 = network.getRegion("to").executeCommand("HelloWorld", "26", "64");
+    s2 = network2.getRegion("to").executeCommand("HelloWorld", "26", "64");
+
+    self.assertEqual(s1,"Hello World says: arg1=26 arg2=64")
+    self.assertEqual(s1, s2,  "Simple Network pickle/unpickle failed.")
+
 

bindings/py/tests/regions/pyregion_test.py

@@ -16,6 +16,7 @@
 # ----------------------------------------------------------------------
 
 import unittest
+import sys
 
 from htm.bindings.regions.PyRegion import PyRegion
 
@@ -104,6 +105,25 @@ def testCallUnimplementedMethod(self):
     self.assertEqual(str(cw.exception),
                      "The method setParameter is not implemented.")
 
+  def testPickle(self):
+    """
+    Test region pickling/unpickling.
+    """
+    y = Y()
+
+    if sys.version_info[0] >= 3:
+      import pickle
+      proto = 3
+    else:
+      import cpickle as pickle
+      proto = 2
+
+    # Simple test: make sure that dumping / loading works...
+    pickledRegion = pickle.dumps(y, proto)
+    y2 = pickle.loads(pickledRegion)
+    self.assertEqual(y.zzz, y2.zzz,  "Simple Region pickle/unpickle failed.")
+    
+
 
 
 if __name__ == "__main__":

src/htm/engine/Network.cpp

@@ -44,6 +44,16 @@ Network::Network() {
   commonInit();
 }
 
+// move constructor
+Network::Network(Network && n) {
+  regions_ = std::move(n.regions_);
+  minEnabledPhase_ = n.minEnabledPhase_;
+  maxEnabledPhase_ = n.maxEnabledPhase_;
+  phaseInfo_ = std::move(n.phaseInfo_);
+  callbacks_ = n.callbacks_;
+  iteration_ = n.iteration_;
+}  
+
 Network::Network(const std::string& filename) {
   commonInit();
   loadFromFile(filename);

src/htm/engine/Network.hpp

@@ -60,30 +60,20 @@ class Link;
    *
    * Create an new Network
    *
-   * @Note if the Network object gets copied it does not do a
-   *       deep copy.  So both copies point to the same set of
-   *       regions and links.  The last Network object to go 
-   *       out-of-scope will delete the regions and links.
    */
   Network();
   Network(const std::string& filename);
 
-  /*
-   * @Note: the pickle functions in the python bindings
-   *        require that the Network object be copyable.
-   *        The default copy constructor is ok.
+  /**
+   * Cannot copy or assign a Network object. But can be moved.
    */
+  Network(Network&&);  // move is allowed
+  Network(const Network&) = delete;
+  void operator=(const Network&) = delete;
 
   /**
    * Destructor.
    *
-   * Destruct the network and unregister it from NuPIC:
-   *
-   * - Uninitialize all regions
-   * - Remove all links
-   * - Delete the regions themselves
-   *
-   * @todo Should we document the tear down steps above?
    */
   ~Network();