Merge branch 'master_community' into cleanup_exceptions

Author: breznak

bindings/py/cpp_src/bindings/engine/py_Engine.cpp

@@ -268,6 +268,21 @@ namespace htm_ext
 
                 r.setInputData(name, s);
             });
+            
+        py_Region.def(py::pickle(
+            [](const Region& self) {
+                std::stringstream ss;
+                  self.save(ss);
+                return py::bytes(ss.str());
+            },
+            // Note: a de-serialized Region will need to be reattached to a Network
+            //           before it could be used.  See Network::addRegion( Region*)
+            [](const py::bytes& s) {
+                std::istringstream ss(s);
+                Region self;
+                self.load(ss);
+                return self;
+        }));
 
 
         py_Region.def("getParameterInt32", &Region::getParameterInt32)
@@ -408,6 +423,11 @@ namespace htm_ext
                     , py::arg("name")
                     , py::arg("nodeType" )
                     , py::arg("nodeParams"));
+        py_Network.def("addRegion", (Region_Ptr_t (htm::Network::*)(
+                    Region_Ptr_t&))
+                    &htm::Network::addRegion,
+                    "add region for deserialization."
+                    , py::arg("region"));
 
         py_Network.def("getRegions", &htm::Network::getRegions)
             .def("getRegion",          &htm::Network::getRegion)
@@ -426,8 +446,21 @@ namespace htm_ext
         py_Network.def("save",      &htm::Network::save)
             .def("load",            &htm::Network::load)
             .def("saveToFile",      &htm::Network::saveToFile, py::arg("file"), py::arg("fmt") = SerializableFormat::BINARY)
-            .def("loadFromFile",    &htm::Network::loadFromFile, py::arg("file"), py::arg("fmt") = SerializableFormat::BINARY);
+            .def("loadFromFile",    &htm::Network::loadFromFile, py::arg("file"), py::arg("fmt") = SerializableFormat::BINARY)
+            .def("__eq__",          &htm::Network::operator==);
 
+        py_Network.def(py::pickle(
+            [](const Network& self) {
+                std::stringstream ss;
+                self.save(ss);
+                return py::bytes(ss.str());
+            },
+            [](const py::bytes& s) {
+                std::istringstream ss(s);
+                Network self;
+                self.load(ss);
+                return self;  
+        }));
 
         py_Network.def("link", &htm::Network::link
             , "Defines a link between regions"

bindings/py/cpp_src/plugin/PyBindRegion.cpp

@@ -291,6 +291,8 @@ namespace py = pybind11;
         , className_(className)
 
     {
+        // Make a local copy of the Spec
+        createSpec(module_.c_str(), nodeSpec_, className_.c_str());
 
         cereal_adapter_load(wrapper);
     }
@@ -304,30 +306,39 @@ namespace py = pybind11;
         // 1. serialize main state using pickle
         // 2. call class method to serialize external state
 
-        // 1. Serialize main state of the Python module
-				//    We want this to end up in the open stream obtained from bundle.
-				//    a. We first pickle the python into a temporary file.
-				//    b. copy the file into our open stream.
-
-				std::string tmp_pickle = "pickle.tmp";
-		    py::tuple args = py::make_tuple(tmp_pickle, "wb");
-		    auto f = py::module::import("__builtin__").attr("file")(*args);
+        //    Serialize main state of the Python module
+				//    We want this to end up in a string that we pass back to Cereal.
+				//    a. We first pickle the python into an in-memory byte stream.
+				//    b. We then convert that to a Base64 std::string that is returned.
+        //
+        //  Basicly, we are executing the following Python code:
+        //    import io
+        //    import base64
+        //    import pickle
+        //    f = io.BytesIO()
+        //    pickle.dump(node, f, 3)
+        //    b = f.getvalue()
+        //    content = str(base64.b64encode(b))
+        //    f.close()
+        
+		    py::tuple args;
+		    auto f = py::module::import("io").attr("BytesIO")();
 
 #if PY_MAJOR_VERSION >= 3
 		    auto pickle = py::module::import("pickle");
+		    args = py::make_tuple(node_, f, 3);   // use type 3 protocol
 #else
 		    auto pickle = py::module::import("cPickle");
-#endif
 		    args = py::make_tuple(node_, f, 2);   // use type 2 protocol
+#endif
 		    pickle.attr("dump")(*args);
-		    pickle.attr("close")();
-
-				// copy the pickle into the out string
-				std::ifstream pfile(tmp_pickle.c_str(), std::ios::binary);
-				std::string content((std::istreambuf_iterator<char>(pfile)),
-				                     std::istreambuf_iterator<char>());
-				pfile.close();
-		 		Path::remove(tmp_pickle);
+
+				// copy the pickle stream into the content as a base64 encoded utf8 string
+        py::bytes b = f.attr("getvalue")();
+        args = py::make_tuple(b);
+		    std::string content = py::str(py::module::import("base64").attr("b64encode")(*args));
+       
+		    f.attr("close")();
 		    return content;
     }
     std::string PyBindRegion::extraSerialize() const
@@ -354,31 +365,34 @@ namespace py = pybind11;
 		void PyBindRegion::pickleDeserialize(std::string p) {
         // 1. deserialize main state using pickle
         // 2. call class method to deserialize external state
-
-				std::ofstream des;
-				std::string tmp_pickle = "pickle.tmp";
-
-
-			  std::ofstream pfile(tmp_pickle.c_str(), std::ios::binary);
-				pfile.write(p.c_str(), p.size());
-				pfile.close();
-
-
-		// Tell Python to un-pickle using what is now in the pickle.tmp file.
-        py::args args = py::make_tuple(tmp_pickle, "rb");
-        auto f = py::module::import("__builtin__").attr("file")(*args);
+        //
+		    // Tell Python to un-pickle using what is in the string p.
+        // but first we need to convert the base64 string into bytes.
+        //
+        // Basically we are executing the following Python code:
+        //   import base64
+        //   import io
+        //   import pickle
+        //   b = base64.b64decode(bytes(p))
+        //   f = io.BytesIO(b)
+        //   node = pickle.load(f)
+        //   f.close()
+        
+        py::args args;
+        args = py::make_tuple(py::bytes(p));
+        py::bytes b = py::module::import("base64").attr("b64decode")(*args);
+        args = py::make_tuple(b);
+		    auto f = py::module::import("io").attr("BytesIO")(*args);
 
 #if PY_MAJOR_VERSION >= 3
         auto pickle = py::module::import("pickle");
 #else
         auto pickle = py::module::import("cPickle");
 #endif
+        args = py::make_tuple(f);
+        node_ = pickle.attr("load")(*args);
 
-        args = py::make_tuple(node_, f);
-        pickle.attr("load")(*args);
-
-        pickle.attr("close")();
-				Path::remove(tmp_pickle);
+        f.attr("close")();
 		}
 
 		void PyBindRegion::extraDeserialize(std::string e) {

bindings/py/cpp_src/plugin/PyBindRegion.hpp

@@ -131,8 +131,8 @@ namespace htm
 
 
     private:
-        std::string module_;
-        std::string className_;
+        std::string module_;     // Full path to the class.
+        std::string className_;  // Just the name of the class.
 
         pybind11::object node_;
 

bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp

@@ -103,6 +103,10 @@ namespace htm
         {
             throw Exception(__FILE__, __LINE__, e.what());
         }
+        catch (const py::cast_error& e)
+        {
+            throw Exception(__FILE__, __LINE__, e.what());
+        }
         catch (htm::Exception & e)
         {
             throw htm::Exception(e);
@@ -127,6 +131,10 @@ namespace htm
         {
             throw Exception(__FILE__, __LINE__, e.what());
         }
+        catch (const py::cast_error& e)
+        {
+            throw Exception(__FILE__, __LINE__, e.what());
+        }
         catch (htm::Exception & e)
         {
             throw htm::Exception(e);
@@ -160,7 +168,7 @@ namespace htm
 		* when its name is used in a Network::addRegion() call.
 		*
 		* @param className -- the name of the Python class that implements the region.
-		* @param module    -- the module (shared library) in which the class resides.
+		* @param module    -- the module (full path and file) in which the class resides.
 		*/
 		inline static void registerPyRegion(const std::string& module, const std::string& className) {
 		    std::string nodeType = "py." + className;

bindings/py/packaging/src/htm/bindings/regions/PyRegion.py

@@ -24,9 +24,12 @@
 import collections
 
 RealNumpyDType = numpy.float32
-from abc import ABCMeta, abstractmethod
-
-class DictReadOnlyWrapper(collections.Mapping):
+if sys.version > '3':                                                           
+    from abc import ABC,abstractmethod       
+    # see http://www.programmersought.com/article/7351237937/                               
+else:                                                                           
+    from abc import ABCMeta, abstractmethod 
+class DictReadOnlyWrapper(collections.abc.Mapping):
   """
   Provides read-only access to a dict. When dict items are mutable, they can
   still be mutated in-place, but dict items can't be reassigned.
@@ -44,10 +47,10 @@ def __len__(self):
   def __getitem__(self, key):
     return self._d[key]
 
-if sys.version_info[0] >= 3:
+if sys.version > '3':
   # Compile the metaclass at runtime because it's invalid python2 syntax.
   exec("""
-class _PyRegionMeta(object, metaclass=ABCMeta):
+class _PyRegionMeta(object):
   pass
 """)
 else:

bindings/py/tests/regions/__init__.py

@@ -0,0 +1,16 @@
+# ----------------------------------------------------------------------
+# HTM Community Edition of NuPIC
+# Copyright (C) 2015, Numenta, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Affero Public License for more details.
+#
+# You should have received a copy of the GNU Affero Public License
+# along with this program.  If not, see http://www.gnu.org/licenses.
+# ----------------------------------------------------------------------

bindings/py/tests/regions/network_test.py

@@ -19,7 +19,11 @@
 import unittest
 import pytest
 import numpy as np
+import sys
+import os
+import pickle
 
+    
 from htm.bindings.regions.PyRegion import PyRegion
 from htm.bindings.sdr import SDR
 import htm.bindings.engine_internal as engine
@@ -96,11 +100,12 @@ def setUp(self):
     engine.Network.cleanup()
     engine.Network.registerPyRegion(LinkRegion.__module__, LinkRegion.__name__)
 
-  @pytest.mark.skip(reason="pickle support needs work...another PR")
   def testSerializationWithPyRegion(self):
     """Test  (de)serialization of network containing a python region"""
     engine.Network.registerPyRegion(__name__,
                                     SerializationTestPyRegion.__name__)
+                                    
+    file_path = "SerializationTest.stream"
     try:
       srcNet = engine.Network()
       srcNet.addRegion(SerializationTestPyRegion.__name__,
@@ -111,12 +116,20 @@ def testSerializationWithPyRegion(self):
                        }))
 
       # Serialize
-      srcNet.saveToFile("SerializationTest.stream")
+      # Note: This will do the following:
+      #    - Call network.saveToFile(), in C++. this opens the file.
+      #    - that calls network.save(stream) 
+      #    - that will use Cereal to serialize the Network object.
+      #    - that will serialize the Region object.
+      #    - that will serialize PyBindRegion object because this is a python Region.
+      #    - that will use pickle to serialize SerializationTestPyRegion in
+      #                serialization_test_py_region.py into Base64.
+      srcNet.saveToFile(file_path, engine.SerializableFormat.BINARY)
 
 
       # Deserialize
       destNet = engine.Network()
-      destNet.loadFromFile("SerializationTest.stream")
+      destNet.loadFromFile(file_path)
 
       destRegion = destNet.getRegion(SerializationTestPyRegion.__name__)
 
@@ -125,6 +138,8 @@ def testSerializationWithPyRegion(self):
 
     finally:
       engine.Network.unregisterPyRegion(SerializationTestPyRegion.__name__)
+      if os.path.isfile(file_path):
+        os.unlink("SerializationTest.stream")
 
 
   def testSimpleTwoRegionNetworkIntrospection(self):
@@ -174,7 +189,6 @@ def testNetworkLinkTypeValidation(self):
     network.link("from", "to", "", "", "UInt32", "Real32")
 
 
-  @pytest.mark.skip(reason="parameter types don't match.")
   def testParameters(self):
 
     n = engine.Network()
@@ -333,4 +347,35 @@ def testExecuteCommand2(self):
     result = r.executeCommand("HelloWorld", 42, lst)
     self.assertTrue(result == "Hello World says: arg1=42 arg2=['list arg', 86]")
 
+  def testNetworkPickle(self):
+    """
+    Test region pickling/unpickling.
+    """
+    network = engine.Network()
+    r_from = network.addRegion("from", "py.LinkRegion", "")
+    r_to = network.addRegion("to", "py.LinkRegion", "")
+    cnt = r_from.getOutputElementCount("UInt32")
+    self.assertEqual(5, cnt)
+
+    network.link("from", "to", "", "", "UInt32", "UInt32")
+    network.link("from", "to", "", "", "Real32", "Real32")
+    network.link("from", "to", "", "", "Real32", "UInt32")
+    network.link("from", "to", "", "", "UInt32", "Real32")
+    network.initialize()
+    
+    if sys.version_info[0] >= 3:
+      proto = 3
+    else:
+      proto = 2
+
+    # Simple test: make sure that dumping / loading works...
+    pickledNetwork = pickle.dumps(network, proto)
+    network2 = pickle.loads(pickledNetwork)
+    
+    s1 = network.getRegion("to").executeCommand("HelloWorld", "26", "64");
+    s2 = network2.getRegion("to").executeCommand("HelloWorld", "26", "64");
+
+    self.assertEqual(s1,"Hello World says: arg1=26 arg2=64")
+    self.assertEqual(s1, s2,  "Simple Network pickle/unpickle failed.")
+