Merge pull request #747 from htm-community/pickled_network2

Continuation of fixes for Pickled networkAPI

Author: breznak

bindings/py/cpp_src/bindings/engine/py_Engine.cpp

@@ -445,7 +445,8 @@ namespace htm_ext
         py_Network.def("save",      &htm::Network::save)
             .def("load",            &htm::Network::load)
             .def("saveToFile",      &htm::Network::saveToFile, py::arg("file"), py::arg("fmt") = SerializableFormat::BINARY)
-            .def("loadFromFile",    &htm::Network::loadFromFile, py::arg("file"), py::arg("fmt") = SerializableFormat::BINARY);
+            .def("loadFromFile",    &htm::Network::loadFromFile, py::arg("file"), py::arg("fmt") = SerializableFormat::BINARY)
+            .def("__eq__",          &htm::Network::operator==);
 
         py_Network.def(py::pickle(
             [](const Network& self) {

bindings/py/cpp_src/plugin/PyBindRegion.cpp

@@ -291,6 +291,8 @@ namespace py = pybind11;
         , className_(className)
 
     {
+        // Make a local copy of the Spec
+        createSpec(module_.c_str(), nodeSpec_, className_.c_str());
 
         cereal_adapter_load(wrapper);
     }

bindings/py/cpp_src/plugin/PyBindRegion.hpp

@@ -131,8 +131,8 @@ namespace htm
 
 
     private:
-        std::string module_;
-        std::string className_;
+        std::string module_;     // Full path to the class.
+        std::string className_;  // Just the name of the class.
 
         pybind11::object node_;
 

bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp

@@ -168,7 +168,7 @@ namespace htm
 		* when its name is used in a Network::addRegion() call.
 		*
 		* @param className -- the name of the Python class that implements the region.
-		* @param module    -- the module (shared library) in which the class resides.
+		* @param module    -- the module (full path and file) in which the class resides.
 		*/
 		inline static void registerPyRegion(const std::string& module, const std::string& className) {
 		    std::string nodeType = "py." + className;

bindings/py/packaging/src/htm/bindings/regions/PyRegion.py

@@ -24,9 +24,12 @@
 import collections
 
 RealNumpyDType = numpy.float32
-from abc import ABCMeta, abstractmethod
-
-class DictReadOnlyWrapper(collections.Mapping):
+if sys.version > '3':                                                           
+    from abc import ABC,abstractmethod       
+    # see http://www.programmersought.com/article/7351237937/                               
+else:                                                                           
+    from abc import ABCMeta, abstractmethod 
+class DictReadOnlyWrapper(collections.abc.Mapping):
   """
   Provides read-only access to a dict. When dict items are mutable, they can
   still be mutated in-place, but dict items can't be reassigned.
@@ -44,10 +47,10 @@ def __len__(self):
   def __getitem__(self, key):
     return self._d[key]
 
-if sys.version_info[0] >= 3:
+if sys.version > '3':
   # Compile the metaclass at runtime because it's invalid python2 syntax.
   exec("""
-class _PyRegionMeta(object, metaclass=ABCMeta):
+class _PyRegionMeta(object):
   pass
 """)
 else:

bindings/py/tests/regions/pyregion_test.py

@@ -50,30 +50,7 @@ def __init__(self):
 class PyRegionTest(unittest.TestCase):
 
 
-  def testNoInit(self):
-    """Test unimplemented init method"""
-    class NoInit(PyRegion):
-      pass
 
-    with self.assertRaises(TypeError) as cw:
-      _ni = NoInit()
-
-    self.assertEqual(str(cw.exception), "Can't instantiate abstract class " +
-      "NoInit with abstract methods __init__, compute, initialize")
-
-
-  def testUnimplementedAbstractMethods(self):
-    """Test unimplemented abstract methods"""
-    # Test unimplemented getSpec (results in NotImplementedError)
-    with self.assertRaises(NotImplementedError):
-      X.getSpec()
-
-    # Test unimplemented abstract methods (x can't be instantiated)
-    with self.assertRaises(TypeError) as cw:
-      _x = X()
-
-    self.assertEqual(str(cw.exception), "Can't instantiate abstract class " +
-      "X with abstract methods compute, initialize")
 
   def testUnimplementedNotImplementedMethods(self):
     """Test unimplemented @not_implemented methods"""

py/tests/advanced/regions/L46aNetwork_test.py

@@ -0,0 +1,125 @@
+'''
+Created on 6 Nov 2019
+
+@author: fred
+'''
+import unittest
+
+import pickle
+import sys
+import numpy as np
+
+from htm.advanced.support.register_regions import registerAllAdvancedRegions
+from htm.bindings.engine_internal import Network
+from htm.advanced.frameworks.location.location_network_creation import createL4L6aLocationColumn
+
+cortical_params = {
+  # Column parameters
+    
+  # L2 Parameters
+  # Adapted from htmresearch.frameworks.layers.l2_l4_inference.L4L2Experiment#getDefaultL2Params
+  # L4 Parameters
+    "l4_cellsPerColumn": 24,
+    "l4_columnCount": 16,
+    "l4_connectedPermanence": 0.6,
+    "l4_permanenceIncrement": 0.1,
+    "l4_permanenceDecrement": 0.02,
+    "l4_apicalPredictedSegmentDecrement": 0.0,
+    "l4_basalPredictedSegmentDecrement": 0.0,
+    "l4_initialPermanence": 1.0,
+    "l4_activationThreshold": 8,
+    "l4_minThreshold": 8,
+    "l4_reducedBasalThreshold": 8,
+    "l4_sampleSize": 10,
+    "l4_implementation": "ApicalTiebreak",
+
+  # L6a Parameters
+    "l6a_moduleCount": 10,
+    "l6a_dimensions": 2,
+    "l6a_connectedPermanence": 0.5,
+    "l6a_permanenceIncrement": 0.1,
+    "l6a_permanenceDecrement": 0.0,
+    "l6a_initialPermanence": 1.0,
+    "l6a_activationThreshold": 8,
+    "l6a_initialPermanence": 1.0,
+    "l6a_learningThreshold": 8,
+    "l6a_sampleSize": 10,
+    "l6a_cellsPerAxis": 10,
+    "l6a_scale": 10,
+    "l6a_orientation": 60,
+    "l6a_bumpOverlapMethod": "probabilistic"
+    }
+
+class TestSimpleSPTMNetwork(unittest.TestCase):
+
+    def _create_network(self, L4Params, L6aParams):
+        """
+        Constructor.
+        """
+        network = Network()
+        
+        # Create network
+        network = createL4L6aLocationColumn(network=network,
+                                            L4Params=L4Params,
+                                            L6aParams=L6aParams,
+                                            inverseReadoutResolution=None,
+                                            baselineCellsPerAxis=6,
+                                            suffix="")
+
+        network.initialize()
+        return network
+        
+    def setUp(self):
+        registerAllAdvancedRegions()
+
+        self._params = cortical_params
+        
+        L4Params = {param.split("_")[1]:value for param, value in self._params.items() if param.startswith("l4")}
+        L6aParams = {param.split("_")[1]:value for param, value in self._params.items() if param.startswith("l6a")}
+        # Configure L6a self._htm_parameters
+        numModules = L6aParams["moduleCount"]
+        L6aParams["scale"] = [L6aParams["scale"]] * numModules
+        angle = L6aParams["orientation"] // numModules
+        orientation = list(range(angle // 2, angle * numModules, angle))
+        L6aParams["orientation"] = np.radians(orientation).tolist()
+                
+        self.network = self._create_network(L4Params, L6aParams)
+        
+    def tearDown(self):
+        self.network = None
+        
+    def _run_network(self, network):
+        """
+        Run the network with fixed data.
+        """
+        motorInput = network.getRegion("motorInput")
+        sensorInput = network.getRegion("sensorInput")
+        motorInput.executeCommand('addDataToQueue', [0,0])
+        sensorInput.executeCommand('addDataToQueue', [1,2,3], False, 0)
+        
+        network.run(1)
+        
+        L4Region = network.getRegion("L4")
+        activeL4Cells = np.array(L4Region.getOutputArray("activeCells")).nonzero()[0]
+        
+        return activeL4Cells
+        
+    def testAL246aCorticalColumnPickle(self):
+        """
+        Test that L246aCorticalColumn can be pickled.
+        """
+        if sys.version_info[0] >= 3:
+            proto = 3
+        else:
+            proto = 2
+        # Simple test: make sure that dumping / loading works...
+        pickledColumn = pickle.dumps(self.network, proto)
+        network2 = pickle.loads(pickledColumn)
+        s1 = self._run_network(self.network)
+        s2 = self._run_network(network2)
+        self.assertTrue(np.array_equal(s1, s2))
+        
+
+if __name__ == "__main__":
+    #import sys;sys.argv = ['', 'Test.testName']
+    unittest.main()

setup.py

@@ -29,6 +29,8 @@
 from setuptools.command.test import test as BaseTestCommand
 # see https://stackoverflow.com/questions/44323474/distutils-core-vs-setuptools-with-c-extension
 from setuptools import Extension
+from pathlib import Path
+from sys import version_info
 
 # NOTE:  To debug the python bindings in a debugger, use the procedure
 #        described here: https://pythonextensionpatterns.readthedocs.io/en/latest/debugging/debug_in_ide.html
@@ -264,6 +266,11 @@ def generateExtensions(platform, build_type):
   and then create the extension libraries in Repository/build/Release/distr/src/nupic/bindings.
   Note: for Windows it will force a X64 build.
   """
+  
+  # Make sure that .py code gets copied during any build so it is included in .whl
+  if version_info >= (3, 4):
+    Path(os.path.join(PY_BINDINGS, 'cpp_src', 'CMakeLists.txt')).touch()
+  
   cwd = os.getcwd()
   scriptsDir = os.path.join(REPO_DIR, "build", "scripts")
   try:
@@ -290,7 +297,6 @@ def configure(platform, build_type):
   cwd = os.getcwd()
 
   print("Python version: {}\n".format(sys.version))
-  from sys import version_info
   if version_info > (3, 0):
     # Build a Python 3.x library
     PY_VER = "-DBINDING_BUILD=Python3"

src/htm/engine/Input.cpp

@@ -55,10 +55,10 @@ void Input::addLink(const std::shared_ptr<Link> link, std::shared_ptr<Output> sr
   // Make sure we don't already have a link to the same output
   for (const auto &it : links_) {
     const Output* o = (*it).getSrc();
-    NTA_CHECK(srcOutput.get() != o) << "addLink -- link from region "
-                << srcOutput->getRegion()->getName() << " output "
-                << srcOutput->getName() << " to region " << region_->getName()
-                << " input " << getName() << " already exists";
+    NTA_CHECK(srcOutput.get() != o) << "Input::addLink() -- link from output="
+                << srcOutput->getRegion()->getName() << "."
+                << srcOutput->getName() << " to input=" << region_->getName()
+                << "." << getName() << " already exists";
   }
 
   links_.push_back(link);

src/htm/engine/Network.cpp

@@ -109,12 +109,11 @@ std::shared_ptr<Region> Network::addRegion(std::shared_ptr<Region>& r) {
   NTA_CHECK(r != nullptr);
   r->network_ = this;
   regions_[r->getName()] = r;
+  
+  // If a region is added, initially set the phase to the default phase.
+  // The phase can be changed later.
+  setDefaultPhase_(r.get());
 
-  // We must make a copy of the phases set here because
-  // setPhases_ will be passing this back down into
-  // the region.
-  std::set<UInt32> phases = r->getPhases();
-  setPhases_(r.get(), phases);
   return r;
 }
 
@@ -155,13 +154,11 @@ void Network::setPhases_(Region *r, std::set<UInt32> &phases) {
     if (item != phaseInfo_[i].end() && !insertPhase) {
       phaseInfo_[i].erase(item);
     } else if (insertPhase) {
+      // add the new phase(s_
       phaseInfo_[i].insert(r);
     }
   }
 
-  // keep track (redundantly) of phases inside the Region also, for
-  // serialization
-  r->setPhases(phases);
 
   resetEnabledPhases_();
 }
@@ -443,12 +440,10 @@ std::shared_ptr<Region> Network::getRegion(const std::string& name) const {
 std::vector<std::shared_ptr<Link>> Network::getLinks() const {
   std::vector<std::shared_ptr<Link>> links;
 
-  for (UInt32 phase = minEnabledPhase_; phase <= maxEnabledPhase_; phase++) {
-    for (auto r : phaseInfo_[phase]) {
-      for (auto &input : r->getInputs()) {
-        for (auto &link : input.second->getLinks()) {
-          links.push_back(link);
-        }
+  for (auto p : regions_) {
+    for (auto &input : p.second->getInputs()) {
+      for (auto &link : input.second->getLinks()) {
+        links.push_back(link);
       }
     }
   }
@@ -521,13 +516,9 @@ void Network::post_load() {
   for(auto p: regions_) {
     std::shared_ptr<Region>& r = p.second;
     r->network_ = this;
-    std::set<UInt32> phases = r->getPhases();
-    setPhases_(r.get(), phases);
     r->evaluateLinks();      // Create the input buffers.
   }
 
-  NTA_CHECK(maxEnabledPhase_ < phaseInfo_.size())
-      << "maxphase: " << maxEnabledPhase_ << " size: " << phaseInfo_.size();
 
   // Note: When serialized, the output buffers are saved
   //       by each Region.  After restore we need to
@@ -564,6 +555,65 @@ void Network::post_load() {
 
 }
 
+std::string Network::phasesToString() const {
+  std::stringstream ss;
+  ss << "{";
+  ss << "minEnabledPhase_: " << minEnabledPhase_ << ", ";
+  ss << "maxEnabledPhase_: " << maxEnabledPhase_ << ", ";
+  ss << "info: [";
+  for (auto phase : phaseInfo_) {
+    ss << "[";
+    for (auto region : phase) {
+      ss << region->getName() << ", ";
+    }
+    ss << "]";
+  }
+  ss << "]}";
+  return ss.str();
+}
+void Network::phasesFromString(const std::string& phaseString) {
+  std::string content = phaseString;
+  content.erase(std::remove(content.begin(), content.end(), ','), content.end());
+  std::stringstream ss(content);
+  std::string tag;
+  std::set<Region *> phase;
+  
+  NTA_CHECK(ss.peek() == '{') << "Invalid phase deserialization";
+  ss.ignore(1);
+  ss >> tag;
+  NTA_CHECK(tag == "minEnabledPhase_:");
+  ss >> minEnabledPhase_;
+  ss >> tag;
+  NTA_CHECK(tag == "maxEnabledPhase_:");
+  ss >> maxEnabledPhase_;
+  ss >> tag;
+  NTA_CHECK(tag == "info:") << "Invalid phase deserialization";
+  ss >> std::ws;
+  NTA_CHECK(ss.peek() == '[') << "Invalid phase deserialization";
+  ss.ignore(1);
+  ss >> std::ws;
+  while (ss.peek() != ']') {
+    ss >> std::ws;
+    if (ss.peek() == '[') {
+      ss.ignore(1);
+      ss >> std::ws;
+      while (ss.peek() != ']') {
+        ss >> tag;
+        auto it = regions_.find(tag);
+        NTA_CHECK(it != regions_.end()) << "Region '" << tag << "' not found while decoding phase.";
+        phase.insert(it->second.get());
+        ss >> std::ws;
+      }
+      ss.ignore(1); // ']'
+      phaseInfo_.push_back(phase);
+      phase.clear();
+    }
+  }
+  ss >> std::ws;
+  ss.ignore(1); // ']'
+}
+
+
 void Network::enableProfiling() {
   for (auto p: regions_) {
     std::shared_ptr<Region> r = p.second;