Neural Network widget (#2553)

* Neural Network widget

* NN documentation

* fix import order: python stdlib imports come first

Author: ajdapretnar

Orange/tests/test_neural_network.py

@@ -5,7 +5,7 @@
 
 from Orange.data import Table
 from Orange.classification import NNClassificationLearner
-from Orange.modelling import NNLearner
+from Orange.modelling import NNLearner, ConstantLearner
 from Orange.regression import NNRegressionLearner
 from Orange.evaluation import CA, CrossValidation, MSE
 
@@ -15,22 +15,30 @@ class TestNNLearner(unittest.TestCase):
     def setUpClass(cls):
         cls.iris = Table('iris')
         cls.housing = Table('housing')
+        cls.learner = NNLearner()
 
     def test_NN_classification(self):
         results = CrossValidation(self.iris, [NNClassificationLearner()], k=3)
-        self.assertGreater(CA(results), 0.90)
+        ca = CA(results)
+        self.assertGreater(ca, 0.8)
+        self.assertLess(ca, 0.99)
 
     def test_NN_regression(self):
-        results = CrossValidation(self.housing, [NNRegressionLearner()], k=3)
-        scorer = MSE()
-        self.assertLess(scorer(results)[0], 35)
+        const = ConstantLearner()
+        results = CrossValidation(self.housing, [NNRegressionLearner(), const],
+                                  k=3)
+        mse = MSE()
+        res = mse(results)
+        self.assertLess(res[0], 35)
+        self.assertLess(res[0], res[1])
 
     def test_NN_model(self):
-        results = CrossValidation(self.iris, [NNLearner()], k=3)
+        results = CrossValidation(self.iris, [self.learner], k=3)
         self.assertGreater(CA(results), 0.90)
-        results = CrossValidation(self.housing, [NNLearner()], k=3)
-        scorer = MSE()
-        self.assertLess(scorer(results)[0], 35)
+        results = CrossValidation(self.housing, [self.learner], k=3)
+        mse = MSE()
+        res = mse(results)
+        self.assertLess(res[0], 35)
 
     def test_NN_classification_predict_single_instance(self):
         lrn = NNClassificationLearner()

Orange/widgets/model/icons/NN.svg

@@ -0,0 +1,92 @@
+import re
+import sys
+
+from AnyQt.QtWidgets import QApplication
+from AnyQt.QtCore import Qt
+
+from Orange.data import Table
+from Orange.modelling import NNLearner
+from Orange.widgets import gui
+from Orange.widgets.settings import Setting
+from Orange.widgets.utils.owlearnerwidget import OWBaseLearner
+
+
+class OWNNLearner(OWBaseLearner):
+    name = "Neural Network"
+    description = "A multi-layer perceptron (MLP) algorithm with " \
+                  "backpropagation."
+    icon = "icons/NN.svg"
+    priority = 90
+
+    LEARNER = NNLearner
+
+    activation = ["identity", "logistic", "tanh", "relu"]
+    act_lbl = ["Identity", "Logistic", "tanh", "ReLu"]
+    solver = ["lbfgs", "sgd", "adam"]
+    solv_lbl = ["L-BFGS-B", "SGD", "Adam"]
+
+    learner_name = Setting("Neural Network")
+    hidden_layers_input = Setting("100,")
+    activation_index = Setting(3)
+    solver_index = Setting(2)
+    alpha = Setting(0.0001)
+    max_iterations = Setting(200)
+
+    def add_main_layout(self):
+        box = gui.vBox(self.controlArea, "Network")
+        self.hidden_layers_edit = gui.lineEdit(
+            box, self, "hidden_layers_input", label="Neurons per hidden layer:",
+            orientation=Qt.Horizontal, callback=self.settings_changed,
+            tooltip="A list of integers defining neurons. Length of list "
+                    "defines the number of layers. E.g. 4, 2, 2, 3.",
+            placeholderText="e.g. 100,")
+        self.activation_combo = gui.comboBox(
+            box, self, "activation_index", orientation=Qt.Horizontal,
+            label="Activation:", items=[i for i in self.act_lbl],
+            callback=self.settings_changed)
+        self.solver_combo = gui.comboBox(
+            box, self, "solver_index", orientation=Qt.Horizontal,
+            label="Solver:", items=[i for i in self.solv_lbl],
+            callback=self.settings_changed)
+        self.alpha_spin = gui.doubleSpin(
+            box, self, "alpha", 1e-5, 1.0, 1e-2,
+            label="Alpha:", decimals=5, alignment=Qt.AlignRight,
+            callback=self.settings_changed, controlWidth=80)
+        self.max_iter_spin = gui.spin(
+            box, self, "max_iterations", 10, 300, step=10,
+            label="Max iterations:", orientation=Qt.Horizontal,
+            alignment=Qt.AlignRight, callback=self.settings_changed,
+            controlWidth=80)
+
+    def create_learner(self):
+        return self.LEARNER(
+            hidden_layer_sizes=self.get_hidden_layers(),
+            activation=self.activation[self.activation_index],
+            solver=self.solver[self.solver_index],
+            alpha=self.alpha,
+            max_iter=self.max_iterations,
+            preprocessors=self.preprocessors)
+
+    def get_learner_parameters(self):
+        return (("Hidden layers", ', '.join(map(str, self.get_hidden_layers()))),
+                ("Activation", self.act_lbl[self.activation_index]),
+                ("Solver", self.solv_lbl[self.solver_index]),
+                ("Alpha", self.alpha),
+                ("Max iterations", self.max_iterations))
+
+    def get_hidden_layers(self):
+        layers = tuple(map(int, re.findall(r'\d+', self.hidden_layers_input)))
+        if not layers:
+            layers = (100,)
+            self.hidden_layers_edit.setText("100,")
+        return layers
+
+
+if __name__ == "__main__":
+    a = QApplication(sys.argv)
+    ow = OWNNLearner()
+    d = Table(sys.argv[1] if len(sys.argv) > 1 else 'iris')
+    ow.set_data(d)
+    ow.show()
+    a.exec_()
+    ow.saveSettings()

Orange/widgets/model/owneuralnetwork.py

@@ -0,0 +1,9 @@
+from Orange.widgets.model.owneuralnetwork import OWNNLearner
+from Orange.widgets.tests.base import WidgetTest, WidgetLearnerTestMixin
+
+
+class TestOWNeuralNetwork(WidgetTest, WidgetLearnerTestMixin):
+    def setUp(self):
+        self.widget = self.create_widget(OWNNLearner,
+                                         stored_settings={"auto_apply": False})
+        self.init()

Orange/widgets/model/tests/test_owneuralnetwork.py

@@ -80,19 +80,20 @@ Model
 .. toctree::
    :maxdepth: 1
 
-   widgets/model/naivebayes
-   widgets/model/logisticregression
-   widgets/model/tree
-   widgets/model/knn
-   widgets/model/loadmodel
    widgets/model/constant
+   widgets/model/cn2ruleinduction
+   widgets/model/knn
+   widgets/model/tree
    widgets/model/randomforest
-   widgets/model/savemodel
    widgets/model/svm
-   widgets/model/cn2ruleinduction
+   widgets/model/linearregression
+   widgets/model/logisticregression
+   widgets/model/naivebayes
    widgets/model/adaboost
+   widgets/model/neuralnetwork
    widgets/model/stochasticgradient
-   widgets/model/linearregression
+   widgets/model/loadmodel
+   widgets/model/savemodel
 
 
 Unsupervised

doc/visual-programming/source/index.rst

@@ -0,0 +1,69 @@
+Neural Network
+==============
+
+.. figure:: icons/nn.png
+
+A multi-layer perceptron (MLP) algorithm with backpropagation.
+
+Signals
+-------
+
+**Inputs**:
+
+-  **Data**
+
+   A data set
+
+-  **Preprocessor**
+
+   Preprocessing method(s)
+
+**Outputs**:
+
+-  **Learner**
+
+   A MLP learning algorithm with settings as specified in the dialog.
+
+-  **Model**
+
+   A trained model. Output signal sent only if input *Data* is present.
+
+Description
+-----------
+
+The **Neural Network** widget uses sklearn's `Multi\-layer Perceptron algorithm <http://scikit-learn.org/stable/modules/neural_networks_supervised.html>`_ that can learn non-linear models as well as linear.
+
+.. figure:: images/NeuralNetwork-stamped.png
+
+1. A name under which it will appear in other widgets. The default name is
+   "Neural Network".
+2. Set model parameters:
+   -  Neurons per hidden layer: defined as the ith element represents the number of neurons in the ith hidden layer. E.g. a neural network with 3 layers can be defined as 2, 3, 2.
+   -  Activation function for the hidden layer:
+      -  Identity: no-op activation, useful to implement linear bottleneck
+      -  Logistic: the logistic sigmoid function
+      -  tanh: the hyperbolic tan function
+      -  ReLu: the rectified linear unit function
+   -  Solver for weight optimization:
+      -  L-BFGS-B: an optimizer in the family of quasi-Newton methods
+      -  SGD: stochastic gradient descent
+      -  Adam: stochastic gradient-based optimizer
+   -  Alpha: L2 penalty (regularization term) parameter
+   -  Max iterations: maximum number of iterations
+
+   Other parameters are set to `sklearn's defaults <http://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html>`_.
+
+3. Produce a report. 
+4. When the box is ticked (*Apply Automatically*), the widget will
+   communicate changes automatically. Alternatively, click *Apply*. 
+
+Examples
+--------
+
+The first example is a classification task on *iris* data set. We compare the results of **Neural Network** with the :doc:`Logistic Regression <../model/logisticregression>`.
+
+.. figure:: images/NN-Example-Test.png
+
+The second example is a prediction task, still using the *iris* data. This workflow shows how to use the *Learner* output. We input the **Neural Network** prediction model into :doc:`Predictions <../evaluation/predictions>` and observe the predicted values.
+
+.. figure:: images/NN-Example-Predict.png