docs: model: tensorflow: Python API examples

Fixes: #433

Author: sk-ip

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Docstrings and doctestable examples to `record.py`.
 - Inputs can be validated using operations
   - `validate` parameter in `Input` takes `Operation.instance_name`
+- Test tensorflow DNNEstimator documentation exaples in CI
+- Add python code for tensorflow DNNEstimator
 ### Fixed
 - New model tutorial mentions file paths that should be edited.
 

docs/plugins/dffml_model.rst

@@ -154,58 +154,32 @@ Generating train and test data
   make sure to take a BACKUP of files with same name in the directory
   from where this command is run as it overwrites any existing files.
 
-.. code-block:: console
+.. literalinclude:: /../model/tensorflow/examples/tfdnnr/train_data.sh
+
+.. literalinclude:: /../model/tensorflow/examples/tfdnnr/test_data.sh
+
+Train the model
+
+.. literalinclude:: /../model/tensorflow/examples/tfdnnr/train.sh
+
+Assess the accuracy
+
+.. literalinclude:: /../model/tensorflow/examples/tfdnnr/accuracy.sh
+
+Output
+
+.. code-block::
 
-    $ cat > train.csv << EOF
-    Feature1,Feature2,TARGET
-    0.93,0.68,3.89
-    0.24,0.42,1.75
-    0.36,0.68,2.75
-    0.53,0.31,2.00
-    0.29,0.25,1.32
-    0.29,0.52,2.14
-    EOF
-    $ cat > test.csv << EOF
-    Feature1,Feature2,TARGET
-    0.57,0.84,3.65
-    0.95,0.19,2.46
-    0.23,0.15,0.93
-    EOF
-    $ dffml train \
-        -model tfdnnr \
-        -model-epochs 300 \
-        -model-steps 2000 \
-        -model-predict TARGET:float:1 \
-        -model-hidden 8 16 8 \
-        -sources s=csv \
-        -source-filename train.csv \
-        -model-features \
-          Feature1:float:1 \
-          Feature2:float:1 \
-        -log debug
-    Enabling debug log shows tensorflow losses...
-    $ dffml accuracy \
-        -model tfdnnr \
-        -model-predict TARGET:float:1 \
-        -model-hidden 8 16 8 \
-        -sources s=csv \
-        -source-filename test.csv \
-        -model-features \
-          Feature1:float:1 \
-          Feature2:float:1 \
-        -log critical
     0.9468210011
-    $ echo -e 'Feature1,Feature2,TARGET\n0.21,0.18,0.84\n' | \
-      dffml predict all \
-        -model tfdnnr \
-        -model-predict TARGET:float:1 \
-        -model-hidden 8 16 8 \
-        -sources s=csv \
-        -source-filename /dev/stdin \
-        -model-features \
-          Feature1:float:1 \
-          Feature2:float:1 \
-        -log critical
+
+Make a prediction
+
+.. literalinclude:: /../model/tensorflow/examples/tfdnnr/predict.sh
+
+Output
+
+.. code-block:: json
+
     [
         {
             "extra": {},
@@ -225,6 +199,10 @@ Generating train and test data
         }
     ]
 
+Example usage of Tensorflow DNNEstimator model using python API
+
+.. literalinclude:: /../model/tensorflow/examples/tfdnnr/tfdnnr.py
+
 The ``NaN`` in ``confidence`` is the expected behaviour. (See TODO in
 predict).
 

model/tensorflow/dffml_model_tensorflow/dnnr.py

@@ -195,58 +195,32 @@ class DNNRegressionModel(Model):
       make sure to take a BACKUP of files with same name in the directory
       from where this command is run as it overwrites any existing files.
 
-    .. code-block:: console
-
-        $ cat > train.csv << EOF
-        Feature1,Feature2,TARGET
-        0.93,0.68,3.89
-        0.24,0.42,1.75
-        0.36,0.68,2.75
-        0.53,0.31,2.00
-        0.29,0.25,1.32
-        0.29,0.52,2.14
-        EOF
-        $ cat > test.csv << EOF
-        Feature1,Feature2,TARGET
-        0.57,0.84,3.65
-        0.95,0.19,2.46
-        0.23,0.15,0.93
-        EOF
-        $ dffml train \\
-            -model tfdnnr \\
-            -model-epochs 300 \\
-            -model-steps 2000 \\
-            -model-predict TARGET:float:1 \\
-            -model-hidden 8 16 8 \\
-            -sources s=csv \\
-            -source-filename train.csv \\
-            -model-features \\
-              Feature1:float:1 \\
-              Feature2:float:1 \\
-            -log debug
-        Enabling debug log shows tensorflow losses...
-        $ dffml accuracy \\
-            -model tfdnnr \\
-            -model-predict TARGET:float:1 \\
-            -model-hidden 8 16 8 \\
-            -sources s=csv \\
-            -source-filename test.csv \\
-            -model-features \\
-              Feature1:float:1 \\
-              Feature2:float:1 \\
-            -log critical
+    .. literalinclude:: /../model/tensorflow/examples/tfdnnr/train_data.sh
+
+    .. literalinclude:: /../model/tensorflow/examples/tfdnnr/test_data.sh
+
+    Train the model
+
+    .. literalinclude:: /../model/tensorflow/examples/tfdnnr/train.sh
+
+    Assess the accuracy
+
+    .. literalinclude:: /../model/tensorflow/examples/tfdnnr/accuracy.sh
+
+    Output
+
+    .. code-block::
+
         0.9468210011
-        $ echo -e 'Feature1,Feature2,TARGET\\n0.21,0.18,0.84\\n' | \\
-          dffml predict all \\
-            -model tfdnnr \\
-            -model-predict TARGET:float:1 \\
-            -model-hidden 8 16 8 \\
-            -sources s=csv \\
-            -source-filename /dev/stdin \\
-            -model-features \\
-              Feature1:float:1 \\
-              Feature2:float:1 \\
-            -log critical
+
+    Make a prediction
+
+    .. literalinclude:: /../model/tensorflow/examples/tfdnnr/predict.sh
+
+    Output
+
+    .. code-block:: json
+
         [
             {
                 "extra": {},
@@ -266,6 +240,10 @@ class DNNRegressionModel(Model):
             }
         ]
 
+    Example usage of Tensorflow DNNEstimator model using python API
+
+    .. literalinclude:: /../model/tensorflow/examples/tfdnnr/tfdnnr.py
+
     The ``NaN`` in ``confidence`` is the expected behaviour. (See TODO in
     predict).
 

model/tensorflow/examples/tfdnnr/__init__.py

@@ -0,0 +1,10 @@
+dffml accuracy \
+  -model tfdnnr \
+  -model-predict TARGET:float:1 \
+  -model-hidden 8 16 8 \
+  -sources s=csv \
+  -source-filename test.csv \
+  -model-features \
+    Feature1:float:1 \
+    Feature2:float:1 \
+  -log critical

model/tensorflow/examples/tfdnnr/accuracy.sh

@@ -0,0 +1,11 @@
+echo -e 'Feature1,Feature2,TARGET\n0.21,0.18,0.84\n' | \
+  dffml predict all \
+  -model tfdnnr \
+  -model-predict TARGET:float:1 \
+  -model-hidden 8 16 8 \
+  -sources s=csv \
+  -source-filename /dev/stdin \
+  -model-features \
+    Feature1:float:1 \
+    Feature2:float:1 \
+  -log critical

model/tensorflow/examples/tfdnnr/predict.sh

@@ -0,0 +1,6 @@
+cat > test.csv << EOF
+Feature1,Feature2,TARGET
+0.57,0.84,3.65
+0.95,0.19,2.46
+0.23,0.15,0.93
+EOF

model/tensorflow/examples/tfdnnr/test_data.sh

@@ -0,0 +1,59 @@
+import os
+import ast
+import sys
+import json
+import tempfile
+import contextlib
+import subprocess
+import unittest.mock
+
+from dffml.util.os import chdir
+
+
+def sh_filepath(filename):
+    return os.path.join(os.path.dirname(__file__), filename)
+
+
+@contextlib.contextmanager
+def directory_with_csv_files():
+    with tempfile.TemporaryDirectory() as tempdir:
+        with chdir(tempdir):
+            subprocess.check_output(["bash", sh_filepath("train_data.sh")])
+            subprocess.check_output(["bash", sh_filepath("test_data.sh")])
+            yield tempdir
+
+
+class TestExample(unittest.TestCase):
+    def python_test(self, filename):
+        # Path to target file
+        filepath = os.path.join(os.path.dirname(__file__), filename)
+        # Capture output
+        stdout = subprocess.check_output([sys.executable, filepath])
+        lines = stdout.decode().split("\n")
+        # Check the Accuracy
+        self.assertRegex(lines[0], r"Accuracy:  [-+]?\d*\.?\d+|\d+")
+        # Check the TARGET
+        self.assertIsInstance(round(ast.literal_eval(lines[1])["TARGET"]), int)
+
+    def test_python_filenames(self):
+        with directory_with_csv_files() as tempdir:
+            self.python_test("tfdnnr.py")
+
+    def test_shell(self):
+        with directory_with_csv_files() as tempdir:
+            # Run training
+            subprocess.check_output(["bash", sh_filepath("train.sh")])
+            # Check the Accuracy
+            stdout = subprocess.check_output(
+                ["bash", sh_filepath("accuracy.sh")]
+            )
+            self.assertRegex(stdout.decode().strip(), r"[-+]?\d*\.?\d+|\d+")
+            # Make the prediction
+            stdout = subprocess.check_output(
+                ["bash", sh_filepath("predict.sh")]
+            )
+            records = json.loads(stdout.decode())
+            # Check the TARGET
+            self.assertIsInstance(
+                round(records[0]["prediction"]["TARGET"]["value"]), int
+            )

model/tensorflow/examples/tfdnnr/test_tfdnnr.py

@@ -0,0 +1,26 @@
+from dffml import CSVSource, Features, DefFeature
+from dffml.noasync import train, accuracy, predict
+from dffml_model_tensorflow.dnnr import DNNRegressionModel
+
+model = DNNRegressionModel(
+    features=Features(
+        DefFeature("Feature1", float, 1), DefFeature("Feature2", float, 1),
+    ),
+    predict=DefFeature("TARGET", float, 1),
+    epochs=300,
+    steps=2000,
+    hidden=[8, 16, 8],
+)
+
+# Train the model
+train(model, "train.csv")
+
+# Assess accuracy (alternate way of specifying data source)
+print("Accuracy:", accuracy(model, CSVSource(filename="test.csv")))
+
+# Make prediction
+for i, features, prediction in predict(
+    model, {"Feature1": 0.21, "Feature2": 0.18, "TARGET": 0.84},
+):
+    features["TARGET"] = prediction["TARGET"]["value"]
+    print(features)

model/tensorflow/examples/tfdnnr/tfdnnr.py

@@ -0,0 +1,12 @@
+dffml train \
+  -model tfdnnr \
+  -model-epochs 300 \
+  -model-steps 2000 \
+  -model-predict TARGET:float:1 \
+  -model-hidden 8 16 8 \
+  -sources s=csv \
+  -source-filename train.csv \
+  -model-features \
+    Feature1:float:1 \
+    Feature2:float:1 \
+  -log debug