model: Switch directory config parameter to pathlib.Path

Signed-off-by: John Andersen <[email protected]>

Author: pdxjohnny

CHANGELOG.md

@@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Test tensorflow DNNClassifier documentation exaples in CI
 - config directories and files associated with ConfigLoaders have been renamed
   to configloader.
+- Model config directory parameters are now `pathlib.Path` objects
 
 ## [0.3.4] - 2020-02-28
 ### Added

dffml/model/model.py

@@ -79,6 +79,14 @@ class Model(BaseDataFlowFacilitatorObject):
 
     CONFIG = ModelConfig
 
+    def __init__(self, config):
+        super().__init__(config)
+        # TODO Just in case its a string. We should make it so that on
+        # instantiation of an @config we convert properties to their correct
+        # types.
+        if isinstance(getattr(self.config, "directory", None), str):
+            self.config.directory = pathlib.Path(self.config.directory)
+
     def __call__(self) -> ModelContext:
         self._make_config_directory()
         return self.CONTEXT(self)

docs/plugins/dffml_model.rst

@@ -127,9 +127,9 @@ Example usage of Tensorflow DNNClassifier model using python API
   - default: 30
   - Number of iterations to pass over all records in a source
 
-- directory: String
+- directory: Path
 
-  - default: /home/user/.cache/dffml/tensorflow
+  - default: ~/.cache/dffml/tensorflow
   - Directory where state should be saved
 
 - hidden: List of integers
@@ -248,9 +248,9 @@ predict).
   - default: 30
   - Number of iterations to pass over all records in a source
 
-- directory: String
+- directory: Path
 
-  - default: /home/user/.cache/dffml/tensorflow
+  - default: ~/.cache/dffml/tensorflow
   - Directory where state should be saved
 
 - hidden: List of integers
@@ -426,9 +426,9 @@ Implemented using Tensorflow hub pretrained models.
   - default: 10
   - Number of iterations to pass over all records in a source
 
-- directory: String
+- directory: Path
 
-  - default: /home/user/.cache/dffml/tensorflow_hub
+  - default: ~/.cache/dffml/tensorflow_hub
   - Directory where state should be saved
 
 dffml_model_scratch
@@ -832,8 +832,8 @@ Ensure that `predict` and `accuracy` for these algorithms uses training data.
 
   - Features to train on
 
-- directory: String
+- directory: Path
 
-  - default: /home/user/.cache/dffml/scikit-{Entrypoint}
+  - default: ~/.cache/dffml/scikit-{entrypoint}
   - Directory where state should be saved
 

model/scikit/dffml_model_scikit/__init__.py

@@ -309,9 +309,9 @@
 
   - Features to train on
 
-- directory: String
+- directory: Path
 
-  - default: /home/user/.cache/dffml/scikit-{Entrypoint}
+  - default: ~/.cache/dffml/scikit-{entrypoint}
   - Directory where state should be saved
 
 """

model/scikit/dffml_model_scikit/scikit_base.py

@@ -6,6 +6,7 @@
 import os
 import json
 import hashlib
+import pathlib
 from pathlib import Path
 from typing import AsyncIterator, Tuple, Any, NamedTuple
 
@@ -22,7 +23,7 @@
 
 
 class ScikitConfig(ModelConfig, NamedTuple):
-    directory: str
+    directory: pathlib.Path
     predict: Feature
     features: Features
     tcluster: Feature
@@ -55,14 +56,13 @@ def _feature_predict_hash(self):
             "".join([params] + self.features).encode()
         ).hexdigest()
 
-    def _filename(self):
-        return os.path.join(
-            self.parent.config.directory, self._features_hash + ".joblib"
-        )
+    @property
+    def _filepath(self):
+        return self.parent.config.directory / (self._features_hash + ".joblib")
 
     async def __aenter__(self):
-        if os.path.isfile(self._filename()):
-            self.clf = joblib.load(self._filename())
+        if self._filepath.is_file():
+            self.clf = joblib.load(str(self._filepath))
         else:
             config = self.parent.config._asdict()
             del config["directory"]
@@ -88,10 +88,10 @@ async def train(self, sources: Sources):
         ydata = np.array(df[self.parent.config.predict.NAME])
         self.logger.info("Number of input records: {}".format(len(xdata)))
         self.clf.fit(xdata, ydata)
-        joblib.dump(self.clf, self._filename())
+        joblib.dump(self.clf, str(self._filepath))
 
     async def accuracy(self, sources: Sources) -> Accuracy:
-        if not os.path.isfile(self._filename()):
+        if not self._filepath.is_file():
             raise ModelNotTrained("Train model before assessing for accuracy.")
         data = []
         async for record in sources.with_features(self.features):
@@ -110,7 +110,7 @@ async def accuracy(self, sources: Sources) -> Accuracy:
     async def predict(
         self, records: AsyncIterator[Record]
     ) -> AsyncIterator[Tuple[Record, Any, float]]:
-        if not os.path.isfile(self._filename()):
+        if not self._filepath.is_file():
             raise ModelNotTrained("Train model before prediction.")
         async for record in records:
             feature_data = record.features(self.features)
@@ -132,8 +132,8 @@ async def predict(
 
 class ScikitContextUnsprvised(ScikitContext):
     async def __aenter__(self):
-        if os.path.isfile(self._filename()):
-            self.clf = joblib.load(self._filename())
+        if self._filepath.is_file():
+            self.clf = joblib.load(str(self._filepath))
         else:
             config = self.parent.config._asdict()
             del config["directory"]
@@ -152,10 +152,10 @@ async def train(self, sources: Sources):
         xdata = np.array(df)
         self.logger.info("Number of input records: {}".format(len(xdata)))
         self.clf.fit(xdata)
-        joblib.dump(self.clf, self._filename())
+        joblib.dump(self.clf, str(self._filepath))
 
     async def accuracy(self, sources: Sources) -> Accuracy:
-        if not os.path.isfile(self._filename()):
+        if not self._filepath.is_file():
             raise ModelNotTrained("Train model before assessing for accuracy.")
         data = []
         target = []
@@ -205,7 +205,7 @@ async def accuracy(self, sources: Sources) -> Accuracy:
     async def predict(
         self, records: AsyncIterator[Record]
     ) -> AsyncIterator[Tuple[Record, Any, float]]:
-        if not os.path.isfile(self._filename()):
+        if not self._filepath.is_file():
             raise ModelNotTrained("Train model before prediction.")
         estimator_type = self.clf._estimator_type
         if estimator_type is "clusterer":
@@ -240,28 +240,27 @@ def __init__(self, config) -> None:
         super().__init__(config)
         self.saved = {}
 
-    def _filename(self):
-        return os.path.join(
-            self.config.directory,
+    @property
+    def _filepath(self):
+        return self.config.directory / (
             hashlib.sha384(self.config.predict.NAME.encode()).hexdigest()
-            + ".json",
+            + ".json"
         )
 
     async def __aenter__(self) -> "Scikit":
-        path = Path(self._filename())
-        if path.is_file():
-            self.saved = json.loads(path.read_text())
+        if self._filepath.is_file():
+            self.saved = json.loads(self._filepath.read_text())
         return self
 
     async def __aexit__(self, exc_type, exc_value, traceback):
-        Path(self._filename()).write_text(json.dumps(self.saved))
+        self._filepath.write_text(json.dumps(self.saved))
 
 
 class ScikitUnsprvised(Scikit):
-    def _filename(self):
+    @property
+    def _filepath(self):
         model_name = self.SCIKIT_MODEL.__name__
-        return os.path.join(
-            self.config.directory,
+        return self.config.directory / (
             hashlib.sha384(
                 (
                     "".join(
@@ -272,5 +271,5 @@ def _filename(self):
                     )
                 ).encode()
             ).hexdigest()
-            + ".json",
+            + ".json"
         )

model/scikit/dffml_model_scikit/scikit_models.py

@@ -5,6 +5,7 @@
 """
 import os
 import sys
+import pathlib
 
 from sklearn.neural_network import MLPClassifier
 from sklearn.neighbors import KNeighborsClassifier
@@ -221,14 +222,11 @@ def applicable_features(self, features):
     dffml_config_properties = {
         **{
             "directory": (
-                str,
+                pathlib.Path,
                 field(
                     "Directory where state should be saved",
-                    default=os.path.join(
-                        os.path.expanduser("~"),
-                        ".cache",
-                        "dffml",
-                        f"scikit-{entry_point_name}",
+                    default=pathlib.Path(
+                        "~", ".cache", "dffml", f"scikit-{entry_point_name}",
                     ),
                 ),
             ),

model/tensorflow/dffml_model_tensorflow/dnnc.py

@@ -6,6 +6,7 @@
 import abc
 import hashlib
 import inspect
+import pathlib
 from typing import List, Dict, Any, AsyncIterator, Type
 
 import numpy as np
@@ -145,11 +146,9 @@ class DNNClassifierModelConfig:
     epochs: int = field(
         "Number of iterations to pass over all records in a source", default=30
     )
-    directory: str = field(
+    directory: pathlib.Path = field(
         "Directory where state should be saved",
-        default=os.path.join(
-            os.path.expanduser("~"), ".cache", "dffml", "tensorflow"
-        ),
+        default=pathlib.Path("~", ".cache", "dffml", "tensorflow"),
     )
     hidden: List[int] = field(
         "List length is the number of hidden layers in the network. Each entry in the list is the number of nodes in that hidden layer",

model/tensorflow/dffml_model_tensorflow/dnnr.py

@@ -3,6 +3,7 @@
 record.
 """
 import os
+import pathlib
 from typing import List, Dict, Any, AsyncIterator
 
 import numpy as np
@@ -29,11 +30,9 @@ class DNNRegressionModelConfig:
     epochs: int = field(
         "Number of iterations to pass over all records in a source", default=30
     )
-    directory: str = field(
+    directory: pathlib.Path = field(
         "Directory where state should be saved",
-        default=os.path.join(
-            os.path.expanduser("~"), ".cache", "dffml", "tensorflow"
-        ),
+        default=pathlib.Path("~", ".cache", "dffml", "tensorflow"),
     )
     hidden: List[int] = field(
         "List length is the number of hidden layers in the network. Each entry in the list is the number of nodes in that hidden layer",

model/tensorflow/tests/test_dnnc.py

@@ -1,5 +1,6 @@
 import os
 import random
+import pathlib
 import tempfile
 from typing import Type
 
@@ -84,9 +85,7 @@ async def test_config(self):
         )
         self.assertEqual(
             config.directory,
-            os.path.join(
-                os.path.expanduser("~"), ".cache", "dffml", "tensorflow"
-            ),
+            pathlib.Path("~", ".cache", "dffml", "tensorflow"),
         )
         self.assertEqual(config.steps, 3000)
         self.assertEqual(config.epochs, 30)

model/tensorflow/tests/test_dnnr.py

@@ -1,5 +1,6 @@
 import os
 import random
+import pathlib
 import tempfile
 from typing import Type
 
@@ -95,9 +96,7 @@ async def test_config(self):
         )
         self.assertEqual(
             config.directory,
-            os.path.join(
-                os.path.expanduser("~"), ".cache", "dffml", "tensorflow"
-            ),
+            pathlib.Path("~", ".cache", "dffml", "tensorflow"),
         )
         self.assertEqual(config.steps, 3000)
         self.assertEqual(config.epochs, 30)