base: Configurable implements args, config methods

pdxjohnny · pdxjohnny · commit 5b58db27e09c · 2019-11-24T18:35:01.000-08:00
Signed-off-by: John Andersen &lt;johnandersenpdx@gmail.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Features were moved from ModelContext to ModelConfig
 - CI is now run via GitHub Actions
 - CI testing script is now verbose
+- args and config methods of all classes no longer require implementation.
+  BaseConfigurable handles exporting of arguments and creation of config objects
+  for each class based off of the CONFIG property of that class. The CONFIG
+  property is a class which has been decorated with dffml.base.config to make it
+  a dataclass.
 ### Fixed
 - DataFlows with multiple possibilities for a source for an input, now correctly
   look through all possible sources instead of just the first one.
diff --git a/dffml/base.py b/dffml/base.py
@@ -4,16 +4,58 @@
 """
 import abc
 import inspect
+import argparse
+import contextlib
+import dataclasses
 from argparse import ArgumentParser
-from typing import Dict, Any, Tuple, NamedTuple
+from typing import Dict, Any, Tuple, NamedTuple, Type
 
+try:
+    from typing import get_origin, get_args
+except ImportError:
+    # Added in Python 3.8
+    def get_origin(t):
+        return getattr(t, "__origin__", None)
+
+    def get_args(t):
+        return getattr(t, "__args__", None)
+
+
+from .util.cli.arg import Arg
 from .util.data import traverse_config_set, traverse_config_get
 
 from .util.entrypoint import Entrypoint
 
 from .log import LOGGER
 
 
+class ParseExpandAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        if not isinstance(values, list):
+            values = [values]
+        setattr(namespace, self.dest, self.LIST_CLS(*values))
+
+
+# Maps classes to their ParseClassNameAction
+LIST_ACTIONS: Dict[Type, Type] = {}
+
+
+def list_action(list_cls):
+    """
+    Action to take a list of values and make them values in the list of type
+    list_class. Which will be a class descendent from AsyncContextManagerList.
+    """
+    LIST_ACTIONS.setdefault(
+        list_cls,
+        type(
+            f"Parse{list_cls.__qualname__}Action",
+            (ParseExpandAction,),
+            {"LIST_CLS": list_cls},
+        ),
+    )
+    return LIST_ACTIONS[list_cls]
+
+
 class MissingArg(Exception):
     """
     Raised when a BaseConfigurable is missing an argument from the args dict it
@@ -64,6 +106,13 @@ def __str__(self):
         return repr(self)
 
 
+def config(cls):
+    """
+    Decorator to create a dataclass
+    """
+    return dataclasses.dataclass(eq=True, frozen=True)(cls)
+
+
 class ConfigurableParsingNamespace(object):
     def __init__(self):
         self.dest = None
@@ -144,9 +193,14 @@ def config_get(cls, config, above, *path) -> BaseConfig:
         args_above = cls.add_orig_label() + list(path)
         label_above = cls.add_label(*above) + list(path)
         no_label_above = cls.add_label(*above)[:-1] + list(path)
+
+        arg = None
         try:
             arg = traverse_config_get(args, *args_above)
         except KeyError as error:
+            pass
+
+        if arg is None:
             raise MissingArg(
                 "Arg %r missing from %s%s%s"
                 % (
@@ -155,23 +209,30 @@ def config_get(cls, config, above, *path) -> BaseConfig:
                     "." if args_above[:-1] else "",
                     ".".join(args_above[:-1]),
                 )
-            ) from error
-        try:
+            )
+
+        value = None
+        # Try to get the value specific to this label
+        with contextlib.suppress(KeyError):
             value = traverse_config_get(config, *label_above)
-        except KeyError as error:
-            try:
+
+        # Try to get the value specific to this plugin
+        if value is None:
+            with contextlib.suppress(KeyError):
                 value = traverse_config_get(config, *no_label_above)
-            except KeyError as error:
-                if "default" in arg:
-                    return arg["default"]
-                raise MissingConfig(
-                    "%s missing %r from %s"
-                    % (
-                        cls.__qualname__,
-                        label_above[-1],
-                        ".".join(label_above[:-1]),
-                    )
-                ) from error
+
+        if value is None:
+            # Return default if not found and available
+            if "default" in arg:
+                return arg["default"]
+            raise MissingConfig(
+                "%s missing %r from %s"
+                % (
+                    cls.__qualname__,
+                    label_above[-1],
+                    ".".join(label_above[:-1]),
+                )
+            )
 
         if value is None and "default" in arg:
             return arg["default"]
@@ -197,19 +258,67 @@ def config_get(cls, config, above, *path) -> BaseConfig:
         return value
 
     @classmethod
-    @abc.abstractmethod
-    def args(cls, *above) -> Dict[str, Any]:
+    def args(cls, args, *above) -> Dict[str, Arg]:
         """
         Return a dict containing arguments required for this class
         """
+        if getattr(cls, "CONFIG", None) is None:
+            raise AttributeError(
+                f"{cls.__qualname__} requires CONFIG property or implementation of args() classmethod"
+            )
+        for field in dataclasses.fields(cls.CONFIG):
+            arg = Arg(type=field.type)
+            # HACK For detecting dataclasses._MISSING_TYPE
+            if "dataclasses._MISSING_TYPE" not in repr(field.default):
+                arg["default"] = field.default
+            if field.type == bool:
+                arg["action"] = "store_true"
+            elif inspect.isclass(field.type):
+                if issubclass(field.type, list):
+                    arg["nargs"] = "+"
+                    if not hasattr(field.type, "SINGLETON"):
+                        raise AttributeError(
+                            f"{field.type.__qualname__} missing attribute SINGLETON"
+                        )
+                    arg["action"] = list_action(field.type)
+                    arg["type"] = field.type.SINGLETON
+                if hasattr(arg["type"], "load"):
+                    # TODO (python3.8) Use Protocol
+                    arg["type"] = arg["type"].load
+            elif get_origin(field.type) is list:
+                arg["type"] = get_args(field.type)[0]
+                arg["nargs"] = "+"
+            if "help" in field.metadata:
+                arg["help"] = field.metadata["help"]
+            cls.config_set(args, above, field.name, arg)
+        return args
 
     @classmethod
-    @abc.abstractmethod
     def config(cls, config, *above):
         """
         Create the BaseConfig required to instantiate this class by parsing the
         config dict.
         """
+        if getattr(cls, "CONFIG", None) is None:
+            raise AttributeError(
+                f"{cls.__qualname__} requires CONFIG property or implementation of config() classmethod"
+            )
+        # Build the arguments to the CONFIG class
+        kwargs: Dict[str, Any] = {}
+        for field in dataclasses.fields(cls.CONFIG):
+            kwargs[field.name] = got = cls.config_get(
+                config, above, field.name
+            )
+            if inspect.isclass(got) and issubclass(got, BaseConfigurable):
+                try:
+                    kwargs[field.name] = got.withconfig(
+                        config, *above, *cls.add_label()
+                    )
+                except MissingConfig:
+                    kwargs[field.name] = got.withconfig(
+                        config, *above, *cls.add_label()[:-1]
+                    )
+        return cls.CONFIG(**kwargs)
 
     @classmethod
     def withconfig(cls, config, *above):
diff --git a/dffml/feature/feature.py b/dffml/feature/feature.py
@@ -301,6 +301,7 @@ def length(self) -> int:
 class Features(list):
 
     TIMEOUT: int = 60 * 2
+    SINGLETON = Feature
 
     LOGGER = LOGGER.getChild("Features")
 
diff --git a/tests/test_base.py b/tests/test_base.py
@@ -0,0 +1,153 @@
+import unittest
+from dataclasses import field
+from typing import List
+
+from dffml.base import BaseDataFlowFacilitatorObject, config, list_action
+from dffml.feature.feature import DefFeature, Feature, Features
+from dffml.source.source import BaseSource
+from dffml.source.csv import CSVSource
+from dffml.source.json import JSONSource
+from dffml.util.entrypoint import entry_point, base_entry_point
+from dffml.util.cli.arg import Arg
+from dffml.util.cli.cmd import parse_unknown
+
+
+@config
+class FakeTestingConfig:
+    files: List[str]
+    features: Features
+    name: str = field(metadata={"help": "Name of FakeTesting"})
+    label: str = "unlabeled"
+    readonly: bool = False
+    source: BaseSource = JSONSource
+
+
+@base_entry_point("dffml.test", "test")
+class BaseTesting(BaseDataFlowFacilitatorObject):
+    pass  # pragma: no cov
+
+
+@entry_point("fake")
+class FakeTesting(BaseTesting):
+
+    CONFIG = FakeTestingConfig
+
+
+class TestAutoArgsConfig(unittest.TestCase):
+    def test_00_args(self):
+        self.maxDiff = 99999
+        self.assertEqual(
+            FakeTesting.args({}),
+            {
+                "test": {
+                    "arg": None,
+                    "config": {
+                        "fake": {
+                            "arg": None,
+                            "config": {
+                                "files": {
+                                    "arg": Arg(type=str, nargs="+"),
+                                    "config": {},
+                                },
+                                "features": {
+                                    "arg": Arg(
+                                        type=Feature.load,
+                                        nargs="+",
+                                        action=list_action(Features),
+                                    ),
+                                    "config": {},
+                                },
+                                "name": {
+                                    "arg": Arg(
+                                        type=str, help="Name of FakeTesting"
+                                    ),
+                                    "config": {},
+                                },
+                                "readonly": {
+                                    "arg": Arg(
+                                        type=bool,
+                                        action="store_true",
+                                        default=False,
+                                    ),
+                                    "config": {},
+                                },
+                                "label": {
+                                    "arg": Arg(type=str, default="unlabeled"),
+                                    "config": {},
+                                },
+                                "source": {
+                                    "arg": Arg(
+                                        type=BaseSource.load,
+                                        default=JSONSource,
+                                    ),
+                                    "config": {},
+                                },
+                            },
+                        }
+                    },
+                }
+            },
+        )
+
+    def test_config_defaults(self):
+        config = FakeTesting.config(
+            parse_unknown(
+                "--test-fake-name",
+                "feedface",
+                "--test-files",
+                "a",
+                "b",
+                "c",
+                "--test-source-filename",
+                "file.json",
+                "--test-features",
+                "def:Year:int:1",
+                "def:Commits:int:10",
+            )
+        )
+        self.assertEqual(config.files, ["a", "b", "c"])
+        self.assertEqual(config.name, "feedface")
+        self.assertEqual(config.label, "unlabeled")
+        self.assertFalse(config.readonly)
+        self.assertTrue(isinstance(config.source, JSONSource))
+        self.assertEqual(config.source.config.filename, "file.json")
+        self.assertEqual(
+            config.features,
+            Features(
+                DefFeature("Year", int, 1), DefFeature("Commits", int, 10)
+            ),
+        )
+
+    def test_config_set(self):
+        config = FakeTesting.config(
+            parse_unknown(
+                "--test-fake-name",
+                "feedface",
+                "--test-fake-label",
+                "default-label",
+                "--test-fake-readonly",
+                "--test-files",
+                "a",
+                "b",
+                "c",
+                "--test-fake-source",
+                "csv",
+                "--test-source-filename",
+                "file.csv",
+                "--test-features",
+                "def:Year:int:1",
+                "def:Commits:int:10",
+            )
+        )
+        self.assertEqual(config.files, ["a", "b", "c"])
+        self.assertEqual(config.name, "feedface")
+        self.assertEqual(config.label, "default-label")
+        self.assertTrue(config.readonly)
+        self.assertTrue(isinstance(config.source, CSVSource))
+        self.assertEqual(config.source.config.filename, "file.csv")
+        self.assertEqual(
+            config.features,
+            Features(
+                DefFeature("Year", int, 1), DefFeature("Commits", int, 10)
+            ),
+        )
