source: json: Use auto args and config

Signed-off-by: John Andersen <[email protected]>

Author: pdxjohnny

dffml/base.py

@@ -110,7 +110,11 @@ def config(cls):
     """
     Decorator to create a dataclass
     """
-    return dataclasses.dataclass(eq=True, frozen=True)(cls)
+    datacls = dataclasses.dataclass(eq=True, frozen=True)(cls)
+    datacls._replace = lambda self, *args, **kwargs: dataclasses.replace(
+        self, *args, **kwargs
+    )
+    return datacls
 
 
 class ConfigurableParsingNamespace(object):

dffml/source/file.py

@@ -1,35 +1,38 @@
 # SPDX-License-Identifier: MIT
 # Copyright (c) 2019 Intel Corporation
 import os
+import io
 import abc
-import asyncio
-import gzip
 import bz2
+import gzip
 import lzma
+import asyncio
 import zipfile
-import io
-from typing import NamedTuple, Tuple, Dict
 from contextlib import contextmanager
+from dataclasses import dataclass, field, fields
+from typing import NamedTuple, Tuple, Dict, List
 
-from ..base import BaseConfig
+from ..base import config
 from .source import BaseSource
 from ..util.cli.arg import Arg
 from ..util.cli.cmd import CMD
+from ..util.entrypoint import entry_point
 
 
-class FileSourceConfig(BaseConfig, NamedTuple):
+@config
+class FileSourceConfig:
     filename: str
     label: str = "unlabeled"
     readonly: bool = False
 
 
+@entry_point("file")
 class FileSource(BaseSource):
     """
     FileSource reads and write from a file on open / close.
     """
 
-    ENTRY_POINT_ORIG_LABEL = "file"
-    ENTRY_POINT_LABEL = ENTRY_POINT_ORIG_LABEL
+    CONFIG = FileSourceConfig
 
     async def __aenter__(self) -> "BaseSourceContext":
         await self._open()
@@ -108,25 +111,3 @@ async def load_fd(self, fd):
     @abc.abstractmethod
     async def dump_fd(self, fd):
         pass  # pragma: no cover
-
-    @classmethod
-    def args(cls, args, *above) -> Dict[str, Arg]:
-        cls.config_set(args, above, "filename", Arg())
-        cls.config_set(
-            args,
-            above,
-            "readonly",
-            Arg(type=bool, action="store_true", default=False),
-        )
-        cls.config_set(
-            args, above, "label", Arg(type=str, default="unlabeled")
-        )
-        return args
-
-    @classmethod
-    def config(cls, config, *above):
-        return FileSourceConfig(
-            filename=cls.config_get(config, above, "filename"),
-            readonly=cls.config_get(config, above, "readonly"),
-            label=cls.config_get(config, above, "label"),
-        )

dffml/source/json.py

@@ -8,14 +8,18 @@
 
 from ..repo import Repo
 from .memory import MemorySource
-from .file import FileSource
+from .file import FileSource, FileSourceConfig
 from ..util.entrypoint import entry_point
 
 from .log import LOGGER
 
 LOGGER = LOGGER.getChild("json")
 
 
+class JSONSourceConfig(FileSourceConfig):
+    pass  # pragma: no cov
+
+
 @dataclass
 class OpenJSONFile:
     data: Dict[str, Dict]
@@ -39,6 +43,7 @@ class JSONSource(FileSource, MemorySource):
     stored in memory.
     """
 
+    CONFIG = JSONSourceConfig
     OPEN_JSON_FILES: Dict[str, OpenJSONFile] = {}
     OPEN_JSON_FILES_LOCK: asyncio.Lock = asyncio.Lock()
 

dffml/source/memory.py

@@ -6,7 +6,7 @@
 import asyncio
 from typing import Any, Dict, List, NamedTuple, AsyncIterator
 
-from ..base import BaseConfig
+from ..base import config
 from ..repo import Repo
 from .source import BaseSourceContext, BaseSource
 from ..util.cli.arg import Arg
@@ -25,7 +25,8 @@ async def repo(self, src_url: str) -> Repo:
         return self.parent.mem.get(src_url, Repo(src_url))
 
 
-class MemorySourceConfig(BaseConfig, NamedTuple):
+@config
+class MemorySourceConfig:
     repos: List[Repo]
 
 
@@ -35,23 +36,11 @@ class MemorySource(BaseSource):
     Stores repos in a dict in memory
     """
 
+    CONFIG = MemorySourceConfig
     CONTEXT = MemorySourceContext
 
-    def __init__(self, config: BaseConfig) -> None:
+    def __init__(self, config: MemorySourceConfig) -> None:
         super().__init__(config)
         self.mem: Dict[str, Repo] = {}
         if isinstance(self.config, MemorySourceConfig):
             self.mem = {repo.src_url: repo for repo in self.config.repos}
-
-    @classmethod
-    def args(cls, args, *above) -> Dict[str, Arg]:
-        cls.config_set(
-            args, above, "keys", Arg(type=str, nargs="+", default=[])
-        )
-        return args
-
-    @classmethod
-    def config(cls, config, *above):
-        return MemorySourceConfig(
-            repos=list(map(Repo, cls.config_get(config, above, "keys")))
-        )

tests/source/test_file.py

@@ -61,7 +61,10 @@ def test_args(self):
                         "file": {
                             "arg": None,
                             "config": {
-                                "filename": {"arg": Arg(), "config": {}},
+                                "filename": {
+                                    "arg": Arg(type=str),
+                                    "config": {},
+                                },
                                 "readonly": {
                                     "arg": Arg(
                                         type=bool,

tests/source/test_json.py

@@ -1,11 +1,10 @@
 # SPDX-License-Identifier: MIT
 # Copyright (c) 2019 Intel Corporation
-from dffml.source.file import FileSourceConfig
-from dffml.source.json import JSONSource
+from dffml.source.json import JSONSource, JSONSourceConfig
 from dffml.util.testing.source import FileSourceTest
 from dffml.util.asynctestcase import AsyncTestCase
 
 
-class TestCustomSQliteSource(FileSourceTest, AsyncTestCase):
+class TestJSONSource(FileSourceTest, AsyncTestCase):
     async def setUpSource(self):
-        return JSONSource(FileSourceConfig(filename=self.testfile))
+        return JSONSource(JSONSourceConfig(filename=self.testfile))

tests/test_integration_cli.py

@@ -0,0 +1,69 @@
+"""
+This file contains integration tests. We use the CLI to exercise functionality of
+various DFFML classes and constructs.
+"""
+import io
+import contextlib
+
+from dffml.cli.cli import CLI
+from dffml.util.asynctestcase import AsyncTestCase
+
+from .test_cli import non_existant_tempfile
+
+
+class IntegrationCLITestCase(AsyncTestCase):
+    async def setUp(self):
+        super().setUp()
+        self.stdout = io.StringIO()
+        self._stack = contextlib.ExitStack().__enter__()
+
+    async def tearDown(self):
+        super().tearDown()
+        self._stack.__exit__(None, None, None)
+
+    def mktempfile(self):
+        return self._stack.enter_context(non_existant_tempfile())
+
+
+class TestList(IntegrationCLITestCase):
+    async def test_repos(self):
+        src_urls = ["A", "B", "C"]
+        with contextlib.redirect_stdout(self.stdout):
+            await CLI.cli(
+                "list",
+                "repos",
+                "-sources",
+                "feed=memory",
+                "-source-repos",
+                *src_urls,
+            )
+        stdout = self.stdout.getvalue()
+        for src_url in src_urls:
+            self.assertIn(src_url, stdout)
+
+
+class TestMerge(IntegrationCLITestCase):
+    async def test_memory_to_json(self):
+        src_urls = ["A", "B", "C"]
+        filename = self.mktempfile()
+        await CLI.cli(
+            "merge",
+            "dest=json",
+            "src=memory",
+            "-source-dest-filename",
+            filename,
+            "-source-src-repos",
+            *src_urls,
+        )
+        with contextlib.redirect_stdout(self.stdout):
+            await CLI.cli(
+                "list",
+                "repos",
+                "-sources",
+                "tmp=json",
+                "-source-tmp-filename",
+                filename,
+            )
+        stdout = self.stdout.getvalue()
+        for src_url in src_urls:
+            self.assertIn(src_url, stdout)