util: cli: arg: Add configloading ability to parse_unknown

Signed-off-by: sakshamarora1 <[email protected]>

Author: sakshamarora1

CHANGELOG.md

@@ -6,11 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### Added
+- Configloading ablity from CLI using "@" before filename
 - Docstrings and doctestable example for DataFlowSource
 - XGBoost Regression Model
 - Pre-Trained PyTorch torchvision Models
 - Spacy model for NER
-- Added ability to rename outputs using GetSingle
+- Ability to rename outputs using GetSingle
 - Tutorial for using NLP operations with models
 - Operations plugin for NLP wrapping spacy and scikit functions
 - Support for default value in a Definition

dffml/util/cli/arg.py

@@ -6,7 +6,7 @@
 from ..data import traverse_config_set
 
 
-def parse_unknown(*unknown):
+async def parse_unknown(*unknown, configloaders=None):
     parsed = {}
     name = []
     add_to_parsed = []
@@ -24,6 +24,8 @@ def parse_unknown(*unknown):
             name = arg.lstrip("-").split("-")
             add_to_parsed = []
         else:
+            if arg.startswith("@") and configloaders is not None:
+                _, arg = await configloaders.load_file(arg[1:])
             add_to_parsed.append(arg)
     if unknown and name:
         if not add_to_parsed:

dffml/util/cli/cmd.py

@@ -17,6 +17,7 @@
 from ..data import export_dict
 from .arg import Arg, parse_unknown
 from ...base import config, mkarg, field
+from ...configloader.configloader import ConfigLoaders
 
 DisplayHelp = "Display help message"
 
@@ -204,7 +205,10 @@ async def do_run(self):
     @classmethod
     async def cli(cls, *args):
         parser, (args, unknown) = await cls.parse_args(*args)
-        args.extra_config = parse_unknown(*unknown)
+        async with ConfigLoaders() as configloaders:
+            args.extra_config = await parse_unknown(
+                *unknown, configloaders=configloaders
+            )
         if (
             getattr(cls, "run", None) is not None
             and getattr(args, "cmd", None) is None

model/tensorflow/tests/test_dnnc.py

@@ -60,7 +60,7 @@ def tearDownClass(cls):
 
     async def test_config(self):
         config = self.model.__class__.config(
-            parse_unknown(
+            await parse_unknown(
                 "--model-predict",
                 "feature_name:int:1",
                 "--model-classifications",

model/tensorflow/tests/test_dnnr.py

@@ -64,7 +64,7 @@ def tearDownClass(cls):
     async def test_config(self):
         # Setting up configuration for model
         config = self.model.__class__.config(
-            parse_unknown(
+            await parse_unknown(
                 "--model-predict",
                 "TARGET:float:1",
                 "--model-features",

service/http/tests/test_routes.py

@@ -155,7 +155,7 @@ async def test_models(self):
 
 class TestRoutesConfigure(TestRoutesRunning, AsyncTestCase):
     async def test_source(self):
-        config = parse_unknown(
+        config = await parse_unknown(
             "--source-filename", "dataset.csv", "-source-allowempty"
         )
         async with self.post("/configure/source/csv/salary", json=config) as r:
@@ -177,7 +177,7 @@ async def test_source(self):
                     self.assertIn("salaryctx", self.cli.app["source_contexts"])
 
     async def test_source_error(self):
-        config = parse_unknown("--source-file", "dataset.csv")
+        config = await parse_unknown("--source-file", "dataset.csv")
         with self.assertRaisesRegex(ServerException, "missing.*filename"):
             async with self.post("/configure/source/csv/salary", json=config):
                 pass  # pramga: no cov
@@ -193,7 +193,7 @@ async def test_model(self):
         with tempfile.TemporaryDirectory() as tempdir, patch.object(
             Model, "load", new=model_load
         ):
-            config = parse_unknown(
+            config = await parse_unknown(
                 "--model-directory",
                 tempdir,
                 "--model-features",
@@ -230,7 +230,7 @@ async def test_model(self):
 
     async def test_model_config_error(self):
         # Should be directory, not folder
-        config = parse_unknown("--model-directory", "mymodel_dir")
+        config = await parse_unknown("--model-directory", "mymodel_dir")
         with patch.object(Model, "load", new=model_load):
             with self.assertRaisesRegex(ServerException, "missing.*features"):
                 async with self.post(

tests/df/test_memory.py

@@ -61,19 +61,19 @@ def test_args(self):
             },
         )
 
-    @patch.object(BaseKeyValueStore, "load", load_kvstore_with_args)
-    def test_config_default_label(self):
-        was = MemoryRedundancyChecker.config(
-            parse_unknown(
-                "--rchecker-memory-kvstore",
-                "withargs",
-                "--rchecker-memory-kvstore-withargs-filename",
-                "somefile",
+    async def test_config_default_label(self):
+        with patch.object(BaseKeyValueStore, "load", load_kvstore_with_args):
+            was = MemoryRedundancyChecker.config(
+                await parse_unknown(
+                    "--rchecker-memory-kvstore",
+                    "withargs",
+                    "--rchecker-memory-kvstore-withargs-filename",
+                    "somefile",
+                )
             )
-        )
-        self.assertEqual(type(was), MemoryRedundancyCheckerConfig)
-        self.assertEqual(type(was.kvstore), KeyValueStoreWithArguments)
-        self.assertEqual(
-            type(was.kvstore.config), KeyValueStoreWithArgumentsConfig
-        )
-        self.assertEqual(was.kvstore.config.filename, "somefile")
+            self.assertEqual(type(was), MemoryRedundancyCheckerConfig)
+            self.assertEqual(type(was.kvstore), KeyValueStoreWithArguments)
+            self.assertEqual(
+                type(was.kvstore.config), KeyValueStoreWithArgumentsConfig
+            )
+            self.assertEqual(was.kvstore.config.filename, "somefile")

tests/source/test_csv.py

@@ -54,9 +54,9 @@ async def test_tag(self):
                 self.assertEqual("1", rows["untagged"]["0"]["feed"])
                 self.assertEqual("2", rows["sometag"]["0"]["face"])
 
-    def test_config_default(self):
+    async def test_config_default(self):
         config = CSVSource.config(
-            parse_unknown("--source-csv-filename", "feedface")
+            await parse_unknown("--source-csv-filename", "feedface")
         )
         self.assertEqual(config.filename, "feedface")
         self.assertEqual(config.tag, "untagged")
@@ -66,9 +66,9 @@ def test_config_default(self):
         self.assertFalse(config.allowempty)
         self.assertIsNone(config.loadfiles)
 
-    def test_config_set(self):
+    async def test_config_set(self):
         config = CSVSource.config(
-            parse_unknown(
+            await parse_unknown(
                 "--source-csv-filename",
                 "feedface",
                 "--source-csv-tag",

tests/source/test_file.py

@@ -79,18 +79,18 @@ def test_args(self):
             },
         )
 
-    def test_config_readonly_default(self):
+    async def test_config_readonly_default(self):
         config = FileSource.config(
-            parse_unknown("--source-file-filename", "feedface")
+            await parse_unknown("--source-file-filename", "feedface")
         )
         self.assertEqual(config.filename, "feedface")
         self.assertEqual(config.tag, "untagged")
         self.assertFalse(config.readwrite)
         self.assertFalse(config.allowempty)
 
-    def test_config_readonly_set(self):
+    async def test_config_readonly_set(self):
         config = FileSource.config(
-            parse_unknown(
+            await parse_unknown(
                 "--source-file-filename",
                 "feedface",
                 "--source-file-tag",

tests/test_base.py

@@ -16,6 +16,7 @@
 from dffml.util.entrypoint import entrypoint, base_entry_point
 from dffml.util.cli.arg import Arg
 from dffml.util.cli.cmd import parse_unknown
+from dffml.util.asynctestcase import IntegrationCLITestCase
 
 
 @config
@@ -41,7 +42,7 @@ class FakeTesting(BaseTesting):
     CONFIG = FakeTestingConfig
 
 
-class TestAutoArgsConfig(unittest.TestCase):
+class TestAutoArgsConfig(IntegrationCLITestCase):
     def test_00_args(self):
         self.maxDiff = 99999
         self.assertEqual(
@@ -105,9 +106,9 @@ def test_00_args(self):
             },
         )
 
-    def test_config_defaults(self):
+    async def test_config_defaults(self):
         config = FakeTesting.config(
-            parse_unknown(
+            await parse_unknown(
                 "--test-fake-name",
                 "feedface",
                 "--test-num",
@@ -140,9 +141,9 @@ def test_config_defaults(self):
         )
         self.assertEqual(config.nums, (100,))
 
-    def test_config_set(self):
+    async def test_config_set(self):
         config = FakeTesting.config(
-            parse_unknown(
+            await parse_unknown(
                 "--test-fake-name",
                 "feedface",
                 "--test-num",