Merge branch 'development' into alex_collector_args

alexandraBara · alexandraBara · commit 46fb2279718a · 2025-07-16T09:10:17.000-05:00
diff --git a/README.md b/README.md
@@ -17,10 +17,10 @@ The Node Scraper CLI can be used to run Node Scraper plugins on a target system.
 options are available:
 
 ```sh
-usage: node-scraper [-h] [--sys-name STRING] [--sys-location {LOCAL,REMOTE}]
-                    [--sys-interaction-level {PASSIVE,INTERACTIVE,DISRUPTIVE}] [--sys-sku STRING] [--sys-platform STRING]
-                    [--plugin-configs [STRING ...]] [--system-config STRING] [--connection-config STRING] [--log-path STRING]
-                    [--log-level {CRITICAL,FATAL,ERROR,WARN,WARNING,INFO,DEBUG,NOTSET}] [--gen-reference-config]
+usage: node-scraper [-h] [--sys-name STRING] [--sys-location {LOCAL,REMOTE}] [--sys-interaction-level {PASSIVE,INTERACTIVE,DISRUPTIVE}]
+                    [--sys-sku STRING] [--sys-platform STRING] [--plugin-configs [STRING ...]] [--system-config STRING]
+                    [--connection-config STRING] [--log-path STRING] [--log-level {CRITICAL,FATAL,ERROR,WARN,WARNING,INFO,DEBUG,NOTSET}]
+                    [--gen-reference-config]
                     {run-plugins,describe,gen-plugin-config} ...
 
 node scraper CLI
@@ -38,14 +38,12 @@ options:
   --sys-location {LOCAL,REMOTE}
                         Location of target system (default: LOCAL)
   --sys-interaction-level {PASSIVE,INTERACTIVE,DISRUPTIVE}
-                        Specify system interaction level, used to determine the type of actions that plugins can perform (default:
-                        INTERACTIVE)
+                        Specify system interaction level, used to determine the type of actions that plugins can perform (default: INTERACTIVE)
   --sys-sku STRING      Manually specify SKU of system (default: None)
   --sys-platform STRING
                         Specify system platform (default: None)
   --plugin-configs [STRING ...]
-                        built-in config names or paths to plugin config JSONs. Available built-in configs: NodeStatus (default:
-                        None)
+                        built-in config names or paths to plugin config JSONs. Available built-in configs: NodeStatus (default: None)
   --system-config STRING
                         Path to system config json (default: None)
   --connection-config STRING
@@ -54,7 +52,8 @@ options:
   --log-level {CRITICAL,FATAL,ERROR,WARN,WARNING,INFO,DEBUG,NOTSET}
                         Change python log level (default: INFO)
   --gen-reference-config
-                        Generate reference config. File will be written to ./reference_config.json. (default: False)
+                        Generate reference config from system. Writes to ./reference_config.json. (default: False)
+
 
 ```
 
@@ -254,8 +253,8 @@ Here is an example of a comprehensive plugin config that specifies analyzer args
 ```
 
 2. **'gen-reference-config' command**
-This command can be used generate a reference config that is populated with current system
-configurations. The plugins that use analyzer args, where applied, will be populated with system
+This command can be used to generate a reference config that is populated with current system
+configurations. Plugins that use analyzer args (where applicable) will be populated with system
 data.
 Sample command:
 ```sh
@@ -286,8 +285,16 @@ This will generate the following config:
   },
   "result_collators": {}
 ```
-This can be later used on a different platform for comparison, using the steps at #2:
+This config can later be used on a different platform for comparison, using the steps at #2:
 ```sh
 node-scraper --plugin-configs reference_config.json
 
 ```
+
+An alternate way to generate a reference config is by using log files from a previous run. The
+example below uses log files from 'scraper_logs_<path>/':
+```sh
+node-scraper gen-plugin-config --gen-reference-config-from-logs scraper_logs_<path>/ --output-path custom_output_dir
+```
+This will generate a reference config that includes plugins with logged results in
+'scraper_log_<path>' and save the new config to 'custom_output_dir/reference_config.json'.
diff --git a/nodescraper/cli/cli.py b/nodescraper/cli/cli.py
@@ -152,13 +152,6 @@ def build_parser(
         help="Generate reference config from system. Writes to ./reference_config.json.",
     )
 
-    parser.add_argument(
-        "--gen-reference-config-from-logs",
-        dest="reference_config_from_logs",
-        type=log_path_arg,
-        help="Generate reference config from previous run logfiles. Writes to ./reference_config.json.",
-    )
-
     subparsers = parser.add_subparsers(dest="subcmd", help="Subcommands")
 
     run_plugin_parser = subparsers.add_parser(
@@ -188,6 +181,13 @@ def build_parser(
         help="Generate a config for a plugin or list of plugins",
     )
 
+    config_builder_parser.add_argument(
+        "--gen-reference-config-from-logs",
+        dest="reference_config_from_logs",
+        type=log_path_arg,
+        help="Generate reference config from previous run logfiles. Writes to --output-path/reference_config.json if provided, otherwise ./reference_config.json.",
+    )
+
     config_builder_parser.add_argument(
         "--plugins",
         nargs="*",
@@ -346,24 +346,28 @@ def main(arg_input: Optional[list[str]] = None):
             parse_describe(parsed_args, plugin_reg, config_reg, logger)
 
         if parsed_args.subcmd == "gen-plugin-config":
-            parse_gen_plugin_config(parsed_args, plugin_reg, config_reg, logger)
 
-        if parsed_args.reference_config_from_logs:
-            ref_config = generate_reference_config_from_logs(
-                parsed_args.reference_config_from_logs, plugin_reg, logger
-            )
-            path = os.path.join(os.getcwd(), "reference_config.json")
-            try:
-                with open(path, "w") as f:
-                    json.dump(
-                        ref_config.model_dump(mode="json", exclude_none=True),
-                        f,
-                        indent=2,
-                    )
-                    logger.info("Reference config written to: %s", path)
-            except Exception as exp:
-                logger.error(exp)
-            sys.exit(0)
+            if parsed_args.reference_config_from_logs:
+                ref_config = generate_reference_config_from_logs(
+                    parsed_args.reference_config_from_logs, plugin_reg, logger
+                )
+                output_path = os.getcwd()
+                if parsed_args.output_path:
+                    output_path = parsed_args.output_path
+                path = os.path.join(output_path, "reference_config.json")
+                try:
+                    with open(path, "w") as f:
+                        json.dump(
+                            ref_config.model_dump(mode="json", exclude_none=True),
+                            f,
+                            indent=2,
+                        )
+                        logger.info("Reference config written to: %s", path)
+                except Exception as exp:
+                    logger.error(exp)
+                sys.exit(0)
+
+            parse_gen_plugin_config(parsed_args, plugin_reg, config_reg, logger)
 
         parsed_plugin_args = {}
         for plugin, plugin_args in plugin_arg_map.items():
diff --git a/nodescraper/cli/helper.py b/nodescraper/cli/helper.py
@@ -31,6 +31,8 @@
 from pathlib import Path
 from typing import Optional, Tuple
 
+from pydantic import BaseModel
+
 from nodescraper.cli.inputargtypes import ModelArgHandler
 from nodescraper.configbuilder import ConfigBuilder
 from nodescraper.configregistry import ConfigRegistry
@@ -284,6 +286,33 @@ def log_system_info(log_path: str | None, system_info: SystemInfo, logger: loggi
             logger.error(exp)
 
 
+def extract_analyzer_args_from_model(
+    plugin_cls: type, data_model: BaseModel, logger: logging.Logger
+) -> Optional[BaseModel]:
+    """Extract analyzer args from a plugin and a data model.
+
+    Args:
+        plugin_cls (type): The plugin class from registry.
+        data_model (BaseModel): System data model.
+        logger (logging.Logger): logger.
+
+    Returns:
+        Optional[BaseModel]: Instance of analyzer args model or None if unavailable.
+    """
+    if not hasattr(plugin_cls, "ANALYZER_ARGS") or not plugin_cls.ANALYZER_ARGS:
+        logger.warning(
+            "Plugin: %s does not support reference config creation. No analyzer args defined.",
+            getattr(plugin_cls, "__name__", str(plugin_cls)),
+        )
+        return None
+
+    try:
+        return plugin_cls.ANALYZER_ARGS.build_from_model(data_model)
+    except NotImplementedError as e:
+        logger.info("%s: %s", plugin_cls.__name__, str(e))
+        return None
+
+
 def generate_reference_config(
     results: list[PluginResult], plugin_reg: PluginRegistry, logger: logging.Logger
 ) -> PluginConfig:
@@ -314,18 +343,9 @@ def generate_reference_config(
             continue
 
         plugin = plugin_reg.plugins.get(obj.source)
-        if not plugin.ANALYZER_ARGS:
-            logger.warning(
-                "Plugin: %s does not support reference config creation. No analyzer args defined, skipping.",
-                obj.source,
-            )
-            continue
 
-        args = None
-        try:
-            args = plugin.ANALYZER_ARGS.build_from_model(data_model)
-        except NotImplementedError as nperr:
-            logger.info(nperr)
+        args = extract_analyzer_args_from_model(plugin, data_model, logger)
+        if not args:
             continue
         plugins[obj.source] = {"analysis_args": {}}
         plugins[obj.source]["analysis_args"] = args.model_dump(exclude_none=True)
@@ -357,24 +377,20 @@ def generate_reference_config_from_logs(
         dm_path = Path(dm)
         dm_payload = json.loads(dm_path.read_text(encoding="utf-8"))
         plugin = plugin_reg.plugins.get(task_res.parent)
-        data_model = plugin.DATA_MODEL.model_validate(dm_payload)
-
-        if not plugin.ANALYZER_ARGS:
+        if not plugin:
             logger.warning(
-                "Plugin: %s does not support reference config creation. No analyzer args defined.",
+                "Plugin %s not found in the plugin registry: %s.",
                 task_res.parent,
             )
             continue
 
-        args = None
+        data_model = plugin.DATA_MODEL.model_validate(dm_payload)
 
-        try:
-            args = plugin.ANALYZER_ARGS.build_from_model(data_model)
-        except NotImplementedError as nperr:
-            logger.info(nperr)
+        args = extract_analyzer_args_from_model(plugin, data_model, logger)
+        if not args:
             continue
-        plugins[task_res.parent] = {"analysis_args": {}}
-        plugins[task_res.parent]["analysis_args"] = args.model_dump(exclude_none=True)
+
+        plugins[task_res.parent] = {"analysis_args": args.model_dump(exclude_none=True)}
 
     plugin_config.plugins = plugins
     return plugin_config
diff --git a/nodescraper/models/taskresult.py b/nodescraper/models/taskresult.py
@@ -25,7 +25,7 @@
 ###############################################################################
 import datetime
 import logging
-from typing import Optional
+from typing import Any, Optional
 
 from pydantic import BaseModel, Field, field_serializer, field_validator
 
@@ -67,7 +67,18 @@ def serialize_status(self, status: ExecutionStatus, _info) -> str:
 
     @field_validator("status", mode="before")
     @classmethod
-    def _coerce_status(cls, v):
+    def validate_status(cls, v: Any):
+        """Validator to ensure `status` is a valid ExecutionStatus enum.
+
+        Args:
+            v (Any): The input value to validate (can be str or ExecutionStatus).
+
+        Returns:
+            ExecutionStatus: The validated enum value.
+
+        Raises:
+            ValueError: If the string is not a valid enum name.
+        """
         if isinstance(v, ExecutionStatus):
             return v
         if isinstance(v, str):
diff --git a/test/unit/framework/test_cli_helper.py b/test/unit/framework/test_cli_helper.py
