Merge branch 'development' into feature/mypy-precommit

Author: sidamd

README.md

@@ -21,13 +21,14 @@ usage: node-scraper [-h] [--sys-name STRING] [--sys-location {LOCAL,REMOTE}] [--
                     [--sys-sku STRING] [--sys-platform STRING] [--plugin-configs [STRING ...]] [--system-config STRING]
                     [--connection-config STRING] [--log-path STRING] [--log-level {CRITICAL,FATAL,ERROR,WARN,WARNING,INFO,DEBUG,NOTSET}]
                     [--gen-reference-config]
-                    {run-plugins,describe,gen-plugin-config} ...
+                    {summary,run-plugins,describe,gen-plugin-config} ...
 
 node scraper CLI
 
 positional arguments:
-  {run-plugins,describe,gen-plugin-config}
+  {summary,run-plugins,describe,gen-plugin-config}
                         Subcommands
+    summary             Generates summary csv file
     run-plugins         Run a series of plugins
     describe            Display details on a built-in config or plugin
     gen-plugin-config   Generate a config for a plugin or list of plugins
@@ -38,7 +39,8 @@ options:
   --sys-location {LOCAL,REMOTE}
                         Location of target system (default: LOCAL)
   --sys-interaction-level {PASSIVE,INTERACTIVE,DISRUPTIVE}
-                        Specify system interaction level, used to determine the type of actions that plugins can perform (default: INTERACTIVE)
+                        Specify system interaction level, used to determine the type of actions that plugins can perform (default:
+                        INTERACTIVE)
   --sys-sku STRING      Manually specify SKU of system (default: None)
   --sys-platform STRING
                         Specify system platform (default: None)
@@ -54,7 +56,6 @@ options:
   --gen-reference-config
                         Generate reference config from system. Writes to ./reference_config.json. (default: False)
 
-
 ```
 
 ### Subcommmands
@@ -167,6 +168,16 @@ This would produce the following config:
 }
 ```
 
+4. **'summary' sub command**
+The 'summary' subcommand can be used to combine results from multiple runs of node-scraper to a
+single summary.csv file. Sample run:
+```sh
+node-scraper summary --summary_path /<path_to_node-scraper_logs>
+```
+This will generate a new file '/<path_to_node-scraper_logs>/summary.csv' file. This file will
+contain the results from all 'nodescraper.csv' files from '/<path_to_node-scarper_logs>'.
+
+
 ### Plugin Configs
 A plugin JSON config should follow the structure of the plugin config model defined here.
 The globals field is a dictionary of global key-value pairs; values in globals will be passed to

dev-setup.sh

@@ -1,7 +1,7 @@
 # Create venv if not already present
 if [ ! -d "venv" ]; then
-    python3 -m pip install virtualenv
-    python3 -m virtualenv venv
+    python3 -m pip install venv
+    python3 -m venv venv
 fi
 
 # Activate the desired venv

nodescraper/base/inbandcollectortask.py

@@ -27,7 +27,7 @@
 from typing import Generic, Optional
 
 from nodescraper.connection.inband import InBandConnection
-from nodescraper.connection.inband.inband import CommandArtifact, FileArtifact
+from nodescraper.connection.inband.inband import BaseFileArtifact, CommandArtifact
 from nodescraper.enums import EventPriority, OSFamily, SystemInteractionLevel
 from nodescraper.generictypes import TCollectArg, TDataModel
 from nodescraper.interfaces import DataCollector, TaskResultHook
@@ -99,7 +99,7 @@ def _run_sut_cmd(
 
     def _read_sut_file(
         self, filename: str, encoding="utf-8", strip: bool = True, log_artifact=True
-    ) -> FileArtifact:
+    ) -> BaseFileArtifact:
         """
         Read a file from the SUT and return its content.
 
@@ -110,7 +110,7 @@ def _read_sut_file(
             log_artifact (bool, optional): whether we should log the contents of the file. Defaults to True.
 
         Returns:
-            FileArtifact: The content of the file read from the SUT, which includes the file name and content
+            BaseFileArtifact: The content of the file read from the SUT, which includes the file name and content
         """
         file_res = self.connection.read_file(filename=filename, encoding=encoding, strip=strip)
         if log_artifact:

nodescraper/cli/cli.py

@@ -35,8 +35,10 @@
 from nodescraper.cli.constants import DEFAULT_CONFIG, META_VAR_MAP
 from nodescraper.cli.dynamicparserbuilder import DynamicParserBuilder
 from nodescraper.cli.helper import (
+    dump_results_to_csv,
     generate_reference_config,
     generate_reference_config_from_logs,
+    generate_summary,
     get_plugin_configs,
     get_system_info,
     log_system_info,
@@ -152,8 +154,34 @@ def build_parser(
         help="Generate reference config from system. Writes to ./reference_config.json.",
     )
 
+    parser.add_argument(
+        "--skip-sudo",
+        dest="skip_sudo",
+        action="store_true",
+        help="Skip plugins that require sudo permissions",
+    )
+
     subparsers = parser.add_subparsers(dest="subcmd", help="Subcommands")
 
+    summary_parser = subparsers.add_parser(
+        "summary",
+        help="Generates summary csv file",
+    )
+
+    summary_parser.add_argument(
+        "--search-path",
+        dest="search_path",
+        type=log_path_arg,
+        help="Path to node-scraper previously generated results.",
+    )
+
+    summary_parser.add_argument(
+        "--output-path",
+        dest="output_path",
+        type=log_path_arg,
+        help="Specifies path for summary.csv.",
+    )
+
     run_plugin_parser = subparsers.add_parser(
         "run-plugins",
         help="Run a series of plugins",
@@ -249,7 +277,7 @@ def setup_logger(log_level: str = "INFO", log_path: str | None = None) -> loggin
     handlers = [logging.StreamHandler(stream=sys.stdout)]
 
     if log_path:
-        log_file_name = os.path.join(log_path, "errorscraper.log")
+        log_file_name = os.path.join(log_path, "nodescraper.log")
         handlers.append(
             logging.FileHandler(filename=log_file_name, mode="wt", encoding="utf-8"),
         )
@@ -327,12 +355,13 @@ def main(arg_input: Optional[list[str]] = None):
 
         parsed_args = parser.parse_args(top_level_args)
         system_info = get_system_info(parsed_args)
+        sname = system_info.name.lower().replace("-", "_").replace(".", "_")
+        timestamp = datetime.datetime.now().strftime("%Y_%m_%d-%I_%M_%S_%p")
 
         if parsed_args.log_path and parsed_args.subcmd not in ["gen-plugin-config", "describe"]:
-            sname = system_info.name.lower().replace("-", "_").replace(".", "_")
             log_path = os.path.join(
                 parsed_args.log_path,
-                f"scraper_logs_{sname}_{datetime.datetime.now().strftime('%Y_%m_%d-%I_%M_%S_%p')}",
+                f"scraper_logs_{sname}_{timestamp}",
             )
             os.makedirs(log_path)
         else:
@@ -342,6 +371,10 @@ def main(arg_input: Optional[list[str]] = None):
         if log_path:
             logger.info("Log path: %s", log_path)
 
+        if parsed_args.subcmd == "summary":
+            generate_summary(parsed_args.search_path, parsed_args.output_path, logger)
+            sys.exit(0)
+
         if parsed_args.subcmd == "describe":
             parse_describe(parsed_args, plugin_reg, config_reg, logger)
 
@@ -392,6 +425,11 @@ def main(arg_input: Optional[list[str]] = None):
             plugin_subparser_map=plugin_subparser_map,
         )
 
+        if parsed_args.skip_sudo:
+            plugin_config_inst_list[-1].global_args.setdefault("collection_args", {})[
+                "skip_sudo"
+            ] = True
+
         log_system_info(log_path, system_info, logger)
     except Exception as e:
         parser.error(str(e))
@@ -407,9 +445,14 @@ def main(arg_input: Optional[list[str]] = None):
     try:
         results = plugin_executor.run_queue()
 
+        dump_results_to_csv(results, sname, log_path, timestamp, logger)
+
         if parsed_args.reference_config:
             ref_config = generate_reference_config(results, plugin_reg, logger)
-            path = os.path.join(os.getcwd(), "reference_config.json")
+            if log_path:
+                path = os.path.join(log_path, "reference_config.json")
+            else:
+                path = os.path.join(os.getcwd(), "reference_config.json")
             try:
                 with open(path, "w") as f:
                     json.dump(

nodescraper/cli/helper.py

@@ -24,6 +24,8 @@
 #
 ###############################################################################
 import argparse
+import csv
+import glob
 import json
 import logging
 import os
@@ -331,7 +333,7 @@ def generate_reference_config(
     for obj in results:
         if obj.result_data.collection_result.status != ExecutionStatus.OK:
             logger.warning(
-                "Plugin: %s result status is %, skipping",
+                "Plugin: %s result status is %s, skipping",
                 obj.source,
                 obj.result_data.collection_result.status,
             )
@@ -344,11 +346,13 @@ def generate_reference_config(
 
         plugin = plugin_reg.plugins.get(obj.source)
 
-        args = extract_analyzer_args_from_model(plugin, data_model, logger)
-        if not args:
-            continue
-        plugins[obj.source] = {"analysis_args": {}}
-        plugins[obj.source]["analysis_args"] = args.model_dump(exclude_none=True)
+        if obj.source not in plugins:
+            plugins[obj.source] = {}
+
+        a_args = extract_analyzer_args_from_model(plugin, data_model, logger)
+        if a_args:
+            plugins[obj.source]["analysis_args"] = a_args.model_dump(exclude_none=True)
+
     plugin_config.plugins = plugins
 
     return plugin_config
@@ -422,3 +426,91 @@ def find_datamodel_and_result(base_path: str) -> list[Tuple[str, str]]:
                 tuple_list.append((datamodel_path, result_path))
 
     return tuple_list
+
+
+def dump_results_to_csv(
+    results: list[PluginResult],
+    nodename: str,
+    log_path: str,
+    timestamp: str,
+    logger: logging.Logger,
+):
+    """dump node-scraper summary results to csv file
+
+    Args:
+        results (list[PluginResult]): list of PluginResults
+        nodename (str): node where results come from
+        log_path (str): path to results
+        timestamp (str): time when results were taken
+        logger (logging.Logger): instance of logger
+    """
+    fieldnames = ["nodename", "plugin", "status", "timestamp", "message"]
+    filename = log_path + "/nodescraper.csv"
+    all_rows = []
+    for res in results:
+        row = {
+            "nodename": nodename,
+            "plugin": res.source,
+            "status": res.status.name,
+            "timestamp": timestamp,
+            "message": res.message,
+        }
+        all_rows.append(row)
+    dump_to_csv(all_rows, filename, fieldnames, logger)
+
+
+def dump_to_csv(all_rows: list, filename: str, fieldnames: list[str], logger: logging.Logger):
+    """dump data to csv
+
+    Args:
+        all_rows (list): rows to be written
+        filename (str): name of file to write to
+        fieldnames (list[str]): header for csv file
+        logger (logging.Logger): isntance of logger
+    """
+    try:
+        with open(filename, "w", newline="") as f:
+            writer = csv.DictWriter(f, fieldnames=fieldnames)
+            writer.writeheader()
+            for row in all_rows:
+                writer.writerow(row)
+    except Exception as exp:
+        logger.error("Could not dump data to csv file: %s", exp)
+    logger.info("Data written to csv file: %s", filename)
+
+
+def generate_summary(search_path: str, output_path: str | None, logger: logging.Logger):
+    """Concatenate csv files into 1 summary csv file
+
+    Args:
+        search_path (str): Path for previous runs
+        output_path (str | None): Path for new summary csv file
+        logger (logging.Logger): instance of logger
+    """
+
+    fieldnames = ["nodename", "plugin", "status", "timestamp", "message"]
+    all_rows = []
+
+    pattern = os.path.join(search_path, "**", "nodescraper.csv")
+    matched_files = glob.glob(pattern, recursive=True)
+
+    if not matched_files:
+        logger.error(f"No nodescraper.csv files found under {search_path}")
+        return
+
+    for filepath in matched_files:
+        logger.info(f"Reading: {filepath}")
+        with open(filepath, newline="") as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                all_rows.append(row)
+
+    if not all_rows:
+        logger.error("No data rows found in matched CSV files.")
+        return
+
+    if not output_path:
+        output_path = os.getcwd()
+
+    output_path = os.path.join(output_path, "summary.csv")
+    dump_to_csv(all_rows, output_path, fieldnames, logger)

nodescraper/connection/inband/__init__.py

@@ -23,7 +23,13 @@
 # SOFTWARE.
 #
 ###############################################################################
-from .inband import CommandArtifact, FileArtifact, InBandConnection
+from .inband import (
+    BaseFileArtifact,
+    BinaryFileArtifact,
+    CommandArtifact,
+    InBandConnection,
+    TextFileArtifact,
+)
 from .inbandlocal import LocalShell
 from .inbandmanager import InBandConnectionManager
 from .sshparams import SSHConnectionParams
@@ -33,6 +39,8 @@
     "LocalShell",
     "InBandConnectionManager",
     "InBandConnection",
-    "FileArtifact",
+    "BaseFileArtifact",
+    "TextFileArtifact",
+    "BinaryFileArtifact",
     "CommandArtifact",
 ]