Merge pull request #44 from amd/alex_ext_plugins

External plugins example

Author: alexandraBara

EXTENDING.md

@@ -0,0 +1,135 @@
+# Extending Node Scraper
+
+This guide covers how to integrate Nodescraper into another Python tool and how to create and use external plugins.
+
+## Table of Contents
+- [nodescraper integration](#nodescraper-integration)
+- [external plugins](#external-plugins)
+
+## nodescraper integration
+Nodescraper can be integrated inside another Python tool by leveraging its classes and functionality.
+See below for a comprehensive example on how to create plugins and run the associated data
+collection and analysis.
+Sample run command:
+```sh
+python3 sample.py
+```
+
+Sample.py file:
+```python
+import logging
+import sys
+from nodescraper.plugins.inband.bios.bios_plugin import BiosPlugin
+from nodescraper.plugins.inband.bios.analyzer_args import BiosAnalyzerArgs
+from nodescraper.plugins.inband.kernel.kernel_plugin import KernelPlugin
+from nodescraper.plugins.inband.kernel.analyzer_args import KernelAnalyzerArgs
+from nodescraper.plugins.inband.os.os_plugin import OsPlugin
+from nodescraper.plugins.inband.os.analyzer_args import OsAnalyzerArgs
+from nodescraper.models.systeminfo import SystemInfo, OSFamily
+from nodescraper.enums import EventPriority, SystemLocation
+from nodescraper.resultcollators.tablesummary import TableSummary
+from nodescraper.connection.inband.inbandmanager import InBandConnectionManager
+from nodescraper.connection.inband.sshparams import SSHConnectionParams
+from nodescraper.pluginregistry import PluginRegistry
+from nodescraper.models.pluginconfig import PluginConfig
+from nodescraper.pluginexecutor import PluginExecutor
+
+def main():
+
+    #setting up my custom logger
+    log_level = "INFO"
+    handlers = [logging.StreamHandler(stream=sys.stdout)]
+    logging.basicConfig(
+        force=True,
+        level=log_level,
+        format="%(asctime)25s %(levelname)10s %(name)25s | %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S %Z",
+        handlers=handlers,
+        encoding="utf-8",
+    )
+    logging.root.setLevel(logging.INFO)
+    logging.getLogger("paramiko").setLevel(logging.ERROR)
+    logger = logging.getLogger("nodescraper")
+
+    #setting up system info
+    system_info = SystemInfo(name="test_host",
+                            platform="X",
+                            os_familty=OSFamily.LINUX,
+                            sku="some_sku")
+
+    #initiate plugins
+    bios_plugin = BiosPlugin(system_info=system_info, logger=logger)
+    kernel_plugin = KernelPlugin(system_info=system_info, logger=logger)
+
+    #launch data collection
+    _ = bios_plugin.collect()
+    _ = kernel_plugin.collect()
+
+    #launch data analysis
+    bios_plugin.analyze(analysis_args=BiosAnalyzerArgs(exp_bios_version="XYZ"))
+    kernel_plugin.analyze(analysis_args=KernelAnalyzerArgs(exp_kernel="ABC"))
+
+    #log plugin data models
+    logger.info(kernel_plugin.data.model_dump())
+    logger.info(bios_plugin.data.model_dump())
+
+    #alternate method
+    all_res = []
+
+    #launch plugin collection & analysis
+    bios_result = bios_plugin.run(analysis_args={"exp_bios_version":"ABC"})
+    all_res.append(bios_result)
+    table_summary = TableSummary()
+    table_summary.collate_results(all_res, None)
+
+    #remote connection
+    system_info.location=SystemLocation.REMOTE
+    ssh_params = SSHConnectionParams(hostname="my_system",
+                                    port=22,
+                                    username="my_username",
+                                    key_filename="/home/user/.ssh/ssh_key")
+    conn_manager = InBandConnectionManager(system_info=system_info, connection_args=ssh_params)
+    os_plugin = OsPlugin(system_info=system_info, logger=logger, connection_manager=conn_manager)
+    os_plugin.run(analysis_args=OsAnalyzerArgs(exp_os="DEF"))
+
+    #run multiple plugins through a queue
+    system_info.location=SystemLocation.LOCAL
+    config_dict = {
+      "global_args": {
+          "collection" : 1,
+          "analysis" : 1
+      },
+      "plugins": {
+        "BiosPlugin": {
+          "analysis_args": {
+            "exp_bios_version": "123",
+          }
+        },
+        "KernelPlugin": {
+          "analysis_args": {
+            "exp_kernel": "ABC",
+          }
+        }
+      },
+      "result_collators": {},
+      "name": "plugin_config",
+      "desc": "Auto generated config"
+      }
+
+    config1 = PluginConfig(**config_dict)
+    plugin_executor = PluginExecutor(
+        logger=logger,
+        plugin_configs=[config1],
+        system_info=system_info
+    )
+    results = plugin_executor.run_queue()
+
+
+
+if __name__ == "__main__":
+    main()
+```
+
+## external plugins
+External plugins can be added and installed in the same env as node-scraper plugins. Find an
+example of an external plugin in **`/docs/node-scraper-external`**

README.md

@@ -18,8 +18,7 @@ system debug.
   - [Global args](#global-args)
   - [Plugin config: `--plugin-configs` command](#plugin-config---plugin-configs-command)
   - [Reference config: `gen-reference-config` command](#reference-config-gen-reference-config-command)
-- [nodescraper integration](#nodescraper-integration)
-
+- **Extending Node Scraper (integration & external plugins)** → See [EXTENDING.md](EXTENDING.md)
 
 ## Installation
 ### Install From Source
@@ -114,7 +113,6 @@ node-scraper --sys-name <remote_host> --sys-location REMOTE --connection-config
 Plugins to run can be specified in two ways, using a plugin JSON config file or using the
 'run-plugins' sub command. These two options are not mutually exclusive and can be used together.
 
-
 #### **'describe' subcommand**
 
 You can use the `describe` subcommand to display details about built-in configs or plugins.
@@ -228,7 +226,6 @@ node-scraper summary --summary_path /<path_to_node-scraper_logs>
 This will generate a new file '/<path_to_node-scraper_logs>/summary.csv' file. This file will
 contain the results from all 'nodescraper.csv' files from '/<path_to_node-scarper_logs>'.
 
-
 ### Configs
 A plugin JSON config should follow the structure of the plugin config model defined here.
 The globals field is a dictionary of global key-value pairs; values in globals will be passed to
@@ -374,128 +371,3 @@ node-scraper gen-plugin-config --gen-reference-config-from-logs scraper_logs_<pa
 ```
 This will generate a reference config that includes plugins with logged results in
 'scraper_log_<path>' and save the new config to 'custom_output_dir/reference_config.json'.
-
-
-## nodescraper integration
-Nodescraper can be integrated inside another Python tool by leveraging its classes and functionality.
-See below for a comprehensive example on how to create plugins and run the associated data
-collection and analysis.
-Sample run command:
-```sh
-python3 sample.py
-```
-
-Sample.py file:
-```python
-import logging
-import sys
-from nodescraper.plugins.inband.bios.bios_plugin import BiosPlugin
-from nodescraper.plugins.inband.bios.analyzer_args import BiosAnalyzerArgs
-from nodescraper.plugins.inband.kernel.kernel_plugin import KernelPlugin
-from nodescraper.plugins.inband.kernel.analyzer_args import KernelAnalyzerArgs
-from nodescraper.plugins.inband.os.os_plugin import OsPlugin
-from nodescraper.plugins.inband.os.analyzer_args import OsAnalyzerArgs
-from nodescraper.models.systeminfo import SystemInfo, OSFamily
-from nodescraper.enums import EventPriority, SystemLocation
-from nodescraper.resultcollators.tablesummary import TableSummary
-from nodescraper.connection.inband.inbandmanager import InBandConnectionManager
-from nodescraper.connection.inband.sshparams import SSHConnectionParams
-from nodescraper.pluginregistry import PluginRegistry
-from nodescraper.models.pluginconfig import PluginConfig
-from nodescraper.pluginexecutor import PluginExecutor
-
-def main():
-
-    #setting up my custom logger
-    log_level = "INFO"
-    handlers = [logging.StreamHandler(stream=sys.stdout)]
-    logging.basicConfig(
-        force=True,
-        level=log_level,
-        format="%(asctime)25s %(levelname)10s %(name)25s | %(message)s",
-        datefmt="%Y-%m-%d %H:%M:%S %Z",
-        handlers=handlers,
-        encoding="utf-8",
-    )
-    logging.root.setLevel(logging.INFO)
-    logging.getLogger("paramiko").setLevel(logging.ERROR)
-    logger = logging.getLogger("nodescraper")
-
-    #setting up system info
-    system_info = SystemInfo(name="test_host",
-                            platform="X",
-                            os_familty=OSFamily.LINUX,
-                            sku="some_sku")
-
-    #initiate plugins
-    bios_plugin = BiosPlugin(system_info=system_info, logger=logger)
-    kernel_plugin = KernelPlugin(system_info=system_info, logger=logger)
-
-    #launch data collection
-    _ = bios_plugin.collect()
-    _ = kernel_plugin.collect()
-
-    #launch data analysis
-    bios_plugin.analyze(analysis_args=BiosAnalyzerArgs(exp_bios_version="XYZ"))
-    kernel_plugin.analyze(analysis_args=KernelAnalyzerArgs(exp_kernel="ABC"))
-
-    #log plugin data models
-    logger.info(kernel_plugin.data.model_dump())
-    logger.info(bios_plugin.data.model_dump())
-
-    #alternate method
-    all_res = []
-
-    #launch plugin collection & analysis
-    bios_result = bios_plugin.run(analysis_args={"exp_bios_version":"ABC"})
-    all_res.append(bios_result)
-    table_summary = TableSummary()
-    table_summary.collate_results(all_res, None)
-
-    #remote connection
-    system_info.location=SystemLocation.REMOTE
-    ssh_params = SSHConnectionParams(hostname="my_system",
-                                    port=22,
-                                    username="my_username",
-                                    key_filename="/home/user/.ssh/ssh_key")
-    conn_manager = InBandConnectionManager(system_info=system_info, connection_args=ssh_params)
-    os_plugin = OsPlugin(system_info=system_info, logger=logger, connection_manager=conn_manager)
-    os_plugin.run(analysis_args=OsAnalyzerArgs(exp_os="DEF"))
-
-    #run multiple plugins through a queue
-    system_info.location=SystemLocation.LOCAL
-    config_dict = {
-      "global_args": {
-          "collection" : 1,
-          "analysis" : 1
-      },
-      "plugins": {
-        "BiosPlugin": {
-          "analysis_args": {
-            "exp_bios_version": "123",
-          }
-        },
-        "KernelPlugin": {
-          "analysis_args": {
-            "exp_kernel": "ABC",
-          }
-        }
-      },
-      "result_collators": {},
-      "name": "plugin_config",
-      "desc": "Auto generated config"
-      }
-
-    config1 = PluginConfig(**config_dict)
-    plugin_executor = PluginExecutor(
-        logger=logger,
-        plugin_configs=[config1],
-        system_info=system_info
-    )
-    results = plugin_executor.run_queue()
-
-
-
-if __name__ == "__main__":
-    main()
-```

docs/node-scraper-external/README.md

@@ -0,0 +1,55 @@
+# node-scraper external plugins (example)
+
+This directory lives at **`/docs/node-scraper-external`** in the `node-scraper` repo and contains
+an example external plugin package you can install in editable mode.
+
+## Installation
+
+Use the same Python environment as `node-scraper`.
+
+```bash
+cd ~/node-scraper
+source venv/bin/activate
+pip install -e ./docs/node-scraper-external
+```
+You should see `ext-nodescraper-plugins` installed in editable mode.
+
+
+## Verify the external package is importable
+
+```bash
+python - <<'PY'
+import ext_nodescraper_plugins
+print("ext_nodescraper_plugins loaded from:", ext_nodescraper_plugins.__file__)
+PY
+```
+
+## Run external plugins
+
+Confirm the CLI sees your external plugin(s):
+
+```bash
+node-scraper run-plugins -h
+node-scraper run-plugins SamplePlugin
+```
+
+## Add your own plugins
+
+Add new modules under the **`ext_nodescraper_plugins/`** package. Example layout:
+
+```
+/docs/node-scraper-external
+├─ pyproject.toml
+└─ ext_nodescraper_plugins/
+   └─ sample/
+      ├─ __init__.py
+      └─ sample_plugin.py
+```
+
+```
+
+Re-install (editable mode picks up code changes automatically, but if you add new files you may
+need to re-run):
+```bash
+pip install -e .
+```

docs/node-scraper-external/ext_nodescraper_plugins/sample/__init__.py

@@ -0,0 +1,25 @@
+from nodescraper.enums import EventCategory, EventPriority, ExecutionStatus
+from nodescraper.interfaces import DataAnalyzer
+from nodescraper.models import TaskResult
+
+from .sample_data import SampleDataModel
+
+
+class SampleAnalyzer(DataAnalyzer[SampleDataModel, None]):
+
+    DATA_MODEL = SampleDataModel
+
+    def analyze_data(self, data: SampleDataModel, args=None) -> TaskResult:
+        if data.some_str != "expected_str":
+            self.result.message = "String does not match expected"
+            self.result.status = ExecutionStatus.ERROR
+            return self.result
+
+            self._log_event(
+                category=EventCategory.OS,
+                description=f"{self.result.message}",
+                data={"expected": "expected_str", "actual": data.some_str},
+                priority=EventPriority.CRITICAL,
+                console_log=True,
+            )
+        return self.result

docs/node-scraper-external/ext_nodescraper_plugins/sample/sample_analyzer.py

@@ -0,0 +1,17 @@
+from nodescraper.base import InBandDataCollector
+from nodescraper.enums import ExecutionStatus
+from nodescraper.models import TaskResult
+
+from .sample_data import SampleDataModel
+
+
+class SampleCollector(InBandDataCollector[SampleDataModel, None]):
+
+    DATA_MODEL = SampleDataModel
+
+    def collect_data(self, args=None) -> tuple[TaskResult, SampleDataModel | None]:
+        sample_data = SampleDataModel(some_str="example123")
+        self.result.message = "Collector ran successfully"
+        self.result.status = ExecutionStatus.OK
+
+        return self.result, sample_data

docs/node-scraper-external/ext_nodescraper_plugins/sample/sample_collector.py

@@ -0,0 +1,5 @@
+from nodescraper.models import DataModel
+
+
+class SampleDataModel(DataModel):
+    some_str: str