Merge branch 'development' into alex_amdsmi_utests

Author: alexandraBara

.github/workflows/code_quality_checks.yml

@@ -4,7 +4,9 @@ name: Code Quality Check
 permissions:
   contents: read
 
-on: [pull_request]
+on:
+  - pull_request
+  - workflow_dispatch
 
 jobs:
   pre-commit:
@@ -13,11 +15,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - name: setup environment
-        run: |
-          ./dev-setup.sh
-      - name: run pre-commit hooks
+      - name: setup environment and run pre-commit hooks
+        shell: bash
         run: |
+          source ./dev-setup.sh
           pre-commit run --all-files --show-diff-on-failure --color=always
       - name: Print message on failure
         if: failure()

.github/workflows/unit-test.yml

@@ -17,19 +17,16 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Install package
-      run: |
-        ./dev-setup.sh
-
     - name: Install xmllint
       run: |
         apt-get update
         apt-get install -y libxml2-utils bc
 
-
-    - name: Run unit tests with coverage
+    - name: Install package and run unit tests with coverage
       id: extract_coverage
+      shell: bash
       run: |
+        source ./dev-setup.sh
         pytest test/unit -s --cov=nodescraper --cov-report=xml --cov-report=term --cov-fail-under=70 --maxfail=1 --disable-warnings -v
 
     - name: Print coverage

README.md

@@ -32,6 +32,27 @@ a python virtual environment and also configures the pre-commit hooks for the pr
 source dev-setup.sh
 ```
 
+Alternatively, follow these manual steps:
+
+### 1. Virtual Environment (Optional)
+```sh
+python3 -m venv venv
+source venv/bin/activate
+```
+On Debian/Ubuntu, you may need: `sudo apt install python3-venv`
+
+### 2. Install from Source (Required)
+```sh
+python3 -m pip install --editable .[dev] --upgrade
+```
+This installs Node Scraper in editable mode with development dependencies. To verify: `node-scraper --help`
+
+### 3. Git Hooks (Optional)
+```sh
+pre-commit install
+```
+Sets up pre-commit hooks for code quality checks. On Debian/Ubuntu, you may need: `sudo apt install pre-commit`
+
 ## CLI Usage
 The Node Scraper CLI can be used to run Node Scraper plugins on a target system. The following CLI
 options are available:

dev-setup.sh

@@ -1,6 +1,7 @@
+#!/usr/bin/env bash
+
 # Create venv if not already present
 if [ ! -d "venv" ]; then
-    python3 -m pip install venv
     python3 -m venv venv
 fi
 

docs/PLUGIN_DOC.md

@@ -36,7 +36,7 @@
 import pkgutil
 import sys
 from pathlib import Path
-from typing import Any, Iterable, List, Type
+from typing import Any, Iterable, List, Optional, Type
 
 LINK_BASE_DEFAULT = "https://github.com/amd/node-scraper/blob/HEAD/"
 REL_ROOT_DEFAULT = "nodescraper/plugins/inband"
@@ -50,7 +50,7 @@ def get_attr(obj: Any, name: str, default: Any = None) -> Any:
         return default
 
 
-def _slice_from_rel_root(p: Path, rel_root: str | None) -> str | None:
+def _slice_from_rel_root(p: Path, rel_root: Optional[str]) -> Optional[str]:
     if not rel_root:
         return None
     parts = list(p.parts)
@@ -63,7 +63,7 @@ def _slice_from_rel_root(p: Path, rel_root: str | None) -> str | None:
     return None
 
 
-def setup_link(class_data, link_base: str, rel_root: str | None) -> str:
+def setup_link(class_data, link_base: str, rel_root: Optional[str]) -> str:
     try:
         file_location = Path(inspect.getfile(class_data)).resolve()
     except Exception:
@@ -80,7 +80,7 @@ def setup_link(class_data, link_base: str, rel_root: str | None) -> str:
     return base + rel_path
 
 
-def get_own_doc(cls: type) -> str | None:
+def get_own_doc(cls: type) -> Optional[str]:
     """
     Return only the __doc__ defined in the class itself, ignore inheritance.
     """
@@ -224,6 +224,57 @@ def add_cmd(s: Any):
     return cmds
 
 
+def extract_regexes_and_args_from_analyzer(
+    analyzer_cls: type, args_cls: Optional[type]
+) -> List[str]:
+    """Extract regex patterns and analyzer args from analyzer class"""
+    if not inspect.isclass(analyzer_cls):
+        return []
+
+    output: List[str] = []
+
+    # Check for ERROR_REGEX class variable (used by RegexAnalyzer subclasses like DmesgAnalyzer)
+    error_regex = get_attr(analyzer_cls, "ERROR_REGEX", None)
+    if error_regex and isinstance(error_regex, list):
+        output.append("**Built-in Regexes:**")
+        for item in error_regex:
+            # ErrorRegex objects have regex, message, event_category attributes
+            if hasattr(item, "regex"):
+                pattern = getattr(item.regex, "pattern", None)
+                message = getattr(item, "message", "")
+                if pattern:
+                    # Truncate long patterns
+                    pattern_str = pattern if len(pattern) < 50 else pattern[:47] + "..."
+                    output.append(f"- {message}: `{pattern_str}`")
+            elif hasattr(item, "pattern"):
+                pattern_str = item.pattern if len(item.pattern) < 50 else item.pattern[:47] + "..."
+                output.append(f"- `{pattern_str}`")
+
+    # Check for other regex-related attributes
+    for attr in dir(analyzer_cls):
+        if "REGEX" in attr.upper() and not attr.startswith("_"):
+            val = get_attr(analyzer_cls, attr, default=None)
+            if val is None or attr == "ERROR_REGEX":
+                continue
+
+            if hasattr(val, "pattern"):
+                output.append(f"**{attr}**: `{val.pattern}`")
+            elif isinstance(val, str):
+                output.append(f"**{attr}**: `{val}`")
+
+    # Extract analyzer args if provided
+    if inspect.isclass(args_cls):
+        anns = get_attr(args_cls, "__annotations__", {}) or {}
+        if anns:
+            output.append("**Analyzer Args:**")
+            for key, value in anns.items():
+                # Format the type annotation
+                type_str = str(value).replace("typing.", "")
+                output.append(f"- `{key}`: {type_str}")
+
+    return output
+
+
 def md_header(text: str, level: int = 2) -> str:
     return f"{'#' * level} {text}\n\n"
 
@@ -257,7 +308,20 @@ def class_vars_dump(cls: type, exclude: set) -> List[str]:
             continue
         if callable(val) or isinstance(val, (staticmethod, classmethod, property)):
             continue
-        out.append(f"**{name}**: `{val}`")
+
+        # Format list values with each item on a new line
+        if isinstance(val, list) and len(val) > 0:
+            val_str = str(val)
+            if len(val_str) > 200:
+                formatted_items = []
+                for item in val:
+                    formatted_items.append(f"  {item}")
+                formatted_list = "[\n" + ",\n".join(formatted_items) + "\n]"
+                out.append(f"**{name}**: `{formatted_list}`")
+            else:
+                out.append(f"**{name}**: `{val}`")
+        else:
+            out.append(f"**{name}**: `{val}`")
     return out
 
 
@@ -279,14 +343,20 @@ def generate_plugin_table_rows(plugins: List[type]) -> List[List[str]]:
                     seen.add(key)
                     uniq.append(c)
             cmds = uniq
+
+        # Extract regexes and args from analyzer
+        regex_and_args = []
+        if inspect.isclass(an):
+            regex_and_args = extract_regexes_and_args_from_analyzer(an, args)
+
         rows.append(
             [
-                f"{p.__module__}.{p.__name__}",
+                p.__name__,
+                "<br>".join(cmds).replace("|", "\\|") if cmds else "-",
+                "<br>".join(regex_and_args).replace("|", "\\|") if regex_and_args else "-",
                 link_anchor(dm, "model") if inspect.isclass(dm) else "-",
                 link_anchor(col, "collector") if inspect.isclass(col) else "-",
                 link_anchor(an, "analyzer") if inspect.isclass(an) else "-",
-                link_anchor(args, "args") if inspect.isclass(args) else "-",
-                "<br>".join(cmds) if cmds else "-",
             ]
         )
     return rows
@@ -302,7 +372,7 @@ def render_table(headers: List[str], rows: List[List[str]]) -> str:
     return "".join(out)
 
 
-def render_collector_section(col: type, link_base: str, rel_root: str | None) -> str:
+def render_collector_section(col: type, link_base: str, rel_root: Optional[str]) -> str:
     hdr = md_header(f"Collector Class {col.__name__}", 2)
     desc = sanitize_doc(get_own_doc(col) or "")
     s = hdr
@@ -335,7 +405,7 @@ def render_collector_section(col: type, link_base: str, rel_root: str | None) ->
     return s
 
 
-def render_analyzer_section(an: type, link_base: str, rel_root: str | None) -> str:
+def render_analyzer_section(an: type, link_base: str, rel_root: Optional[str]) -> str:
     hdr = md_header(f"Data Analyzer Class {an.__name__}", 2)
     desc = sanitize_doc(get_own_doc(an) or "")
     s = hdr
@@ -350,10 +420,18 @@ def render_analyzer_section(an: type, link_base: str, rel_root: str | None) -> s
     if cv:
         s += md_header("Class Variables", 3) + md_list(cv)
 
+    # Add regex patterns if present (pass None for args_cls since we don't have context here)
+    regex_info = extract_regexes_and_args_from_analyzer(an, None)
+    if regex_info:
+        s += md_header("Regex Patterns", 3)
+        if len(regex_info) > 10:
+            s += f"*{len(regex_info)} items defined*\n\n"
+        s += md_list(regex_info)
+
     return s
 
 
-def render_model_section(model: type, link_base: str, rel_root: str | None) -> str:
+def render_model_section(model: type, link_base: str, rel_root: Optional[str]) -> str:
     hdr = md_header(f"{model.__name__} Model", 2)
     desc = sanitize_doc(get_own_doc(model) or "")
     s = hdr
@@ -368,7 +446,7 @@ def render_model_section(model: type, link_base: str, rel_root: str | None) -> s
     return s
 
 
-def render_analyzer_args_section(args_cls: type, link_base: str, rel_root: str | None) -> str:
+def render_analyzer_args_section(args_cls: type, link_base: str, rel_root: Optional[str]) -> str:
     hdr = md_header(f"Analyzer Args Class {args_cls.__name__}", 2)
     desc = sanitize_doc(get_own_doc(args_cls) or "")
     s = hdr
@@ -418,7 +496,14 @@ def all_subclasses(cls: Type) -> set[type]:
     plugins.sort(key=lambda c: f"{c.__module__}.{c.__name__}".lower())
 
     rows = generate_plugin_table_rows(plugins)
-    headers = ["Plugin", "DataModel", "Collector", "Analyzer", "AnalyzerArgs", "Cmd(s)"]
+    headers = [
+        "Plugin",
+        "Collection",
+        "Analysis",
+        "DataModel",
+        "Collector",
+        "Analyzer",
+    ]
 
     collectors, analyzers, models, args_classes = [], [], [], []
     seen_c, seen_a, seen_m, seen_args = set(), set(), set(), set()

docs/generate_plugin_doc_bundle.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from nodescraper.base import InBandDataCollector
 from nodescraper.enums import ExecutionStatus
 from nodescraper.models import TaskResult
@@ -9,7 +11,7 @@ class SampleCollector(InBandDataCollector[SampleDataModel, None]):
 
     DATA_MODEL = SampleDataModel
 
-    def collect_data(self, args=None) -> tuple[TaskResult, SampleDataModel | None]:
+    def collect_data(self, args=None) -> tuple[TaskResult, Optional[SampleDataModel]]:
         sample_data = SampleDataModel(some_str="example123")
         self.result.message = "Collector ran successfully"
         self.result.status = ExecutionStatus.OK

docs/node-scraper-external/ext_nodescraper_plugins/sample/sample_collector.py

@@ -23,12 +23,61 @@
 # SOFTWARE.
 #
 ###############################################################################
-from pydantic import BaseModel
+from typing import Any
+
+from pydantic import BaseModel, model_validator
 
 
 class AnalyzerArgs(BaseModel):
+    """Base class for all analyzer arguments.
+
+    This class provides automatic string stripping for all string values
+    in analyzer args. All analyzer args classes should inherit from this
+    directly.
+
+    """
+
     model_config = {"extra": "forbid", "exclude_none": True}
 
+    @model_validator(mode="before")
+    @classmethod
+    def strip_string_values(cls, data: Any) -> Any:
+        """Strip whitespace from all string values in analyzer args.
+
+        This validator recursively processes:
+        - String values: strips whitespace
+        - Lists: strips strings in lists
+        - Dicts: strips string values in dicts
+        - Other types: left unchanged
+
+        Args:
+            data: The input data to validate
+
+        Returns:
+            The data with all string values stripped
+        """
+        if isinstance(data, dict):
+            return {k: cls._strip_value(v) for k, v in data.items()}
+        return data
+
+    @classmethod
+    def _strip_value(cls, value: Any) -> Any:
+        """Recursively strip string values.
+
+        Args:
+            value: The value to process
+
+        Returns:
+            The processed value
+        """
+        if isinstance(value, str):
+            return value.strip()
+        elif isinstance(value, list):
+            return [cls._strip_value(item) for item in value]
+        elif isinstance(value, dict):
+            return {k: cls._strip_value(v) for k, v in value.items()}
+        return value
+
     @classmethod
     def build_from_model(cls, datamodel):
         """Build analyzer args instance from data model object