Merge branch 'development' into alex_devenum

Author: alexandraBara

CONTRIBUITING.md CONTRIBUTING.mdCONTRIBUITING.md renamed to CONTRIBUTING.md

@@ -131,5 +131,4 @@ if __name__ == "__main__":
 ```
 
 ## external plugins
-External plugins can be added and installed in the same env as node-scraper plugins. Find an
-example of an external plugin in **`/docs/node-scraper-external`**
+External plugins can be added and installed in the same env as node-scraper plugins. See -> [docs/node-scraper-external/README.md](docs/node-scraper-external/README.md)

EXTENDING.md

@@ -19,10 +19,12 @@ system debug.
   - [Plugin config: `--plugin-configs` command](#plugin-config---plugin-configs-command)
   - [Reference config: `gen-reference-config` command](#reference-config-gen-reference-config-command)
 - **Extending Node Scraper (integration & external plugins)** → See [EXTENDING.md](EXTENDING.md)
+- **Full view of the plugins with the associated collectors & analyzers as well as the commands
+invoked by collectors** -> See [docs/PLUGIN_DOC.md](docs/PLUGIN_DOC.md)
 
 ## Installation
 ### Install From Source
-Node Scraper requires Python 3.10+ for installation. After cloning this repository,
+Node Scraper requires Python 3.9+ for installation. After cloning this repository,
 call dev-setup.sh script with 'source'. This script creates an editable install of Node Scraper in
 a python virtual environment and also configures the pre-commit hooks for the project.
 

README.md

@@ -36,9 +36,9 @@
 import pkgutil
 import sys
 from pathlib import Path
-from typing import Any, Iterable, List, Type
+from typing import Any, Iterable, List, Optional, Type
 
-LINK_BASE_DEFAULT = "../"
+LINK_BASE_DEFAULT = "https://github.com/amd/node-scraper/blob/HEAD/"
 REL_ROOT_DEFAULT = "nodescraper/plugins/inband"
 DEFAULT_ROOT_PACKAGE = "nodescraper.plugins"
 
@@ -50,7 +50,7 @@ def get_attr(obj: Any, name: str, default: Any = None) -> Any:
         return default
 
 
-def _slice_from_rel_root(p: Path, rel_root: str | None) -> str | None:
+def _slice_from_rel_root(p: Path, rel_root: Optional[str]) -> Optional[str]:
     if not rel_root:
         return None
     parts = list(p.parts)
@@ -63,7 +63,7 @@ def _slice_from_rel_root(p: Path, rel_root: str | None) -> str | None:
     return None
 
 
-def setup_link(class_data, link_base: str, rel_root: str | None) -> str:
+def setup_link(class_data, link_base: str, rel_root: Optional[str]) -> str:
     try:
         file_location = Path(inspect.getfile(class_data)).resolve()
     except Exception:
@@ -80,7 +80,7 @@ def setup_link(class_data, link_base: str, rel_root: str | None) -> str:
     return base + rel_path
 
 
-def get_own_doc(cls: type) -> str | None:
+def get_own_doc(cls: type) -> Optional[str]:
     """
     Return only the __doc__ defined in the class itself, ignore inheritance.
     """
@@ -224,6 +224,57 @@ def add_cmd(s: Any):
     return cmds
 
 
+def extract_regexes_and_args_from_analyzer(
+    analyzer_cls: type, args_cls: Optional[type]
+) -> List[str]:
+    """Extract regex patterns and analyzer args from analyzer class"""
+    if not inspect.isclass(analyzer_cls):
+        return []
+
+    output: List[str] = []
+
+    # Check for ERROR_REGEX class variable (used by RegexAnalyzer subclasses like DmesgAnalyzer)
+    error_regex = get_attr(analyzer_cls, "ERROR_REGEX", None)
+    if error_regex and isinstance(error_regex, list):
+        output.append("**Built-in Regexes:**")
+        for item in error_regex:
+            # ErrorRegex objects have regex, message, event_category attributes
+            if hasattr(item, "regex"):
+                pattern = getattr(item.regex, "pattern", None)
+                message = getattr(item, "message", "")
+                if pattern:
+                    # Truncate long patterns
+                    pattern_str = pattern if len(pattern) < 50 else pattern[:47] + "..."
+                    output.append(f"- {message}: `{pattern_str}`")
+            elif hasattr(item, "pattern"):
+                pattern_str = item.pattern if len(item.pattern) < 50 else item.pattern[:47] + "..."
+                output.append(f"- `{pattern_str}`")
+
+    # Check for other regex-related attributes
+    for attr in dir(analyzer_cls):
+        if "REGEX" in attr.upper() and not attr.startswith("_"):
+            val = get_attr(analyzer_cls, attr, default=None)
+            if val is None or attr == "ERROR_REGEX":
+                continue
+
+            if hasattr(val, "pattern"):
+                output.append(f"**{attr}**: `{val.pattern}`")
+            elif isinstance(val, str):
+                output.append(f"**{attr}**: `{val}`")
+
+    # Extract analyzer args if provided
+    if inspect.isclass(args_cls):
+        anns = get_attr(args_cls, "__annotations__", {}) or {}
+        if anns:
+            output.append("**Analyzer Args:**")
+            for key, value in anns.items():
+                # Format the type annotation
+                type_str = str(value).replace("typing.", "")
+                output.append(f"- `{key}`: {type_str}")
+
+    return output
+
+
 def md_header(text: str, level: int = 2) -> str:
     return f"{'#' * level} {text}\n\n"
 
@@ -257,7 +308,20 @@ def class_vars_dump(cls: type, exclude: set) -> List[str]:
             continue
         if callable(val) or isinstance(val, (staticmethod, classmethod, property)):
             continue
-        out.append(f"**{name}**: `{val}`")
+
+        # Format list values with each item on a new line
+        if isinstance(val, list) and len(val) > 0:
+            val_str = str(val)
+            if len(val_str) > 200:
+                formatted_items = []
+                for item in val:
+                    formatted_items.append(f"  {item}")
+                formatted_list = "[\n" + ",\n".join(formatted_items) + "\n]"
+                out.append(f"**{name}**: `{formatted_list}`")
+            else:
+                out.append(f"**{name}**: `{val}`")
+        else:
+            out.append(f"**{name}**: `{val}`")
     return out
 
 
@@ -279,14 +343,20 @@ def generate_plugin_table_rows(plugins: List[type]) -> List[List[str]]:
                     seen.add(key)
                     uniq.append(c)
             cmds = uniq
+
+        # Extract regexes and args from analyzer
+        regex_and_args = []
+        if inspect.isclass(an):
+            regex_and_args = extract_regexes_and_args_from_analyzer(an, args)
+
         rows.append(
             [
-                f"{p.__module__}.{p.__name__}",
+                p.__name__,
+                "<br>".join(cmds).replace("|", "\\|") if cmds else "-",
+                "<br>".join(regex_and_args).replace("|", "\\|") if regex_and_args else "-",
                 link_anchor(dm, "model") if inspect.isclass(dm) else "-",
                 link_anchor(col, "collector") if inspect.isclass(col) else "-",
                 link_anchor(an, "analyzer") if inspect.isclass(an) else "-",
-                link_anchor(args, "args") if inspect.isclass(args) else "-",
-                "<br>".join(cmds) if cmds else "-",
             ]
         )
     return rows
@@ -302,14 +372,15 @@ def render_table(headers: List[str], rows: List[List[str]]) -> str:
     return "".join(out)
 
 
-def render_collector_section(col: type, link_base: str, rel_root: str | None) -> str:
+def render_collector_section(col: type, link_base: str, rel_root: Optional[str]) -> str:
     hdr = md_header(f"Collector Class {col.__name__}", 2)
     desc = sanitize_doc(get_own_doc(col) or "")
     s = hdr
     if desc:
         s += md_header("Description", 3) + desc + "\n\n"
     s += md_kv("Bases", str(bases_list(col)))
-    s += md_kv("Link to code", setup_link(col, link_base, rel_root))
+    _url = setup_link(col, link_base, rel_root)
+    s += md_kv("Link to code", f"[{Path(_url).name}]({_url})")
 
     exclude = {"__doc__", "__module__", "__weakref__", "__dict__"}
     cv = class_vars_dump(col, exclude)
@@ -334,60 +405,56 @@ def render_collector_section(col: type, link_base: str, rel_root: str | None) ->
     return s
 
 
-def render_analyzer_section(an: type, link_base: str, rel_root: str | None) -> str:
+def render_analyzer_section(an: type, link_base: str, rel_root: Optional[str]) -> str:
     hdr = md_header(f"Data Analyzer Class {an.__name__}", 2)
     desc = sanitize_doc(get_own_doc(an) or "")
     s = hdr
     if desc:
         s += md_header("Description", 3) + desc + "\n\n"
     s += md_kv("Bases", str(bases_list(an)))
-    s += md_kv("Link to code", setup_link(an, link_base, rel_root))
+    _url = setup_link(an, link_base, rel_root)
+    s += md_kv("Link to code", f"[{Path(_url).name}]({_url})")
 
     exclude = {"__doc__", "__module__", "__weakref__", "__dict__"}
     cv = class_vars_dump(an, exclude)
     if cv:
         s += md_header("Class Variables", 3) + md_list(cv)
 
-    req = get_attr(an, "REQUIRED_DATA", None)
-    s += md_header("Required Data", 3)
-    if req:
-        if isinstance(req, (list, tuple)):
-            s += ", ".join(getattr(m, "__name__", str(m)) for m in req) + "\n\n"
-        else:
-            s += getattr(req, "__name__", str(req)) + "\n\n"
-    else:
-        s += "-\n\n"
+    # Add regex patterns if present (pass None for args_cls since we don't have context here)
+    regex_info = extract_regexes_and_args_from_analyzer(an, None)
+    if regex_info:
+        s += md_header("Regex Patterns", 3)
+        if len(regex_info) > 10:
+            s += f"*{len(regex_info)} items defined*\n\n"
+        s += md_list(regex_info)
 
     return s
 
 
-def render_model_section(model: type, link_base: str, rel_root: str | None) -> str:
+def render_model_section(model: type, link_base: str, rel_root: Optional[str]) -> str:
     hdr = md_header(f"{model.__name__} Model", 2)
     desc = sanitize_doc(get_own_doc(model) or "")
     s = hdr
     if desc:
         s += md_header("Description", 3) + desc + "\n\n"
-    s += md_kv("Link to code", setup_link(model, link_base, rel_root))
+    _url = setup_link(model, link_base, rel_root)
+    s += md_kv("Link to code", f"[{Path(_url).name}]({_url})")
     s += md_kv("Bases", str(bases_list(model)))
     anns = annotations_for_model(model)
     if anns:
         s += md_header("Model annotations and fields", 3) + md_list(anns)
     return s
 
 
-def render_analyzer_args_section(args_cls: type, link_base: str, rel_root: str | None) -> str:
+def render_analyzer_args_section(args_cls: type, link_base: str, rel_root: Optional[str]) -> str:
     hdr = md_header(f"Analyzer Args Class {args_cls.__name__}", 2)
     desc = sanitize_doc(get_own_doc(args_cls) or "")
     s = hdr
     if desc:
         s += md_header("Description", 3) + desc + "\n\n"
     s += md_kv("Bases", str(bases_list(args_cls)))
-    s += md_kv("Link to code", setup_link(args_cls, link_base, rel_root))
-
-    exclude = {"__doc__", "__module__", "__weakref__", "__dict__"}
-    cv = class_vars_dump(args_cls, exclude)
-    if cv:
-        s += md_header("Class Variables", 3) + md_list(cv)
+    _url = setup_link(args_cls, link_base, rel_root)
+    s += md_kv("Link to code", f"[{Path(_url).name}]({_url})")
 
     anns = get_attr(args_cls, "__annotations__", {}) or {}
     if anns:
@@ -429,7 +496,14 @@ def all_subclasses(cls: Type) -> set[type]:
     plugins.sort(key=lambda c: f"{c.__module__}.{c.__name__}".lower())
 
     rows = generate_plugin_table_rows(plugins)
-    headers = ["Plugin", "DataModel", "Collector", "Analyzer", "AnalyzerArgs", "Cmd(s)"]
+    headers = [
+        "Plugin",
+        "Collection",
+        "Analysis",
+        "DataModel",
+        "Collector",
+        "Analyzer",
+    ]
 
     collectors, analyzers, models, args_classes = [], [], [], []
     seen_c, seen_a, seen_m, seen_args = set(), set(), set(), set()

docs/PLUGIN_DOC.md

@@ -24,7 +24,7 @@
 #
 ###############################################################################
 import logging
-from typing import Generic, Optional
+from typing import Generic, Optional, Union
 
 from nodescraper.connection.inband import InBandConnection
 from nodescraper.connection.inband.inband import BaseFileArtifact, CommandArtifact
@@ -49,7 +49,7 @@ def __init__(
         connection: InBandConnection,
         logger: Optional[logging.Logger] = None,
         system_interaction_level: SystemInteractionLevel = SystemInteractionLevel.INTERACTIVE,
-        max_event_priority_level: EventPriority | str = EventPriority.CRITICAL,
+        max_event_priority_level: Union[EventPriority, str] = EventPriority.CRITICAL,
         parent: Optional[str] = None,
         task_result_hooks: Optional[list[TaskResultHook]] = None,
         **kwargs,

docs/generate_plugin_doc_bundle.py

@@ -24,6 +24,7 @@
 #
 ###############################################################################
 import re
+from typing import Union
 
 from pydantic import BaseModel
 
@@ -36,7 +37,7 @@
 class ErrorRegex(BaseModel):
     regex: re.Pattern
     message: str
-    event_category: str | EventCategory = EventCategory.UNKNOWN
+    event_category: Union[str, EventCategory] = EventCategory.UNKNOWN
     event_priority: EventPriority = EventPriority.ERROR
 
 
@@ -54,14 +55,15 @@ class RegexAnalyzer(DataAnalyzer[TDataModel, TAnalyzeArg]):
     """Parent class for all regex based data analyzers."""
 
     def _build_regex_event(
-        self, regex_obj: ErrorRegex, match: str | list[str], source: str
+        self, regex_obj: ErrorRegex, match: Union[str, list[str]], source: str
     ) -> RegexEvent:
         """Build a RegexEvent object from a regex match and source.
 
         Args:
             regex_obj (ErrorRegex): regex object containing the regex pattern, message, category, and priorit
-            match (str | list[str]): matched content from the regex
-            source (str): descriptor for the content where the match was found
+            match (
+        Union[str, list[str]]): matched content from the regex
+                    source (str): descriptor for the content where the match was found
 
         Returns:
             RegexEvent: an instance of RegexEvent containing the match details

nodescraper/base/inbandcollectortask.py

@@ -264,17 +264,18 @@ def build_parser(
             model_type_map = parser_builder.build_plugin_parser()
         except Exception as e:
             print(f"Exception building arg parsers for {plugin_name}: {str(e)}")  # noqa: T201
+            continue
         plugin_subparser_map[plugin_name] = (plugin_subparser, model_type_map)
 
     return parser, plugin_subparser_map
 
 
-def setup_logger(log_level: str = "INFO", log_path: str | None = None) -> logging.Logger:
+def setup_logger(log_level: str = "INFO", log_path: Optional[str] = None) -> logging.Logger:
     """set up root logger when using the CLI
 
     Args:
         log_level (str): log level to use
-        log_path (str | None): optional path to filesystem log location
+        log_path (Optional[str]): optional path to filesystem log location
 
     Returns:
         logging.Logger: logger intstance

nodescraper/base/regexanalyzer.py

@@ -24,8 +24,7 @@
 #
 ###############################################################################
 import argparse
-import types
-from typing import Type
+from typing import Optional, Type
 
 from pydantic import BaseModel
 
@@ -59,7 +58,7 @@ def build_plugin_parser(self) -> dict:
             }
 
             # skip args where generic type has been set to None
-            if types.NoneType in type_class_map:
+            if type(None) in type_class_map:
                 continue
 
             model_arg = self.get_model_arg(type_class_map)
@@ -75,14 +74,14 @@ def build_plugin_parser(self) -> dict:
         return model_type_map
 
     @classmethod
-    def get_model_arg(cls, type_class_map: dict) -> Type[BaseModel] | None:
+    def get_model_arg(cls, type_class_map: dict) -> Optional[Type[BaseModel]]:
         """Get the first type which is a pydantic model from a type class map
 
         Args:
             type_class_map (dict): mapping of type classes
 
         Returns:
-            Type[BaseModel] | None: pydantic model type
+            Optional[Type[BaseModel]]: pydantic model type
         """
         return next(
             (
@@ -164,7 +163,7 @@ def build_model_arg_parser(self, model: type[BaseModel], required: bool) -> list
                 type_class.type_class: type_class for type_class in attr_data.type_classes
             }
 
-            if types.NoneType in type_class_map and len(attr_data.type_classes) == 1:
+            if type(None) in type_class_map and len(attr_data.type_classes) == 1:
                 continue
 
             self.add_argument(type_class_map, attr.replace("_", "-"), required)