chore: merge develop into main (#1853)

Author: kkedziak-splunk

docs/inputs/helper.md

@@ -54,3 +54,20 @@ def stream_events(inputs: smi.InputDefinition, event_writer: smi.EventWriter):
 ```
 
 The two methods' bodies should be filled by the developer.
+
+Alternatively, if you want to have access to the instance of the input script class,
+you can also add the `self` parameter to the methods:
+
+```python
+from splunklib import modularinput as smi
+
+
+def validate_input(self, definition: smi.ValidationDefinition):
+    ...
+
+
+def stream_events(self, inputs: smi.InputDefinition, event_writer: smi.EventWriter):
+    ...
+```
+
+Instead of `self`, you can also use any other name, but it must be the first parameter.

splunk_add_on_ucc_framework/commands/build.py

@@ -13,13 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import ast
 import glob
 import json
 import logging
 import os
 import shutil
 import sys
-from typing import Optional, List, Any
+from pathlib import Path
+from typing import Optional, List, Any, Union
 import subprocess
 import colorama as c
 import fnmatch
@@ -114,7 +116,10 @@ def _modify_and_replace_token_for_oauth_templates(
 
 
 def _add_modular_input(
-    ta_name: str, global_config: global_config_lib.GlobalConfig, outputdir: str
+    ta_name: str,
+    global_config: global_config_lib.GlobalConfig,
+    outputdir: str,
+    gc_path: str,
 ) -> None:
     for service in global_config.inputs:
         input_name = service.get("name")
@@ -132,6 +137,39 @@ def _add_modular_input(
 
         input_helper_module = service.get("inputHelperModule")
 
+        len_of_args = {"stream_events": 2, "validate_input": 1}
+        include_self = {"stream_events": False, "validate_input": False}
+
+        if input_helper_module:
+            addon_root_path = os.path.dirname(gc_path)
+            bin_path = os.path.join(addon_root_path, "package", "bin")
+            helper_path = os.path.join(bin_path, f"{input_helper_module}.py")
+
+            for func in len_of_args:
+                try:
+                    result = _get_num_of_args(func, helper_path)
+
+                    if result is None:
+                        logger.error(
+                            f"Cannot find function definition 'def {func}(...)' in module '{input_helper_module}'."
+                        )
+                        exit(1)
+
+                    if result == len_of_args[func]:
+                        # change nothing
+                        include_self[func] = False
+                    elif result == len_of_args[func] + 1:
+                        # add self to the function
+                        include_self[func] = True
+                    else:
+                        logger.error(
+                            f"Unexpected number of arguments in helper module. "
+                            f"Function '{func}' in '{input_helper_module}' has {result} arguments."
+                        )
+                except FileNotFoundError:
+                    # If no module, that was specified in globalConfig, default one will be created without 'self'
+                    pass
+
         content = (
             utils.get_j2_env()
             .get_template(template)
@@ -141,6 +179,8 @@ def _add_modular_input(
                 description=description,
                 entity=entity,
                 input_helper_module=input_helper_module,
+                stream_events_self=include_self["stream_events"],
+                validate_input_self=include_self["validate_input"],
             )
         )
         input_file_name = os.path.join(outputdir, ta_name, "bin", input_name + ".py")
@@ -164,6 +204,27 @@ def _add_modular_input(
                 helper_file.write(content)
 
 
+def _get_num_of_args(
+    function_name: str,
+    module_path: Union[str, Path],
+) -> Optional[int]:
+    module_path = Path(module_path)
+
+    if not module_path.is_file():
+        raise FileNotFoundError(
+            f"Module path '{module_path}' does not point to a valid file."
+        )
+
+    module_content = ast.parse(module_path.read_text(encoding="utf-8"))
+
+    for node in module_content.body:
+        if isinstance(node, ast.FunctionDef) and node.name == function_name:
+            # Return the number of arguments defined in the function
+            return len(node.args.args)
+
+    return None
+
+
 def _get_ignore_list(
     addon_name: str, ucc_ignore_path: str, output_directory: str
 ) -> List[str]:
@@ -231,14 +292,14 @@ def _get_addon_version(addon_version: Optional[str]) -> str:
                 "repository?). Use `--ta-version` to specify the version you "
                 "want."
             )
-            exit(1)
+            sys.exit(1)
         except exceptions.CouldNotVersionFromGitException:
             logger.error(
                 "Could not find the proper version from git tags. "
                 "Check out "
                 "https://github.com/splunk/addonfactory-ucc-generator/issues/404"
             )
-            exit(1)
+            sys.exit(1)
     return addon_version.strip()
 
 
@@ -546,7 +607,7 @@ def generate(
         )
     if global_config.has_inputs():
         logger.info("Generating inputs code")
-        _add_modular_input(ta_name, global_config, output_directory)
+        _add_modular_input(ta_name, global_config, output_directory, gc_path)
     if global_config.has_alerts():
         logger.info("Generating alerts code")
         alert_builder.generate_alerts(global_config, ta_name, output_directory)

splunk_add_on_ucc_framework/commands/package.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 #
 import logging
+import sys
 import os
 import tarfile
 from typing import Optional
@@ -47,7 +48,7 @@ def package(path_to_built_addon: str, output_directory: Optional[str] = None) ->
             f"Cannot package an add-on without a manifest file. "
             f"Please check the --path provided."
         )
-        exit(1)
+        sys.exit(1)
     with open(app_manifest_path) as _f:
         app_manifest_content = _f.read()
     app_manifest = app_manifest_lib.AppManifest(app_manifest_content)

splunk_add_on_ucc_framework/generators/conf_files/create_account_conf.py

@@ -16,6 +16,7 @@
 from typing import Tuple, List, Dict, Optional
 
 from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator
+from splunk_add_on_ucc_framework.global_config import GlobalConfig
 
 
 class AccountConf(FileGenerator):
@@ -24,13 +25,14 @@ class AccountConf(FileGenerator):
         "file for the configuration mentioned in globalConfig"
     )
 
-    def _set_attributes(self) -> None:
+    def __init__(
+        self, global_config: GlobalConfig, input_dir: str, output_dir: str
+    ) -> None:
+        super().__init__(global_config, input_dir, output_dir)
         self.account_fields: List[Tuple[str, List[str]]] = []
-        if self._global_config.has_configuration():
-            self.conf_spec_file = (
-                self._global_config.namespace.lower() + "_account.conf.spec"
-            )
-            for account in self._global_config.configs:
+        if global_config.has_configuration():
+            self.conf_spec_file = global_config.namespace.lower() + "_account.conf.spec"
+            for account in global_config.configs:
                 # If the endpoint is oauth, which is for getting access_token, conf file entries
                 # should not get created (compatibility to previous versions)
                 if account["name"] == "oauth":

splunk_add_on_ucc_framework/generators/conf_files/create_alert_actions_conf.py

@@ -20,6 +20,7 @@
 
 from splunk_add_on_ucc_framework.commands.modular_alert_builder import normalize
 from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator
+from splunk_add_on_ucc_framework.global_config import GlobalConfig
 
 
 class AlertActionsConf(FileGenerator):
@@ -28,15 +29,18 @@ class AlertActionsConf(FileGenerator):
         "for the custom alert actions defined in globalConfig"
     )
 
-    def _set_attributes(self) -> None:
+    def __init__(
+        self, global_config: GlobalConfig, input_dir: str, output_dir: str
+    ) -> None:
+        super().__init__(global_config, input_dir, output_dir)
         self.conf_file = "alert_actions.conf"
         self.conf_spec_file = f"{self.conf_file}.spec"
-        if self._global_config is None:
+        if global_config is None:
             return
 
         envs = normalize.normalize(
-            self._global_config.alerts,
-            self._global_config.namespace,
+            global_config.alerts,
+            global_config.namespace,
         )
         schema_content = envs["schema.content"]
         self._alert_settings = schema_content["modular_alerts"]

splunk_add_on_ucc_framework/generators/conf_files/create_app_conf.py

@@ -27,13 +27,11 @@ class AppConf(FileGenerator):
     )
 
     def __init__(self, global_config: GlobalConfig, input_dir: str, output_dir: str):
+        super().__init__(global_config, input_dir, output_dir)
         app_manifest = get_app_manifest(input_dir)
         self.description = app_manifest.get_description()
         self.title = app_manifest.get_title()
         self.author = app_manifest.get_authors()[0]["name"]
-        super().__init__(global_config, input_dir, output_dir)
-
-    def _set_attributes(self) -> None:
         self.conf_file = "app.conf"
         self.check_for_updates = "true"
         self.custom_conf = []
@@ -45,16 +43,14 @@ def _set_attributes(self) -> None:
         self.custom_conf.extend(list(self._gc_schema.configs_conf_file_names))
         self.custom_conf.extend(list(self._gc_schema.oauth_conf_file_names))
 
-        if self._global_config.meta.get("checkForUpdates") is False:
+        if global_config.meta.get("checkForUpdates") is False:
             self.check_for_updates = "false"
-        if self._global_config.meta.get("supportedThemes") is not None:
-            self.supported_themes = ", ".join(
-                self._global_config.meta["supportedThemes"]
-            )
+        if global_config.meta.get("supportedThemes") is not None:
+            self.supported_themes = ", ".join(global_config.meta["supportedThemes"])
 
-        self.addon_version = self._global_config.version
+        self.addon_version = global_config.version
         self.is_visible = str(
-            self._global_config.meta.get("isVisible", self._global_config.has_pages())
+            global_config.meta.get("isVisible", global_config.has_pages())
         ).lower()
         self.build = str(int(time()))
 

splunk_add_on_ucc_framework/generators/conf_files/create_commands_conf.py

@@ -16,18 +16,22 @@
 from typing import Dict, List, Optional
 
 from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator
+from splunk_add_on_ucc_framework.global_config import GlobalConfig
 
 
 class CommandsConf(FileGenerator):
     __description__ = (
         "Generates `commands.conf` for custom commands provided in the globalConfig."
     )
 
-    def _set_attributes(self) -> None:
+    def __init__(
+        self, global_config: GlobalConfig, input_dir: str, output_dir: str
+    ) -> None:
+        super().__init__(global_config, input_dir, output_dir)
         self.conf_file = "commands.conf"
-        if self._global_config.has_custom_search_commands():
+        if global_config.has_custom_search_commands():
             self.command_names = []
-            for command in self._global_config.custom_search_commands:
+            for command in global_config.custom_search_commands:
                 self.command_names.append(command["commandName"])
 
     def generate(self) -> Optional[List[Dict[str, str]]]:

splunk_add_on_ucc_framework/generators/conf_files/create_eventtypes_conf.py

@@ -17,6 +17,7 @@
 
 from splunk_add_on_ucc_framework.commands.modular_alert_builder import normalize
 from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator
+from splunk_add_on_ucc_framework.global_config import GlobalConfig
 
 
 class EventtypesConf(FileGenerator):
@@ -25,12 +26,15 @@ class EventtypesConf(FileGenerator):
         " in Adaptive Response of custom alert action in globalConfig"
     )
 
-    def _set_attributes(self) -> None:
+    def __init__(
+        self, global_config: GlobalConfig, input_dir: str, output_dir: str
+    ) -> None:
+        super().__init__(global_config, input_dir, output_dir)
         self.conf_file = "eventtypes.conf"
         self.alert_settings: Dict[str, List[Dict[str, Any]]] = {}
         envs = normalize.normalize(
-            self._global_config.alerts,
-            self._global_config.namespace,
+            global_config.alerts,
+            global_config.namespace,
         )
         schema_content = envs["schema.content"]
         self.alert_settings = schema_content["modular_alerts"]

splunk_add_on_ucc_framework/generators/conf_files/create_inputs_conf.py

@@ -17,6 +17,7 @@
 from typing import Any, Dict, List, Optional
 
 from splunk_add_on_ucc_framework.generators.file_generator import FileGenerator
+from splunk_add_on_ucc_framework.global_config import GlobalConfig
 
 
 class InputsConf(FileGenerator):
@@ -25,13 +26,10 @@ class InputsConf(FileGenerator):
         "file for the services mentioned in globalConfig"
     )
 
-    def _conf_file_name(self, conf_name: str) -> str:
-        return f"{conf_name}.conf"
-
-    def _spec_file_name(self, conf_name: str) -> str:
-        return f"{self._conf_file_name(conf_name)}.spec"
-
-    def _set_attributes(self) -> None:
+    def __init__(
+        self, global_config: GlobalConfig, input_dir: str, output_dir: str
+    ) -> None:
+        super().__init__(global_config, input_dir, output_dir)
         self.conf_file = self._conf_file_name("inputs")
 
         # A list of service names from globalConfig that will be in inputs.conf
@@ -47,10 +45,10 @@ def _set_attributes(self) -> None:
         # (i.e. dict key is the spec file name)
         self.other_spec_files: Dict[str, List[str]] = defaultdict(list)
 
-        if not self._global_config.has_inputs():
+        if not global_config.has_inputs():
             return
 
-        for service in self._global_config.inputs:
+        for service in global_config.inputs:
             default_values = None
 
             # If the service has a conf property, it will be in a separate spec file
@@ -99,6 +97,12 @@ def _set_attributes(self) -> None:
                 prop = f"{field_name} = {field_value}".rstrip()
                 spec_properties.append(prop)
 
+    def _conf_file_name(self, conf_name: str) -> str:
+        return f"{conf_name}.conf"
+
+    def _spec_file_name(self, conf_name: str) -> str:
+        return f"{self._conf_file_name(conf_name)}.spec"
+
     def generate(self) -> Optional[List[Dict[str, str]]]:
         conf_files: List[Dict[str, str]] = []
         conf = self.generate_conf()