feat: additional data extraction (#177)

* feat: additional data extraction

* fix: build fix

* Update tests/test_additional_data_extraction.py

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Co-authored-by: omrozowicz-splunk <86965961+omrozowicz-splunk@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: 3 people

splunk_connect_for_snmp_poller/manager/hec_sender.py

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #   ########################################################################
 import json
+import re
 import time
 
 import requests
@@ -27,6 +28,7 @@
 from splunk_connect_for_snmp_poller.manager.data.inventory_record import InventoryRecord
 from splunk_connect_for_snmp_poller.manager.static.mib_enricher import MibEnricher
 from splunk_connect_for_snmp_poller.manager.variables import (
+    enricher_additional_varbinds,
     enricher_name,
     enricher_oid_family,
 )
@@ -189,7 +191,6 @@ def build_metric_data(
     fields = {
         "metric_name:" + metric_name: metric_value,
         EventField.FREQUENCY.value: ir.frequency_str,
-        EventField.TIME.value: time.time(),
     }
     if mib_enricher:
         _enrich_metric_data(mib_enricher, json_val, fields)
@@ -200,20 +201,38 @@ def build_metric_data(
     builder = init_builder_with_common_data(time.time(), host, index)
     builder.add(EventField.EVENT, EventType.METRIC.value)
 
-    strip_trailing_index_number(fields, metric_name, metric_value, server_config)
+    extract_additional_properties(fields, metric_name, metric_value, server_config)
 
     builder.add_fields(fields)
     return builder.build()
 
 
-def strip_trailing_index_number(fields, metric_name, metric_value, server_config):
+def extract_additional_properties(fields, metric_name, metric_value, server_config):
     result = multi_key_lookup(server_config, (enricher_name, enricher_oid_family))
     oid_families = result if result else []
 
-    if any(metric_name.startswith("sc4snmp." + x) for x in oid_families):
-        stripped = metric_name[: metric_name.rindex("_")]
-        del fields["metric_name:" + metric_name]
-        fields["metric_name:" + stripped] = metric_value
+    for family in oid_families.keys():
+        if metric_name.startswith("sc4snmp." + family):
+            stripped = metric_name[: metric_name.index("_")]
+
+            input_text = metric_name[metric_name.index("_") + 1 :]  # noqa: E203
+
+            entries = oid_families[family][enricher_additional_varbinds]
+            for entry in entries:
+                if "regex" in entry and "names" in entry:
+                    regex = entry["regex"]
+                    names = entry["names"]
+                    names_list = names.split("/")
+
+                    result = re.match(regex, input_text)
+                    if result:
+                        for index, item in enumerate(names_list):
+                            fields[item] = result.group(index + 1)
+                        del fields["metric_name:" + metric_name]
+                        fields["metric_name:" + stripped] = metric_value
+                        # TODO delete blow debug statement
+                        fields["old_metric_name:" + metric_name] = metric_value
+                        continue
 
 
 def build_error_data(

tests/test_additional_data_extraction.py

@@ -0,0 +1,92 @@
+# Copyright 2021 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from unittest import TestCase
+
+from splunk_connect_for_snmp_poller.manager.hec_sender import (
+    extract_additional_properties,
+)
+
+
+class TestAdditionalDataExtraction(TestCase):
+    def test_data_extraction(self):
+        server_config = {
+            "enricher": {
+                "oidFamily": {
+                    "TCP-MIB": {
+                        "additionalVarBinds": [
+                            {
+                                "regex": "([0-9]+_[0-9]+_[0-9]+_[0-9]+)_([0-9]+)_([0-9]+_[0-9]+_[0-9]+_[0-9]+)_([0-9]+)",  # noqa: E501
+                                "names": "IP_one/port/IP_two/index_number",
+                            }
+                        ]
+                    },
+                    "IF-MIB": {
+                        "existingVarBinds": [
+                            {"ifDescr": "interface_desc"},
+                            {"ifPhysAddress": "MAC_address"},
+                        ],
+                        "additionalVarBinds": [{"indexNum": "index_number"}],
+                    },
+                    "UDP-MIB": {
+                        "additionalVarBinds": [
+                            {
+                                "regex": '(ipv4)_"([0-9]+_[0-9]+_[0-9]+_[0-9]+)"_([0-9]+)_(ipv4)_"([0-9]+_[0-9]+_[0-9]+_[0-9]+)"_([0-9]+)_([0-9]+)',  # noqa: E501
+                                "names": "protocol_version_one/IP_one/port_one/protocol_version_two/IP_two/index_number/port_two",  # noqa: E501
+                            }
+                        ]
+                    },
+                }
+            }
+        }
+
+        fields = {
+            "metric_name:sc4snmp.TCP-MIB.tcpConnLocalPort_192_168_0_1_161_127_0_0_1_5": "1111"
+        }
+        fields2 = {"metric_name:sc4snmp.IF-MIB.ifInErrors_2": "173127"}
+        fields3 = {
+            'metric_name:sc4snmp.UDP-MIB.udpEndpointProcess_ipv4_"0_0_0_0"_111_ipv4_"0_0_0_0"_0_13348': "123"
+        }
+
+        extract_additional_properties(
+            fields,
+            "sc4snmp.TCP-MIB.tcpConnLocalPort_192_168_0_1_161_127_0_0_1_5",
+            "1111",
+            server_config,
+        )
+
+        extract_additional_properties(
+            fields2, "sc4snmp.IF-MIB.ifInErrors_2", "173127", server_config
+        )
+
+        extract_additional_properties(
+            fields3,
+            'sc4snmp.UDP-MIB.udpEndpointProcess_ipv4_"0_0_0_0"_111_ipv4_"0_0_0_0"_0_13348',
+            "123",
+            server_config,
+        )
+
+        self.assertEqual(fields["IP_one"], "192_168_0_1")
+        self.assertEqual(fields["port"], "161")
+        self.assertEqual(fields["IP_two"], "127_0_0_1")
+        self.assertEqual(fields["index_number"], "5")
+
+        self.assertEqual(fields3["protocol_version_one"], "ipv4")
+        self.assertEqual(fields3["IP_one"], "0_0_0_0")
+        self.assertEqual(fields3["port_one"], "111")
+        self.assertEqual(fields3["protocol_version_two"], "ipv4")
+        self.assertEqual(fields3["IP_two"], "0_0_0_0")
+        self.assertEqual(fields3["index_number"], "0")
+        self.assertEqual(fields3["port_two"], "13348")