feat: suffix parsing (#187)

* feat: suffix parsing

* fix: build fix

* fix: build fix

* fix: PR comments

* fix: fix

Author: weliasz

splunk_connect_for_snmp_poller/manager/hec_sender.py

@@ -14,7 +14,6 @@
 #    limitations under the License.
 #   ########################################################################
 import json
-import re
 import time
 
 import requests
@@ -27,12 +26,6 @@
 )
 from splunk_connect_for_snmp_poller.manager.data.inventory_record import InventoryRecord
 from splunk_connect_for_snmp_poller.manager.static.mib_enricher import MibEnricher
-from splunk_connect_for_snmp_poller.manager.variables import (
-    enricher_additional_varbinds,
-    enricher_name,
-    enricher_oid_family,
-)
-from splunk_connect_for_snmp_poller.utilities import multi_key_lookup
 
 logger = get_logger(__name__)
 
@@ -77,7 +70,6 @@ def post_data_to_splunk_hec(
     index,
     ir: InventoryRecord,
     additional_metric_fields,
-    server_config,
     one_time_flag=False,
     mib_enricher=None,
     is_error=False,
@@ -93,7 +85,6 @@ def post_data_to_splunk_hec(
             index["metric_index"],
             ir,
             additional_metric_fields,
-            server_config,
             one_time_flag=one_time_flag,
             mib_enricher=mib_enricher,
         )
@@ -184,13 +175,13 @@ def build_metric_data(
     index,
     ir: InventoryRecord,
     additional_metric_fields,
-    server_config,
     one_time_flag=False,
     mib_enricher=None,
 ):
     json_val = json.loads(variables_binds)
     metric_name = json_val["metric_name"]
     metric_value = json_val["_value"]
+    parsed_index = json_val.get("parsed_index")
     fields = {
         f"metric_name:{metric_name}": metric_value,
         EventField.FREQUENCY.value: ir.frequency_str,
@@ -208,47 +199,20 @@ def build_metric_data(
     else:
         builder.add(EventField.SOURCETYPE, "sc4snmp:metric")
 
-    extract_additional_properties(fields, metric_name, metric_value, server_config)
+    extract_additional_properties(fields, metric_name, metric_value, parsed_index)
 
     builder.add_fields(fields)
     return builder.build()
 
 
-def extract_additional_properties(fields, metric_name, metric_value, server_config):
-    result = multi_key_lookup(server_config, (enricher_name, enricher_oid_family))
-    oid_families = result if result else []
-    any_regex_matched = False
-
-    for family in oid_families.keys():
-        if metric_name.startswith("sc4snmp." + family):
-            stripped = metric_name[: metric_name.index("_")]
-            input_text = metric_name[metric_name.index("_") + 1 :]  # noqa: E203
+def extract_additional_properties(fields, metric_name, metric_value, parsed_index):
+    stripped = metric_name[: metric_name.index("_")]
+    del fields["metric_name:" + metric_name]
+    fields["metric_name:" + stripped] = metric_value
 
-            entries = multi_key_lookup(
-                oid_families, (family, enricher_additional_varbinds)
-            )
-            if entries:
-                regex_entries = [
-                    entry["regex"] for entry in entries if "regex" in entry
-                ]
-                for regex in regex_entries:
-                    result = re.match(regex, input_text)
-                    if result:
-                        any_regex_matched = True
-                        for key, value in result.groupdict().items():
-                            fields[key] = value.replace("_", ".")
-                        del fields["metric_name:" + metric_name]
-                        fields["metric_name:" + stripped] = metric_value
-                        break
-            break
-
-    if not any_regex_matched:
-        stripped = metric_name[: metric_name.rindex("_")]
-        input_text = metric_name[metric_name.rindex("_") + 1 :]  # noqa: E203
-
-        fields["index_number"] = input_text
-        del fields["metric_name:" + metric_name]
-        fields["metric_name:" + stripped] = metric_value
+    if parsed_index:
+        for key, value in parsed_index.items():
+            fields[key] = value
 
 
 def build_error_data(

splunk_connect_for_snmp_poller/manager/task_utilities.py

@@ -229,7 +229,6 @@ async def snmp_get_handler(
     ir,
     additional_metric_fields,
     var_binds,
-    server_config,
 ):
     """
     Perform the SNMP Get for an oid,
@@ -261,7 +260,6 @@ async def snmp_get_handler(
                 index,
                 ir,
                 additional_metric_fields,
-                server_config,
                 one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
                 mib_enricher=mib_enricher,
             )
@@ -278,7 +276,6 @@ async def snmp_get_handler(
                 index,
                 ir,
                 additional_metric_fields,
-                server_config,
                 one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
                 is_error=is_error,
             )
@@ -333,7 +330,6 @@ def _any_walk_failure_happened(
     is_metric,
     ir,
     additional_metric_fields,
-    server_config,
     var_binds,
 ):
     is_error, result = prepare_error_message(
@@ -349,7 +345,6 @@ def _any_walk_failure_happened(
             index,
             ir,
             additional_metric_fields,
-            server_config,
             one_time_flag=one_time_flag,
             is_error=is_error,
         )
@@ -390,7 +385,6 @@ async def snmp_bulk_handler(
     ir,
     additional_metric_fields,
     var_binds,
-    server_config,
 ):
     """
     Perform the SNMP Bulk for an array of oids
@@ -426,7 +420,6 @@ async def snmp_bulk_handler(
                     index,
                     ir,
                     additional_metric_fields,
-                    server_config,
                     one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
                     mib_enricher=mib_enricher,
                 )
@@ -443,7 +436,6 @@ async def snmp_bulk_handler(
                     index,
                     ir,
                     additional_metric_fields,
-                    server_config,
                     one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
                     is_error=is_error,
                 )
@@ -464,7 +456,6 @@ async def walk_handler(
     one_time_flag,
     ir,
     additional_metric_fields,
-    server_config,
 ):
     """
     Perform the SNMP Walk for oid end with *,
@@ -493,7 +484,6 @@ async def walk_handler(
             is_metric,
             ir,
             additional_metric_fields,
-            server_config,
             var_binds,
         ):
             if OnetimeFlag.is_a_walk(one_time_flag):
@@ -509,7 +499,6 @@ async def walk_handler(
                 index,
                 ir,
                 additional_metric_fields,
-                server_config,
                 one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
             )
     if OnetimeFlag.is_a_walk(one_time_flag):
@@ -567,7 +556,6 @@ async def walk_handler_with_enricher(
     one_time_flag,
     ir,
     additional_metric_fields,
-    server_config,
 ):
     """
     Perform the SNMP Walk for oid end with *,
@@ -597,7 +585,6 @@ async def walk_handler_with_enricher(
             is_metric,
             ir,
             additional_metric_fields,
-            server_config,
             var_binds,
         ):
             break
@@ -620,7 +607,6 @@ async def walk_handler_with_enricher(
                 index,
                 ir,
                 additional_metric_fields,
-                server_config,
                 one_time_flag=OnetimeFlag.is_a_walk(one_time_flag),
             )
 

splunk_connect_for_snmp_poller/manager/tasks.py

@@ -67,7 +67,6 @@ async def get_snmp_data(
     one_time_flag,
     ir,
     additional_metric_fields,
-    server_config,
 ):
     if var_binds:
         try:
@@ -86,7 +85,6 @@ async def get_snmp_data(
                 ir,
                 additional_metric_fields,
                 var_binds,
-                server_config,
             )
         except Exception as e:
             logger.exception(f"Error occurred while calling {handler.__name__}(): {e}")
@@ -172,7 +170,6 @@ async def snmp_polling_async(
         one_time_flag,
         ir,
         additional_metric_fields,
-        server_config,
     ]
     get_bulk_specific_parameters = [mongo_connection, enricher_presence]
     try:

tests/test_additional_data_extraction.py

@@ -22,70 +22,22 @@
 
 class TestAdditionalDataExtraction(TestCase):
     def test_data_extraction(self):
-        server_config = {
-            "enricher": {
-                "oidFamily": {
-                    "TCP-MIB": {
-                        "additionalVarBinds": [
-                            {
-                                "regex": "(?P<IP_one>[0-9]+_[0-9]+_[0-9]+_[0-9]+)_(?P<port>[0-9]+)_(?P<IP_two>[0-9]+_[0-9]+_[0-9]+_[0-9]+)_(?P<index_number>[0-9]+)",  # noqa: E501
-                            }
-                        ]
-                    },
-                    "IF-MIB": {
-                        "existingVarBinds": [
-                            {"ifDescr": "interface_desc"},
-                            {"ifPhysAddress": "MAC_address"},
-                        ],
-                    },
-                    "UDP-MIB": {
-                        "additionalVarBinds": [
-                            {
-                                "regex": '(?P<protocol_version_one>ipv4)_"(?P<IP_one>[0-9]+_[0-9]+_[0-9]+_[0-9]+)"_(?P<port_one>[0-9]+)_(?P<protocol_version_two>ipv4)_"(?P<IP_two>[0-9]+_[0-9]+_[0-9]+_[0-9]+)"_(?P<index_number>[0-9]+)_(?P<port_two>[0-9]+)',  # noqa: E501
-                            }
-                        ]
-                    },
-                }
-            }
-        }
 
         fields = {
             "metric_name:sc4snmp.TCP-MIB.tcpConnLocalPort_192_168_0_1_161_127_0_0_1_5": "1111"
         }
-        fields2 = {"metric_name:sc4snmp.IF-MIB.ifInErrors_2_1_asdad_23": "173127"}
-        fields3 = {
-            'metric_name:sc4snmp.UDP-MIB.udpEndpointProcess_ipv4_"0_0_0_0"_111_ipv4_"0_0_0_0"_0_13348': "123"
+
+        parsed_index = {
+            "test1": "value1",
+            "test2": "value2",
         }
 
         extract_additional_properties(
             fields,
             "sc4snmp.TCP-MIB.tcpConnLocalPort_192_168_0_1_161_127_0_0_1_5",
             "1111",
-            server_config,
-        )
-
-        extract_additional_properties(
-            fields2, "sc4snmp.IF-MIB.ifInErrors_2_1_asdad_23", "173127", server_config
+            parsed_index,
         )
 
-        extract_additional_properties(
-            fields3,
-            'sc4snmp.UDP-MIB.udpEndpointProcess_ipv4_"0_0_0_0"_111_ipv4_"0_0_0_0"_0_13348',
-            "123",
-            server_config,
-        )
-
-        self.assertEqual(fields["IP_one"], "192.168.0.1")
-        self.assertEqual(fields["port"], "161")
-        self.assertEqual(fields["IP_two"], "127.0.0.1")
-        self.assertEqual(fields["index_number"], "5")
-
-        self.assertEqual(fields2["index_number"], "23")
-
-        self.assertEqual(fields3["protocol_version_one"], "ipv4")
-        self.assertEqual(fields3["IP_one"], "0.0.0.0")
-        self.assertEqual(fields3["port_one"], "111")
-        self.assertEqual(fields3["protocol_version_two"], "ipv4")
-        self.assertEqual(fields3["IP_two"], "0.0.0.0")
-        self.assertEqual(fields3["index_number"], "0")
-        self.assertEqual(fields3["port_two"], "13348")
+        self.assertEqual(fields["test1"], "value1")
+        self.assertEqual(fields["test2"], "value2")