fix: new execution of enricher (#163)

* fix: add delay tp snmp_polling

* fix: change variable of extract_network_interface_data_from_additional_config

* fix: add loggers

* fix: change way of adding additional varbinds

* fix: add loggers

* fix: refactor method to update enricher structure

* fix: add loggers

* fix: update OID

* fix: snmp_pooling function

* fix: update enricher to trigger only in case of exisitng IF-MIB

* fix: add logs and redesign function

* fix: change sequence of enricher check

* fix: refactor code

* fix: delete unnecessary loggers

* fix: refactor variable name for delete_ifmib decorator

* fix: adjusting code to PR comments

* fix: remove unnecessary boolean cast

* fix: add saving InventoryRecord for every host in inventory

* fix: change way of requesting walk for IF-MIB

Author: omrozowicz-splunk

splunk_connect_for_snmp_poller/manager/poller.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import copy
 import functools
 import logging.config
 import threading
@@ -28,6 +29,7 @@
     create_poller_scheduler_entry_key,
     parse_inventory_file,
     return_database_id,
+    update_enricher_config,
 )
 from splunk_connect_for_snmp_poller.manager.profile_matching import (
     assign_profiles_to_device,
@@ -59,6 +61,7 @@ def __init__(self, args, server_config):
         self._unmatched_devices = {}
         self._lock = threading.Lock()
         self._force_refresh = False
+        self._old_enricher = {}
 
     def force_inventory_refresh(self):
         self._force_refresh = True
@@ -104,6 +107,7 @@ def __check_inventory(self):
             # TODO should rethink logic with changed to profiles and oids
             inventory_entry_keys = set()
             inventory_hosts = set()
+            inventory_hosts_with_snmp_data = {}
             profiles = get_profiles(self._server_config)
             for ir in parse_inventory_file(self._args.inventory, profiles):
                 entry_key = create_poller_scheduler_entry_key(ir.host, ir.profile)
@@ -117,7 +121,9 @@ def __check_inventory(self):
                     )
                     continue
                 inventory_entry_keys.add(entry_key)
-                inventory_hosts.add(return_database_id(ir.host))
+                ir_host = return_database_id(ir.host)
+                inventory_hosts.add(return_database_id(ir_host))
+                inventory_hosts_with_snmp_data[ir_host] = copy.deepcopy(ir)
                 if ir.profile == DYNAMIC_PROFILE:
                     self.delete_all_entries_per_host(ir.host)
                     self.add_device_for_profile_matching(ir)
@@ -131,8 +137,37 @@ def __check_inventory(self):
                     else:
                         self.update_schedule_for_changed_conf(entry_key, ir, profiles)
 
+            if server_config_modified:
+                new_enricher = self._server_config.get("enricher", {})
+                if new_enricher != self._old_enricher:
+                    self.run_enricher_check(
+                        new_enricher, profiles, inventory_hosts_with_snmp_data
+                    )
             self.clean_job_inventory(inventory_entry_keys, inventory_hosts)
 
+    def run_enricher_check(
+        self, new_enricher, profiles, inventory_hosts_with_snmp_data
+    ):
+        logger.info(
+            f"Previous enricher: {self._old_enricher} \n New enricher: {new_enricher}"
+        )
+        if new_enricher == {}:
+            logger.debug("Enricher is being deleted from MongoDB")
+            self._mongo.delete_all_static_data()
+            self._old_enricher = {}
+            return
+        for inventory_host in inventory_hosts_with_snmp_data.keys():
+            update_enricher_config(
+                self._old_enricher,
+                new_enricher,
+                self._mongo,
+                profiles,
+                inventory_hosts_with_snmp_data[inventory_host],
+                self._server_config,
+                self.__get_splunk_indexes(),
+            )
+        self._old_enricher = new_enricher
+
     def delete_all_entries_per_host(self, host):
         for entry_key in list(self._jobs_map.keys()):
             if entry_key.split("#")[0] == host:

splunk_connect_for_snmp_poller/manager/poller_utilities.py

@@ -27,13 +27,20 @@
 from splunk_connect_for_snmp_poller.manager.realtime.real_time_data import (
     should_redo_walk,
 )
+from splunk_connect_for_snmp_poller.manager.static.interface_mib_utililities import (
+    extract_network_interface_data_from_additional_config,
+)
 from splunk_connect_for_snmp_poller.manager.task_utilities import parse_port
 from splunk_connect_for_snmp_poller.manager.tasks import snmp_polling
 from splunk_connect_for_snmp_poller.manager.validator.inventory_validator import (
     DYNAMIC_PROFILE,
     is_valid_inventory_line_from_dict,
     should_process_inventory_line,
 )
+from splunk_connect_for_snmp_poller.manager.variables import (
+    enricher_if_mib,
+    enricher_oid_family,
+)
 from splunk_connect_for_snmp_poller.utilities import OnetimeFlag, multi_key_lookup
 
 logger = logging.getLogger(__name__)
@@ -274,6 +281,88 @@ def automatic_realtime_task(
         logger.exception("Error during automatic_realtime_task")
 
 
+def update_enricher_config(
+    old_enricher,
+    new_enricher,
+    mongo,
+    profiles,
+    inventory_host,
+    server_config,
+    splunk_indexes,
+):
+    run_ifmib_walk = is_ifmib_different(old_enricher, new_enricher)
+    if run_ifmib_walk:
+        _update_enricher_config_with_ifmib(
+            profiles, inventory_host, server_config, splunk_indexes
+        )
+    else:
+        _update_enricher_config_for_additional_varbinds(
+            old_enricher, new_enricher, mongo, inventory_host.host, server_config
+        )
+
+
+def _update_enricher_config_with_ifmib(
+    profiles,
+    inventory_host,
+    server_config,
+    splunk_indexes,
+):
+    inventory_host.profile = OidConstant.IF_MIB
+    snmp_polling.delay(
+        inventory_host.to_json(),
+        server_config,
+        splunk_indexes,
+        profiles,
+    )
+
+
+def _update_enricher_config_for_additional_varbinds(
+    old_enricher,
+    new_enricher,
+    mongo,
+    inventory_host,
+    server_config,
+):
+    families_to_delete = deleted_oid_families(old_enricher, new_enricher)
+    mongo.delete_oidfamilies_from_static_data(inventory_host, families_to_delete)
+    additional_enricher_varbinds = modified_oid_families(server_config)
+    mongo.update_static_data_for_one(inventory_host, additional_enricher_varbinds)
+
+
+def is_ifmib_different(old_enricher, new_enricher):
+    new_if_mib = multi_key_lookup(new_enricher, (enricher_oid_family, enricher_if_mib))
+    old_if_mib = multi_key_lookup(old_enricher, (enricher_oid_family, enricher_if_mib))
+    return new_if_mib != old_if_mib
+
+
+def delete_ifmib(func):
+    def function(*args, **kwargs):
+        set_of_oid_families_to_modify = func(*args, **kwargs)
+        if "IF-MIB" in set_of_oid_families_to_modify:
+            if isinstance(set_of_oid_families_to_modify, set):
+                set_of_oid_families_to_modify.remove("IF-MIB")
+            if isinstance(set_of_oid_families_to_modify, dict):
+                set_of_oid_families_to_modify.pop("IF-MIB")
+        return set_of_oid_families_to_modify
+
+    return function
+
+
+@delete_ifmib
+def deleted_oid_families(old_enricher, new_enricher):
+    old_families = old_enricher.get("oidFamily", {})
+    new_families = new_enricher.get("oidFamily", {})
+    return set(old_families.keys()) - set(new_families.keys())
+
+
+@delete_ifmib
+def modified_oid_families(server_config):
+    additional_enricher_varbinds = (
+        extract_network_interface_data_from_additional_config(server_config)
+    )
+    return additional_enricher_varbinds
+
+
 def create_poller_scheduler_entry_key(host, profile):
     return host + "#" + profile
 

splunk_connect_for_snmp_poller/manager/realtime/oid_constant.py

@@ -19,3 +19,4 @@ class OidConstant:
     SYS_UP_TIME_INSTANCE = "1.3.6.1.2.1.1.3.0"
     # see https://www.alvestrand.no/objectid/1.3.6.1.html for a better understanding
     UNIVERSAL_BASE_OID = "1.3.6.1.*"
+    IF_MIB = "1.3.6.1.2.1.2.*"

splunk_connect_for_snmp_poller/manager/variables.py

@@ -16,3 +16,4 @@
 enricher_existing_varbinds = "existingVarBinds"
 enricher_additional_varbinds = "additionalVarBinds"
 enricher_oid_family = "oidFamily"
+enricher_if_mib = "IF-MIB"

splunk_connect_for_snmp_poller/mongo.py

@@ -216,3 +216,32 @@ def update_mib_static_data_for(self, host, existing_data, additional_data):
             return_document=ReturnDocument.AFTER,
         )
         return static_data_dictionary[WalkedHostsRepository.MIB_STATIC_DATA]
+
+    def update_static_data_for_one(self, host, static_data_dictionary):
+        logger.info(f"Updating static data for {host} with {static_data_dictionary}")
+        for oid_family in static_data_dictionary.keys():
+            index_dict = static_data_dictionary[oid_family]
+            if index_dict:
+                self._walked_hosts.update(
+                    {"_id": host},
+                    {
+                        "$set": {
+                            f"MIB-STATIC-DATA.{oid_family}.additionalVarBinds": index_dict
+                        }
+                    },
+                    upsert=True,
+                )
+
+    def delete_oidfamilies_from_static_data(self, host, oid_families):
+        logger.info(f"Deleting oidfamilies {oid_families} from {host}")
+        for oid_family in oid_families:
+            self._walked_hosts.find_one_and_update(
+                {"_id": host},
+                {"$unset": {f"MIB-STATIC-DATA.{oid_family}": ""}},
+            )
+
+    def delete_all_static_data(self):
+        self._walked_hosts.update_many(
+            {},
+            {"$unset": {"MIB-STATIC-DATA": ""}},
+        )

tests/test_poller.py

@@ -0,0 +1,46 @@
+#
+# Copyright 2021 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import sys
+from unittest import TestCase
+from unittest.mock import MagicMock, Mock, patch
+
+sys.modules["splunk_connect_for_snmp_poller.manager.celery_client"] = Mock()
+from splunk_connect_for_snmp_poller.manager.poller import Poller  # noqa: E402
+
+
+class TestPollerUtilities(TestCase):
+    def test_run_enricher_check_when_enricher_is_deleted(self):
+        server_config = {"mongo": ""}
+        with patch(
+            "splunk_connect_for_snmp_poller.mongo.WalkedHostsRepository.__init__"
+        ) as mock:
+            mock.return_value = None
+            obj = Poller([], server_config)
+            obj._old_enricher = {
+                "IF-MIB": {
+                    "existingVarBinds": [
+                        {"ifIndex": "interface_index"},
+                        {"ifDescr": "interface_desc"},
+                        {"ifPhysAddress": "MAC_address"},
+                    ],
+                    "additionalVarBinds": [{"indexNum": "index_number"}],
+                }
+            }
+            with patch(
+                "splunk_connect_for_snmp_poller.mongo.WalkedHostsRepository.delete_all_static_data"
+            ):
+                obj.run_enricher_check({}, [], {"127.0.0.1:161": MagicMock()})
+            self.assertEqual(obj._old_enricher, {})