feat: add partial walk (#1215)

ajasnosz · web-flow · commit 0e1f32f77697 · 2025-07-22T10:15:43.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,7 @@
 - implemented mTLS for Splunk 10
 - update default microk8s to 1.33
 - introduce `splunkMetricNameHyphenToUnderscore` parameter to make metric names follow Splunk schema
+- change default walk to get only SNMPv2-MIB
 
 ### Fixed
 
diff --git a/charts/splunk-connect-for-snmp/templates/NOTES.txt b/charts/splunk-connect-for-snmp/templates/NOTES.txt
@@ -1,2 +1,2 @@
-Walk profiles no longer include IF-MIB family by default.
-If you've used this functionality before, please update the walk profile with ['IF-MIB'] varBind.
+Default walk no longer calls full oid tree, instead it is collecting only 'SNMPv2-MIB'.
+If you want to call full oid for the devices, you have to set enableFullWalk flag to true.
diff --git a/charts/splunk-connect-for-snmp/templates/inventory/job.yaml b/charts/splunk-connect-for-snmp/templates/inventory/job.yaml
@@ -50,6 +50,8 @@ spec:
             value: {{ .Values.scheduler.tasksExpiryTime | quote }}
           - name: CONFIG_FROM_MONGO
             value: {{ quote .Values.UI.enable | default "false" }}
+          - name: ENABLE_FULL_WALK
+            value: {{ .Values.poller.enableFullWalk | default "false" | quote }}
           volumeMounts:
             - name: config
               mountPath: "/app/config"
diff --git a/charts/splunk-connect-for-snmp/templates/ui/_helpers.tpl b/charts/splunk-connect-for-snmp/templates/ui/_helpers.tpl
@@ -51,6 +51,8 @@ spec:
             value: {{ .Values.scheduler.logLevel | default "INFO" }}
           - name: CONFIG_FROM_MONGO
             value: {{ quote .Values.UI.enable | default "false" }}
+          - name: ENABLE_FULL_WALK
+            value: {{ .Values.poller.enableFullWalk | default "false" | quote }}
           volumeMounts:
             - name: config
               mountPath: "/app/config"
diff --git a/charts/splunk-connect-for-snmp/values.schema.json b/charts/splunk-connect-for-snmp/values.schema.json
@@ -370,6 +370,9 @@
         },
         "logLevel": {
           "type": "string"
+        },
+        "enableFullWalk": {
+          "type": "boolean"
         }
       }
     },
diff --git a/charts/splunk-connect-for-snmp/values.yaml b/charts/splunk-connect-for-snmp/values.yaml
@@ -247,6 +247,9 @@ poller:
   # https://splunk.github.io/splunk-connect-for-snmp/main/microk8s/configuration/poller-configuration/#define-usernamesecrets
   usernameSecrets: []
 
+  # flag to enable polling full walk tree for devices
+  enableFullWalk: false
+
   # Here is where polling happens. Learn more on how to configure it here:
   # https://splunk.github.io/splunk-connect-for-snmp/main/microk8s/configuration/poller-configuration/
 
diff --git a/docker_compose/.env b/docker_compose/.env
@@ -86,6 +86,7 @@ WORKER_TRAP_MEMORY_RESERVATIONS=250M
 # Inventory configuration
 INVENTORY_LOG_LEVEL=INFO
 CHAIN_OF_TASKS_EXPIRY_TIME=500
+ENABLE_FULL_WALK=false
 
 # Traps configuration
 SNMP_V3_SECURITY_ENGINE_ID=80003a8c04
diff --git a/docker_compose/docker-compose.yaml b/docker_compose/docker-compose.yaml
@@ -117,6 +117,7 @@ services:
       LOG_LEVEL: ${INVENTORY_LOG_LEVEL:-INFO}
       CHAIN_OF_TASKS_EXPIRY_TIME: ${CHAIN_OF_TASKS_EXPIRY_TIME:-500}
       CONFIG_FROM_MONGO: ${CONFIG_FROM_MONGO:-false}
+      ENABLE_FULL_WALK: ${ENABLE_FULL_WALK:-false}
     volumes:
     - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro
     - ${INVENTORY_FILE_ABSOLUTE_PATH}:/app/inventory/inventory.csv:ro
diff --git a/docs/microk8s/configuration/configuring-profiles.md b/docs/microk8s/configuration/configuring-profiles.md
@@ -137,6 +137,9 @@ Sometimes static profiles have additional functionalities to be used in specific
 
 #### WALK profile
 
+!!!info
+    By default the walk without any profile set is polling only `SNMPv2-MIB`. For changing the scope of the walk you can configure new walk profile or use `enableFullWalk` flag. More about it in [poller configuration](../poller-configuration/#poller-configuration-file).
+
 If you would like to limit the scope of the walk, you should set one of the profiles in the inventory to point to the profile 
 definition of the `walk` type:
 ```yaml
diff --git a/docs/microk8s/configuration/poller-configuration.md b/docs/microk8s/configuration/poller-configuration.md
@@ -23,6 +23,7 @@ poller:
    - sc4snmp-hlab-sha-aes
    - sc4snmp-hlab-sha-des
   logLevel: "WARN"
+  enableFullWalk: false
   inventory: |
     address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete
     10.202.4.202,,2c,public,,,2000,,,
@@ -31,6 +32,9 @@ poller:
 !!! info
     The header's line (`address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete`) is necessary for the correct execution of SC4SNMP. Do not remove it.
 
+### Default walk scope
+The default walk profile is polling only `SNMPv2-MIB`. If the full oid tree walk is required it can be enabled by changing `enableFullWalk` flag to true.
+
 ### IPv6 hostname resolution
 When IPv6 is enabled and device is dual stack, the hostname resolution will try to resolve the name to the IPv6 address first, then to the IPv4 address.
 
diff --git a/docs/microk8s/configuration/step-by-step-poll.md b/docs/microk8s/configuration/step-by-step-poll.md
@@ -8,7 +8,7 @@ In the following example, there are 4 hosts you want to poll from:
 4. `10.202.4.204:163`
    
 To retrieve data from the device efficiently, first determine the specific data needed. Instead of walking through 
-the entire `1.3.6.1`, limit the walk to poll only the necessary data. Configure the `IF-MIB` family for interfaces and 
+the entire `1.3.6.1` or polling only `SNMPv2-MIB`, limit or expand the walk to poll the necessary data. Configure the `IF-MIB` family for interfaces and 
 the `UCD-SNMP-MIB` for CPU-related statistics. In the `scheduler` section of `values.yaml`, define the target group and 
 establish the polling parameters, known as the profile, to gather the desired data precisely. See the following example: 
 
diff --git a/docs/microk8s/sc4snmp-installation.md b/docs/microk8s/sc4snmp-installation.md
@@ -187,6 +187,9 @@ index="netops" sourcetype="sc4snmp:event"
 | mpreview index="netmetrics" | search sourcetype="sc4snmp:metric"
 ```
 
+!!!info
+    Default walk profile is polling only `SNMPv2-MIB`. To enable full oid tree polling see [poller configuration](../configuration/poller-configuration/#poller-configuration-file).
+
 > **_NOTE:_** Before polling starts, SC4SNMP must perform the SNMP WALK process on the device. It is run the first time after configuring the new device, and then during the run time in every `walk_interval`. 
 > Its purpose is to gather all the data and provide meaningful context for the polling records. For example, it might report that your device is so large that the walk takes too long, so the scope of walking needs to be limited.
 > In such cases, enable the small walk. See [walk takes too much time](../../troubleshooting/polling-issues/#walking-a-device-takes-too-much-time).
diff --git a/docs/troubleshooting/polling-issues.md b/docs/troubleshooting/polling-issues.md
@@ -45,6 +45,8 @@ If you put in only the IP address (for example, `127.0.0.1`), then errors will b
 See [Configure small walk profile](../../microk8s/configuration/configuring-profiles/#walk-profile) to enable the small walk 
 functionality.
 
+Check if `poller.enableFullWalk` flag is set to `false`. See [poller configuration](../microk8s/configuration/poller-configuration/#poller-configuration-file).
+
 ## An error of SNMP isWalk=True blocks traffic on the SC4SNMP instance
 
 If you see many `An error of SNMP isWalk=True` errors in your logs, that means that there is a connection problem 
diff --git a/integration_tests/.env b/integration_tests/.env
@@ -78,6 +78,7 @@ WORKER_TRAP_MEMORY_RESERVATIONS=250M
 # Inventory configuration
 INVENTORY_LOG_LEVEL=INFO
 CHAIN_OF_TASKS_EXPIRY_TIME=500
+ENABLE_FULL_WALK=true
 
 # Traps configuration
 SNMP_V3_SECURITY_ENGINE_ID=80003a8c04
diff --git a/integration_tests/splunk_test_utils.py b/integration_tests/splunk_test_utils.py
@@ -54,6 +54,13 @@ def splunk_single_search(service, search):
 """
 
 inventory_template_microk8s = """poller:
+  enableFullWalk: true
+  inventory: |
+    address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete
+"""
+
+inventory_template_microk8s_no_walk = """poller:
+  enableFullWalk: false
   inventory: |
     address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete
 """
@@ -80,6 +87,7 @@ def splunk_single_search(service, search):
 
 TEMPLATE_MAPPING_MICROK8S = {
     "inventory.yaml": inventory_template_microk8s,
+    "inventory2.yaml": inventory_template_microk8s_no_walk,
     "profiles.yaml": profiles_template_microk8s,
     "scheduler_secrets.yaml": poller_secrets_template_microk8s,
     "traps_secrets.yaml": traps_secrets_template_microk8s,
@@ -140,6 +148,23 @@ def upgrade_docker_compose():
     os.system("sudo docker compose up -d")
 
 
+def upgrade_env_compose(variable, new_value, env_path=".env"):
+    lines = []
+    found = False
+    if os.path.exists(env_path):
+        with open(env_path) as f:
+            for line in f:
+                if line.strip().startswith(f"{variable}="):
+                    lines.append(f"{variable}={new_value}\n")
+                    found = True
+                else:
+                    lines.append(line)
+    if not found:
+        lines.append(f"{variable}={new_value}\n")
+    with open(env_path, "w") as f:
+        f.writelines(lines)
+
+
 def create_v3_secrets_compose(
     secret_name="secretv4",
     user_name="snmp-poller",
diff --git a/integration_tests/test_poller_integration.py b/integration_tests/test_poller_integration.py
@@ -29,6 +29,7 @@
     update_profiles_compose,
     update_profiles_microk8s,
     upgrade_docker_compose,
+    upgrade_env_compose,
     upgrade_helm_microk8s,
     yaml_escape_list,
 )
@@ -500,11 +501,12 @@ def setup_small_walk(request):
     if deployment == "microk8s":
         update_profiles_microk8s(profile)
         update_file_microk8s(
-            [f"{trap_external_ip},,2c,public,,,20,walk1,f,"], "inventory.yaml"
+            [f"{trap_external_ip},,2c,public,,,20,walk1,f,"], "inventory2.yaml"
         )
-        upgrade_helm_microk8s(["inventory.yaml", "profiles.yaml"])
+        upgrade_helm_microk8s(["inventory2.yaml", "profiles.yaml"])
     else:
         update_profiles_compose(profile)
+        upgrade_env_compose("ENABLE_FULL_WALK", "false")
         update_inventory_compose([f"{trap_external_ip},,2c,public,,,20,walk1,f,"])
         upgrade_docker_compose()
 
@@ -513,10 +515,11 @@ def setup_small_walk(request):
     yield
     if deployment == "microk8s":
         update_file_microk8s(
-            [f"{trap_external_ip},,2c,public,,,20,walk1,f,t"], "inventory.yaml"
+            [f"{trap_external_ip},,2c,public,,,20,walk1,f,t"], "inventory2.yaml"
         )
-        upgrade_helm_microk8s(["inventory.yaml"])
+        upgrade_helm_microk8s(["inventory2.yaml"])
     else:
+        upgrade_env_compose("ENABLE_FULL_WALK", "true")
         update_inventory_compose([f"{trap_external_ip},,2c,public,,,20,walk1,f,t"])
         upgrade_docker_compose()
 
@@ -545,6 +548,120 @@ def test_check_if_walk_scope_was_smaller(self, setup_splunk):
         assert metric_count > 0
 
 
+@pytest.fixture
+def setup_small_walk_with_full_walk_enabled(request):
+    trap_external_ip = request.config.getoption("trap_external_ip")
+    deployment = request.config.getoption("sc4snmp_deployment")
+    profile = {
+        "walk1": {
+            "condition": {"type": "walk"},
+            "varBinds": [yaml_escape_list(sq("IP-MIB"))],
+        },
+    }
+    if deployment == "microk8s":
+        update_profiles_microk8s(profile)
+        update_file_microk8s(
+            [f"{trap_external_ip},,2c,public,,,20,walk1,f,"], "inventory.yaml"
+        )
+        upgrade_helm_microk8s(["inventory.yaml", "profiles.yaml"])
+    else:
+        update_profiles_compose(profile)
+        upgrade_env_compose("ENABLE_FULL_WALK", "true")
+        update_inventory_compose([f"{trap_external_ip},,2c,public,,,20,walk1,f,"])
+        upgrade_docker_compose()
+    time.sleep(20)
+    yield
+    if deployment == "microk8s":
+        update_file_microk8s(
+            [f"{trap_external_ip},,2c,public,,,20,walk1,f,t"], "inventory.yaml"
+        )
+        upgrade_helm_microk8s(["inventory.yaml"])
+    else:
+        update_inventory_compose([f"{trap_external_ip},,2c,public,,,20,walk1,f,t"])
+        upgrade_docker_compose()
+    time.sleep(20)
+
+
+@pytest.mark.usefixtures("setup_small_walk_with_full_walk_enabled")
+class TestSmallWalkWithFullWalkEnabled:
+    def test_check_if_full_walk_is_done_with_profile_set(self, setup_splunk):
+        time.sleep(20)
+        search_string = (
+            """| mpreview index=netmetrics earliest=-40s | search "TCP-MIB" """
+        )
+        result_count, metric_count = run_retried_single_search(
+            setup_splunk, search_string, 1
+        )
+        assert result_count > 0
+        assert metric_count > 0
+        search_string = (
+            """| mpreview index=netmetrics earliest=-40s | search "IP-MIB" """
+        )
+        result_count, metric_count = run_retried_single_search(
+            setup_splunk, search_string, 2
+        )
+        assert result_count > 0
+        assert metric_count > 0
+
+
+@pytest.fixture
+def setup_partial_walk(request):
+    trap_external_ip = request.config.getoption("trap_external_ip")
+    deployment = request.config.getoption("sc4snmp_deployment")
+
+    if deployment == "microk8s":
+        update_file_microk8s(
+            [f"{trap_external_ip},,2c,public,,,20,,f,"], "inventory2.yaml"
+        )
+        upgrade_helm_microk8s(["inventory2.yaml", "profiles.yaml"])
+    else:
+        upgrade_env_compose("ENABLE_FULL_WALK", "false")
+        update_inventory_compose([f"{trap_external_ip},,2c,public,,,20,,f,"])
+        upgrade_docker_compose()
+    time.sleep(30)
+    yield
+    if deployment == "microk8s":
+        update_file_microk8s(
+            [f"{trap_external_ip},,2c,public,,,20,,f,t"], "inventory.yaml"
+        )
+        upgrade_helm_microk8s(["inventory.yaml"])
+    else:
+        upgrade_env_compose("ENABLE_FULL_WALK", "true")
+        update_inventory_compose([f"{trap_external_ip},,2c,public,,,20,,f,"])
+        upgrade_docker_compose()
+    time.sleep(20)
+
+
+@pytest.mark.usefixtures("setup_partial_walk")
+class TestPartialWalk:
+    def test_check_if_partial_walk_is_done(self, setup_splunk):
+        time.sleep(20)
+        search_string = (
+            """| mpreview index=netmetrics earliest=-20s | search "TCP-MIB" """
+        )
+        result_count, metric_count = run_retried_single_search(
+            setup_splunk, search_string, 1
+        )
+        assert result_count == 0
+        assert metric_count == 0
+        search_string = (
+            """| mpreview index=netmetrics earliest=-20s | search "IP-MIB" """
+        )
+        result_count, metric_count = run_retried_single_search(
+            setup_splunk, search_string, 2
+        )
+        assert result_count == 0
+        assert metric_count == 0
+        search_string = (
+            """| mpreview index=netmetrics earliest=-20s | search "SNMPv2-MIB" """
+        )
+        result_count, metric_count = run_retried_single_search(
+            setup_splunk, search_string, 2
+        )
+        assert result_count > 0
+        assert metric_count > 0
+
+
 @pytest.fixture()
 def setup_v3_connection(request):
     trap_external_ip = request.config.getoption("trap_external_ip")
diff --git a/integration_tests/values.yaml b/integration_tests/values.yaml
@@ -67,6 +67,7 @@ scheduler:
   groups: |
     {}
 poller:
+  enableFullWalk: true
   usernameSecrets:
     - sv3poller
  #   - sc4snmp-hlab-sha-aes
diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/inventory/job.yaml
@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"
diff --git a/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml
@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"
diff --git a/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/inventory/job.yaml
@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"
diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/inventory/job.yaml
@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "true"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"
diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/configmap-backend.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/configmap-backend.yaml
@@ -53,6 +53,8 @@ data:
                 value: INFO
               - name: CONFIG_FROM_MONGO
                 value: "true"
+              - name: ENABLE_FULL_WALK
+                value: "false"
               volumeMounts:
                 - name: config
                   mountPath: "/app/config"
diff --git a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/inventory/job.yaml
@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"
diff --git a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml
diff --git a/splunk_connect_for_snmp/common/inventory_processor.py b/splunk_connect_for_snmp/common/inventory_processor.py
diff --git a/splunk_connect_for_snmp/inventory/loader.py b/splunk_connect_for_snmp/inventory/loader.py
diff --git a/splunk_connect_for_snmp/profiles/walk.yaml b/splunk_connect_for_snmp/profiles/walk.yaml
diff --git a/test/common/test_inventory_processor.py b/test/common/test_inventory_processor.py

Original file line number	Diff line number	Diff line change
`@@ -370,6 +370,9 @@`
`370`	`370`	`},`
`371`	`371`	`"logLevel": {`
`372`	`372`	`"type": "string"`
	`373`	`+ },`
	`374`	`+ "enableFullWalk": {`
	`375`	`+ "type": "boolean"`
`373`	`376`	`}`
`374`	`377`	`}`
`375`	`378`	`},`