MINIFICPP-2689 Move Splunk tests to modular docker tests

- Add more logging to the PutSpunkHTTP processor
- Fail if returned status code is not 0

Author: lordgamez

behave_framework/src/minifi_test_framework/minifi/flow_definition.py

@@ -39,6 +39,9 @@ def add_processor(self, processor: Processor):
     def get_processor(self, processor_name: str) -> Processor | None:
         return next((proc for proc in self.processors if proc.name == processor_name), None)
 
+    def get_controller_service(self, controller_service_name: str) -> ControllerService | None:
+        return next((controller for controller in self.controller_services if controller.name == controller_service_name), None)
+
     def get_parameter_context(self, parameter_context_name: str) -> ParameterContext | None:
         return next((parameter_context for parameter_context in self.parameter_contexts if
                      parameter_context.name == parameter_context_name), None)

behave_framework/src/minifi_test_framework/steps/flow_building_steps.py

@@ -16,6 +16,7 @@
 #
 
 import logging
+import uuid
 from behave import given, step
 
 from minifi_test_framework.containers.directory import Directory
@@ -246,8 +247,18 @@ def step_impl(context: MinifiTestContext, property_name: str, processor_name: st
     processor.add_property(property_name, filtering)
 
 
+@given("the \"{property_name}\" properties of the {processor_name_one} and {processor_name_two} processors are set to the same random guid")
+def step_impl(context, property_name, processor_name_one, processor_name_two):
+    uuid_str = str(uuid.uuid4())
+    context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name_one).add_property(property_name, uuid_str)
+    context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name_two).add_property(property_name, uuid_str)
+
+
 # TLS
 def add_ssl_context_service_for_minifi(context: MinifiTestContext, cert_name: str):
+    ssl_context_service = context.get_or_create_default_minifi_container().flow_definition.get_controller_service("SSLContextService")
+    if ssl_context_service is not None:
+        return
     controller_service = ControllerService(class_name="SSLContextService", service_name="SSLContextService")
     controller_service.add_property("Client Certificate", f"/tmp/resources/{cert_name}.crt")
     controller_service.add_property("Private Key", f"/tmp/resources/{cert_name}.key")

docker/RunBehaveTests.sh

@@ -202,4 +202,5 @@ exec \
     "${docker_dir}/../extensions/opc/tests/features" \
     "${docker_dir}/../extensions/kafka/tests/features" \
     "${docker_dir}/../extensions/couchbase/tests/features" \
-    "${docker_dir}/../extensions/elasticsearch/tests/features"
+    "${docker_dir}/../extensions/elasticsearch/tests/features" \
+    "${docker_dir}/../extensions/splunk/tests/features"

docker/test/integration/cluster/ContainerStore.py

@@ -25,7 +25,6 @@
 from .containers.HttpProxyContainer import HttpProxyContainer
 from .containers.PostgreSQLServerContainer import PostgreSQLServerContainer
 from .containers.MqttBrokerContainer import MqttBrokerContainer
-from .containers.SplunkContainer import SplunkContainer
 from .containers.SyslogUdpClientContainer import SyslogUdpClientContainer
 from .containers.SyslogTcpClientContainer import SyslogTcpClientContainer
 from .containers.MinifiAsPodInKubernetesCluster import MinifiAsPodInKubernetesCluster
@@ -169,14 +168,6 @@ def acquire_container(self, context, container_name: str, engine='minifi-cpp', c
                                                                   network=self.network,
                                                                   image_store=self.image_store,
                                                                   command=command))
-        elif engine == 'splunk':
-            return self.containers.setdefault(container_name,
-                                              SplunkContainer(feature_context=feature_context,
-                                                              name=container_name,
-                                                              vols=self.vols,
-                                                              network=self.network,
-                                                              image_store=self.image_store,
-                                                              command=command))
         elif engine == "syslog-udp-client":
             return self.containers.setdefault(container_name,
                                               SyslogUdpClientContainer(

docker/test/integration/cluster/DockerTestCluster.py

@@ -31,7 +31,6 @@
 from .checkers.GcsChecker import GcsChecker
 from .checkers.PostgresChecker import PostgresChecker
 from .checkers.PrometheusChecker import PrometheusChecker
-from .checkers.SplunkChecker import SplunkChecker
 from .checkers.GrafanaLokiChecker import GrafanaLokiChecker
 from .checkers.ModbusChecker import ModbusChecker
 from .checkers.MqttHelper import MqttHelper
@@ -48,7 +47,6 @@ def __init__(self, context, feature_id):
         self.azure_checker = AzureChecker(self.container_communicator)
         self.gcs_checker = GcsChecker(self.container_communicator)
         self.postgres_checker = PostgresChecker(self.container_communicator)
-        self.splunk_checker = SplunkChecker(self.container_communicator)
         self.prometheus_checker = PrometheusChecker()
         self.grafana_loki_checker = GrafanaLokiChecker()
         self.minifi_controller_executor = MinifiControllerExecutor(self.container_communicator)
@@ -236,22 +234,6 @@ def check_azure_blob_and_snapshot_count(self, blob_and_snapshot_count, timeout_s
     def check_azure_blob_storage_is_empty(self, timeout_seconds):
         return self.azure_checker.check_azure_blob_storage_is_empty(timeout_seconds)
 
-    def check_splunk_event(self, container_name, query):
-        container_name = self.container_store.get_container_name_with_postfix(container_name)
-        return self.splunk_checker.check_splunk_event(container_name, query)
-
-    def check_splunk_event_with_attributes(self, container_name, query, attributes):
-        container_name = self.container_store.get_container_name_with_postfix(container_name)
-        return self.splunk_checker.check_splunk_event_with_attributes(container_name, query, attributes)
-
-    def enable_splunk_hec_indexer(self, container_name, hec_name):
-        container_name = self.container_store.get_container_name_with_postfix(container_name)
-        return self.splunk_checker.enable_splunk_hec_indexer(container_name, hec_name)
-
-    def enable_splunk_hec_ssl(self, container_name, splunk_cert_pem, splunk_key_pem, root_ca_cert_pem):
-        container_name = self.container_store.get_container_name_with_postfix(container_name)
-        return self.splunk_checker.enable_splunk_hec_ssl(container_name, splunk_cert_pem, splunk_key_pem, root_ca_cert_pem)
-
     def check_google_cloud_storage(self, gcs_container_name, content):
         gcs_container_name = self.container_store.get_container_name_with_postfix(gcs_container_name)
         return self.gcs_checker.check_google_cloud_storage(gcs_container_name, content)

docker/test/integration/cluster/ImageStore.py

@@ -65,8 +65,6 @@ def get_image(self, container_engine):
             image = self.__build_postgresql_server_image()
         elif container_engine == "mqtt-broker":
             image = self.__build_mqtt_broker_image()
-        elif container_engine == "splunk":
-            image = self.__build_splunk_image()
         elif container_engine == "kinesis-server":
             image = self.__build_kinesis_image()
         elif container_engine == "reverse-proxy":
@@ -295,9 +293,6 @@ def __build_mqtt_broker_image(self):
     def __build_kinesis_image(self):
         return self.__build_image_by_path(self.test_dir + "/resources/kinesis-mock", 'kinesis-server')
 
-    def __build_splunk_image(self):
-        return self.__build_image_by_path(self.test_dir + "/resources/splunk-hec", 'minifi-splunk')
-
     def __build_reverse_proxy_image(self):
         return self.__build_image_by_path(self.test_dir + "/resources/reverse-proxy", 'reverse-proxy')
 

docker/test/integration/cluster/checkers/SplunkChecker.py

@@ -64,12 +64,6 @@ def acquire_container(self, context, name, engine='minifi-cpp', command=None):
     def acquire_transient_minifi(self, context, name, engine='minifi-cpp'):
         return self.cluster.acquire_transient_minifi(context=context, name=name, engine=engine)
 
-    def start_splunk(self, context):
-        self.cluster.acquire_container(context=context, name='splunk', engine='splunk')
-        self.cluster.deploy_container(name='splunk')
-        assert self.cluster.wait_for_container_startup_to_finish('splunk') or self.cluster.log_app_output()
-        assert self.cluster.enable_splunk_hec_indexer('splunk', 'splunk_hec_token') or self.cluster.log_app_output()
-
     def start_minifi_c2_server(self, context):
         self.cluster.acquire_container(context=context, name="minifi-c2-server", engine="minifi-c2-server")
         self.cluster.deploy_container('minifi-c2-server')
@@ -303,12 +297,6 @@ def check_http_proxy_access(self, http_proxy_container_name, url):
     def check_azure_storage_server_data(self, azure_container_name, object_data):
         assert self.cluster.check_azure_storage_server_data(azure_container_name, object_data) or self.cluster.log_app_output()
 
-    def check_splunk_event(self, splunk_container_name, query):
-        assert self.cluster.check_splunk_event(splunk_container_name, query) or self.cluster.log_app_output()
-
-    def check_splunk_event_with_attributes(self, splunk_container_name, query, attributes):
-        assert self.cluster.check_splunk_event_with_attributes(splunk_container_name, query, attributes) or self.cluster.log_app_output()
-
     def check_google_cloud_storage(self, gcs_container_name, content):
         assert self.cluster.check_google_cloud_storage(gcs_container_name, content) or self.cluster.log_app_output()
 
@@ -384,9 +372,6 @@ def enable_prometheus_in_minifi(self):
     def enable_prometheus_with_ssl_in_minifi(self):
         self.cluster.enable_prometheus_with_ssl_in_minifi()
 
-    def enable_splunk_hec_ssl(self, container_name, splunk_cert_pem, splunk_key_pem, root_ca_cert_pem):
-        self.cluster.enable_splunk_hec_ssl(container_name, splunk_cert_pem, splunk_key_pem, root_ca_cert_pem)
-
     def enable_sql_in_minifi(self):
         self.cluster.enable_sql_in_minifi()