MINIFICPP-2674 Move Couchbase tests to modular docker tests

Closes #2066

Signed-off-by: Ferenc Gerlits <[email protected]>

Author: lordgamez

behave_framework/src/minifi_test_framework/containers/minifi_container.py

@@ -21,7 +21,7 @@
 from minifi_test_framework.core.minifi_test_context import MinifiTestContext
 from minifi_test_framework.containers.file import File
 from minifi_test_framework.minifi.flow_definition import FlowDefinition
-from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage
+from minifi_test_framework.core.ssl_utils import make_cert_without_extended_usage, make_client_cert
 from .container import Container
 
 
@@ -38,6 +38,10 @@ def __init__(self, container_name: str, test_context: MinifiTestContext):
         self.files.append(File("/tmp/resources/minifi_client.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=minifi_client_cert)))
         self.files.append(File("/tmp/resources/minifi_client.key", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=minifi_client_key)))
 
+        clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
+        self.files.append(File("/tmp/resources/clientuser.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=clientuser_cert)))
+        self.files.append(File("/tmp/resources/clientuser.key", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=clientuser_key)))
+
         self.is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(self.client.images.get(test_context.minifi_container_image).history())
 
         self._fill_default_properties()

behave_framework/src/minifi_test_framework/core/helpers.py

@@ -19,6 +19,7 @@
 
 import logging
 import time
+import functools
 from collections.abc import Callable
 
 import docker
@@ -82,3 +83,21 @@ def run_cmd_in_docker_image(image_name: str, cmd: str | list, network: str) -> s
 
 def run_shell_cmd_in_docker_image(image_name: str, cmd: str, network: str) -> str:
     return run_cmd_in_docker_image(image_name, ["/bin/sh", "-c", cmd], network)
+
+
+def retry_check(max_tries: int = 5, retry_interval_seconds: int = 1):
+    """
+    Decorator for retrying a checker function that returns a boolean. The decorated function is called repeatedly until it returns True
+    or the maximum number of attempts is reached. The maximum number of attempts and the interval between attempts in seconds can be configured.
+    """
+    def retry_check_func(func):
+        @functools.wraps(func)
+        def retry_wrapper(*args, **kwargs):
+            for i in range(max_tries):
+                if func(*args, **kwargs):
+                    return True
+                if i < max_tries - 1:
+                    time.sleep(retry_interval_seconds)
+            return False
+        return retry_wrapper
+    return retry_check_func

behave_framework/src/minifi_test_framework/steps/flow_building_steps.py

@@ -29,6 +29,11 @@
 from minifi_test_framework.minifi.processor import Processor
 
 
+@given("a MiNiFi CPP server with yaml config")
+def step_impl(context: MinifiTestContext):
+    pass  # TODO(lordgamez): Needs to be implemented after JSON config is set to be default
+
+
 @given("a transient MiNiFi flow with a LogOnDestructionProcessor processor")
 def step_impl(context: MinifiTestContext):
     context.get_or_create_default_minifi_container().command = ["/bin/sh", "-c", "timeout 10s ./bin/minifi.sh run && sleep 100"]
@@ -140,6 +145,7 @@ def step_impl(context: MinifiTestContext, parameter_name: str, parameter_value:
 
 
 @step('a directory at "{directory}" has a file with the content "{content}" in the "{flow_name}" flow')
+@step("a directory at '{directory}' has a file with the content '{content}' in the '{flow_name}' flow")
 def step_impl(context: MinifiTestContext, directory: str, content: str, flow_name: str):
     new_content = content.replace("\\n", "\n")
     new_dir = Directory(directory)
@@ -148,6 +154,7 @@ def step_impl(context: MinifiTestContext, directory: str, content: str, flow_nam
 
 
 @step('a directory at "{directory}" has a file with the content "{content}"')
+@step("a directory at '{directory}' has a file with the content '{content}'")
 def step_impl(context: MinifiTestContext, directory: str, content: str):
     context.execute_steps(f'given a directory at "{directory}" has a file with the content "{content}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow')
 
@@ -240,14 +247,22 @@ def step_impl(context: MinifiTestContext, property_name: str, processor_name: st
 
 
 # TLS
-@given("an ssl context service is set up for {processor_name}")
-@given("an ssl context service with a manual CA cert file is set up for {processor_name}")
-def step_impl(context, processor_name):
+def add_ssl_context_service_for_minifi(context: MinifiTestContext, cert_name: str):
     controller_service = ControllerService(class_name="SSLContextService", service_name="SSLContextService")
-    controller_service.add_property("Client Certificate", "/tmp/resources/minifi_client.crt")
-    controller_service.add_property("Private Key", "/tmp/resources/minifi_client.key")
+    controller_service.add_property("Client Certificate", f"/tmp/resources/{cert_name}.crt")
+    controller_service.add_property("Private Key", f"/tmp/resources/{cert_name}.key")
     controller_service.add_property("CA Certificate", "/tmp/resources/root_ca.crt")
     context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service)
 
+
+@given("an ssl context service is set up")
+def step_impl(context: MinifiTestContext):
+    add_ssl_context_service_for_minifi(context, "minifi_client")
+
+
+@given("an ssl context service is set up for {processor_name}")
+@given("an ssl context service with a manual CA cert file is set up for {processor_name}")
+def step_impl(context, processor_name):
+    add_ssl_context_service_for_minifi(context, "minifi_client")
     processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name)
     processor.add_property('SSL Context Service', 'SSLContextService')

docker/RunBehaveTests.sh

@@ -200,4 +200,5 @@ exec \
     "${docker_dir}/../extensions/sql/tests/features" \
     "${docker_dir}/../extensions/llamacpp/tests/features" \
     "${docker_dir}/../extensions/opc/tests/features" \
-    "${docker_dir}/../extensions/kafka/tests/features"
+    "${docker_dir}/../extensions/kafka/tests/features" \
+    "${docker_dir}/../extensions/couchbase/tests/features"

docker/requirements.txt

@@ -10,5 +10,4 @@ azure-storage-blob==12.24.1
 prometheus-api-client==0.5.5
 humanfriendly==10.0
 requests<2.29  # https://github.com/docker/docker-py/issues/3113
-couchbase==4.3.5
 paho-mqtt==2.1.0

docker/test/integration/cluster/ContainerStore.py

@@ -36,7 +36,6 @@
 from .containers.GrafanaLokiContainer import GrafanaLokiContainer
 from .containers.GrafanaLokiContainer import GrafanaLokiOptions
 from .containers.ReverseProxyContainer import ReverseProxyContainer
-from .containers.CouchbaseServerContainer import CouchbaseServerContainer
 from .FeatureContext import FeatureContext
 
 
@@ -266,14 +265,6 @@ def acquire_container(self, context, container_name: str, engine='minifi-cpp', c
                                                                     network=self.network,
                                                                     image_store=self.image_store,
                                                                     command=command))
-        elif engine == "couchbase-server":
-            return self.containers.setdefault(container_name,
-                                              CouchbaseServerContainer(feature_context=feature_context,
-                                                                       name=container_name,
-                                                                       vols=self.vols,
-                                                                       network=self.network,
-                                                                       image_store=self.image_store,
-                                                                       command=command))
         else:
             raise Exception('invalid flow engine: \'%s\'' % engine)
 

docker/test/integration/cluster/DockerTestCluster.py

@@ -35,7 +35,6 @@
 from .checkers.SplunkChecker import SplunkChecker
 from .checkers.GrafanaLokiChecker import GrafanaLokiChecker
 from .checkers.ModbusChecker import ModbusChecker
-from .checkers.CouchbaseChecker import CouchbaseChecker
 from .checkers.MqttHelper import MqttHelper
 from utils import get_peak_memory_usage, get_minifi_pid, get_memory_usage, retry_check
 
@@ -56,7 +55,6 @@ def __init__(self, context, feature_id):
         self.grafana_loki_checker = GrafanaLokiChecker()
         self.minifi_controller_executor = MinifiControllerExecutor(self.container_communicator)
         self.modbus_checker = ModbusChecker(self.container_communicator)
-        self.couchbase_checker = CouchbaseChecker()
         self.mqtt_helper = MqttHelper()
 
     def cleanup(self):
@@ -447,8 +445,5 @@ def set_value_on_plc_with_modbus(self, container_name, modbus_cmd):
     def enable_ssl_in_nifi(self):
         self.container_store.enable_ssl_in_nifi()
 
-    def is_data_present_in_couchbase(self, doc_id: str, bucket_name: str, expected_data: str, expected_data_type: str):
-        return self.couchbase_checker.is_data_present_in_couchbase(doc_id, bucket_name, expected_data, expected_data_type)
-
     def publish_test_mqtt_message(self, topic: str, message: str):
         self.mqtt_helper.publish_test_mqtt_message(topic, message)

docker/test/integration/cluster/checkers/CouchbaseChecker.py

@@ -85,11 +85,6 @@ def start_minifi_c2_server(self, context):
         self.cluster.deploy_container('minifi-c2-server')
         assert self.cluster.wait_for_container_startup_to_finish('minifi-c2-server') or self.cluster.log_app_output()
 
-    def start_couchbase_server(self, context):
-        self.cluster.acquire_container(context=context, name='couchbase-server', engine='couchbase-server')
-        self.cluster.deploy_container('couchbase-server')
-        assert self.cluster.wait_for_container_startup_to_finish('couchbase-server') or self.cluster.log_app_output()
-
     def start_nifi(self, context):
         self.cluster.acquire_container(context=context, name='nifi', engine='nifi')
         self.cluster.deploy_container('nifi')
@@ -507,8 +502,5 @@ def set_value_on_plc_with_modbus(self, container_name, modbus_cmd):
     def enable_ssl_in_nifi(self):
         self.cluster.enable_ssl_in_nifi()
 
-    def check_is_data_present_on_couchbase(self, doc_id: str, bucket_name: str, expected_data: str, expected_data_type: str):
-        assert self.cluster.is_data_present_in_couchbase(doc_id, bucket_name, expected_data, expected_data_type)
-
     def publish_test_mqtt_message(self, topic, message):
         self.cluster.publish_test_mqtt_message(topic, message)