MINIFICPP-2677 Move GCP tests to modular docker tests

Author: lordgamez

behave_framework/src/minifi_test_framework/containers/directory.py

@@ -16,10 +16,12 @@
 #
 
 class Directory:
-    def __init__(self, path):
+    def __init__(self, path, files: dict[str, str] | None = None, mode="rw"):
         self.path = path
         self.files: dict[str, str] = {}
-        self.mode = "rw"
+        if files is not None:
+            self.files = files
+        self.mode = mode
 
     def add_file(self, file_name: str, content: str):
         self.files[file_name] = content

behave_framework/src/minifi_test_framework/steps/flow_building_steps.py

@@ -90,6 +90,15 @@ def step_impl(context: MinifiTestContext, property_name: str, processor_name: st
         processor.add_property(property_name, property_value)
 
 
+@step('the "{property_name}" property of the {controller_name} controller service is set to "{property_value}"')
+def step_impl(context: MinifiTestContext, property_name: str, controller_name: str, property_value: str):
+    controller_service = context.get_or_create_default_minifi_container().flow_definition.get_controller_service(controller_name)
+    if property_value == "(not set)":
+        controller_service.remove_property(property_name)
+    else:
+        controller_service.add_property(property_name, property_value)
+
+
 @step('a Funnel with the name "{funnel_name}" is set up')
 def step_impl(context: MinifiTestContext, funnel_name: str):
     context.get_or_create_default_minifi_container().flow_definition.add_funnel(Funnel(funnel_name))

docker/RunBehaveTests.sh

@@ -203,4 +203,5 @@ exec \
     "${docker_dir}/../extensions/kafka/tests/features" \
     "${docker_dir}/../extensions/couchbase/tests/features" \
     "${docker_dir}/../extensions/elasticsearch/tests/features" \
-    "${docker_dir}/../extensions/splunk/tests/features"
+    "${docker_dir}/../extensions/splunk/tests/features" \
+    "${docker_dir}/../extensions/gcp/tests/features"

docker/test/integration/cluster/ContainerStore.py

@@ -21,7 +21,6 @@
 from .containers.KinesisServerContainer import KinesisServerContainer
 from .containers.S3ServerContainer import S3ServerContainer
 from .containers.AzureStorageServerContainer import AzureStorageServerContainer
-from .containers.FakeGcsServerContainer import FakeGcsServerContainer
 from .containers.HttpProxyContainer import HttpProxyContainer
 from .containers.PostgreSQLServerContainer import PostgreSQLServerContainer
 from .containers.MqttBrokerContainer import MqttBrokerContainer
@@ -144,14 +143,6 @@ def acquire_container(self, context, container_name: str, engine='minifi-cpp', c
                                                                           network=self.network,
                                                                           image_store=self.image_store,
                                                                           command=command))
-        elif engine == 'fake-gcs-server':
-            return self.containers.setdefault(container_name,
-                                              FakeGcsServerContainer(feature_context=feature_context,
-                                                                     name=container_name,
-                                                                     vols=self.vols,
-                                                                     network=self.network,
-                                                                     image_store=self.image_store,
-                                                                     command=command))
         elif engine == 'postgresql-server':
             return self.containers.setdefault(container_name,
                                               PostgreSQLServerContainer(feature_context=feature_context,

docker/test/integration/cluster/DockerTestCluster.py

@@ -28,7 +28,6 @@
 from .MinifiControllerExecutor import MinifiControllerExecutor
 from .checkers.AwsChecker import AwsChecker
 from .checkers.AzureChecker import AzureChecker
-from .checkers.GcsChecker import GcsChecker
 from .checkers.PostgresChecker import PostgresChecker
 from .checkers.PrometheusChecker import PrometheusChecker
 from .checkers.GrafanaLokiChecker import GrafanaLokiChecker
@@ -45,7 +44,6 @@ def __init__(self, context, feature_id):
         self.container_store = ContainerStore(self.container_communicator.create_docker_network(feature_id), context.image_store, context.kubernetes_proxy, feature_id=feature_id)
         self.aws_checker = AwsChecker(self.container_communicator)
         self.azure_checker = AzureChecker(self.container_communicator)
-        self.gcs_checker = GcsChecker(self.container_communicator)
         self.postgres_checker = PostgresChecker(self.container_communicator)
         self.prometheus_checker = PrometheusChecker()
         self.grafana_loki_checker = GrafanaLokiChecker()
@@ -234,14 +232,6 @@ def check_azure_blob_and_snapshot_count(self, blob_and_snapshot_count, timeout_s
     def check_azure_blob_storage_is_empty(self, timeout_seconds):
         return self.azure_checker.check_azure_blob_storage_is_empty(timeout_seconds)
 
-    def check_google_cloud_storage(self, gcs_container_name, content):
-        gcs_container_name = self.container_store.get_container_name_with_postfix(gcs_container_name)
-        return self.gcs_checker.check_google_cloud_storage(gcs_container_name, content)
-
-    def is_gcs_bucket_empty(self, container_name):
-        container_name = self.container_store.get_container_name_with_postfix(container_name)
-        return self.gcs_checker.is_gcs_bucket_empty(container_name)
-
     def check_query_results(self, postgresql_container_name, query, number_of_rows, timeout_seconds):
         postgresql_container_name = self.container_store.get_container_name_with_postfix(postgresql_container_name)
         return self.postgres_checker.check_query_results(postgresql_container_name, query, number_of_rows, timeout_seconds)

docker/test/integration/cluster/checkers/GcsChecker.py

@@ -297,12 +297,6 @@ def check_http_proxy_access(self, http_proxy_container_name, url):
     def check_azure_storage_server_data(self, azure_container_name, object_data):
         assert self.cluster.check_azure_storage_server_data(azure_container_name, object_data) or self.cluster.log_app_output()
 
-    def check_google_cloud_storage(self, gcs_container_name, content):
-        assert self.cluster.check_google_cloud_storage(gcs_container_name, content) or self.cluster.log_app_output()
-
-    def check_empty_gcs_bucket(self, gcs_container_name):
-        assert self.cluster.is_gcs_bucket_empty(gcs_container_name) or self.cluster.log_app_output()
-
     def check_minifi_log_contents(self, line, timeout_seconds=60, count=1):
         self.check_container_log_contents("minifi-cpp", line, timeout_seconds, count)
 

docker/test/integration/cluster/containers/FakeGcsServerContainer.py

@@ -18,7 +18,6 @@
 from minifi.core.Funnel import Funnel
 
 from minifi.controllers.SSLContextService import SSLContextService
-from minifi.controllers.GCPCredentialsControllerService import GCPCredentialsControllerService
 from minifi.controllers.ODBCService import ODBCService
 from minifi.controllers.KubernetesControllerService import KubernetesControllerService
 from minifi.controllers.JsonRecordSetWriter import JsonRecordSetWriter
@@ -525,14 +524,6 @@ def step_impl(context, protocol):
     context.test.acquire_container(context=context, name=client_name, engine=client_name)
 
 
-# google cloud storage setup
-@given("a Google Cloud storage server is set up")
-@given("a Google Cloud storage server is set up with some test data")
-@given('a Google Cloud storage server is set up and a single object with contents "preloaded data" is present')
-def step_impl(context):
-    context.test.acquire_container(context=context, name="fake-gcs-server", engine="fake-gcs-server")
-
-
 def setUpSslContextServiceForProcessor(context, processor_name: str):
     minifi_crt_file = '/tmp/resources/minifi_client.crt'
     minifi_key_file = '/tmp/resources/minifi_client.key'
@@ -560,31 +551,6 @@ def step_impl(context):
     context.test.acquire_container(context=context, name="tcp-client", engine="tcp-client")
 
 
-@given(u'the {processor_one} processor is set up with a GCPCredentialsControllerService to communicate with the Google Cloud storage server')
-def step_impl(context, processor_one):
-    gcp_controller_service = GCPCredentialsControllerService(credentials_location="Use Anonymous credentials")
-    p1 = context.test.get_node_by_name(processor_one)
-    p1.controller_services.append(gcp_controller_service)
-    p1.set_property("GCP Credentials Provider Service", gcp_controller_service.name)
-    processor = context.test.get_node_by_name(processor_one)
-    processor.set_property("Endpoint Override URL", f"fake-gcs-server-{context.feature_id}:4443")
-
-
-@given(u'the {processor_one} and the {processor_two} processors are set up with a GCPCredentialsControllerService to communicate with the Google Cloud storage server')
-def step_impl(context, processor_one, processor_two):
-    gcp_controller_service = GCPCredentialsControllerService(credentials_location="Use Anonymous credentials")
-    p1 = context.test.get_node_by_name(processor_one)
-    p2 = context.test.get_node_by_name(processor_two)
-    p1.controller_services.append(gcp_controller_service)
-    p1.set_property("GCP Credentials Provider Service", gcp_controller_service.name)
-    p2.controller_services.append(gcp_controller_service)
-    p2.set_property("GCP Credentials Provider Service", gcp_controller_service.name)
-    processor_one = context.test.get_node_by_name(processor_one)
-    processor_one.set_property("Endpoint Override URL", f"fake-gcs-server-{context.feature_id}:4443")
-    processor_two = context.test.get_node_by_name(processor_two)
-    processor_two.set_property("Endpoint Override URL", f"fake-gcs-server-{context.feature_id}:4443")
-
-
 # SQL
 @given("an ODBCService is setup up for {processor_name} with the name \"{service_name}\"")
 def step_impl(context, processor_name, service_name):
@@ -874,17 +840,6 @@ def step_impl(context, minifi_container_name, log_pattern, duration):
     context.test.check_container_log_matches_regex(minifi_container_name, log_pattern, humanfriendly.parse_timespan(duration), count=1)
 
 
-# Google Cloud Storage
-@then('an object with the content \"{content}\" is present in the Google Cloud storage')
-def step_imp(context, content):
-    context.test.check_google_cloud_storage("fake-gcs-server", content)
-
-
-@then("the test bucket of Google Cloud Storage is empty")
-def step_impl(context):
-    context.test.check_empty_gcs_bucket("fake-gcs-server")
-
-
 # Prometheus
 @given("a Prometheus server is set up")
 def step_impl(context):