MINIFICPP-2687 Move Python extension docker tests to modular docker tests

Author: lordgamez

behave_framework/src/minifi_test_framework/containers/minifi_container.py

@@ -147,9 +147,11 @@ def _fill_default_properties(self):
         if self.is_fhs:
             self.properties["nifi.flow.configuration.file"] = "/etc/nifi-minifi-cpp/config.yml"
             self.properties["nifi.extension.path"] = "/usr/lib64/nifi-minifi-cpp/extensions/*"
+            self.properties["nifi.python.processor.dir"] = '/var/lib/nifi-minifi-cpp/minifi-python'
         else:
             self.properties["nifi.flow.configuration.file"] = "./conf/config.yml"
             self.properties["nifi.extension.path"] = "../extensions/*"
+            self.properties["nifi.python.processor.dir"] = '/opt/minifi/minifi-current/minifi-python'
         self.properties["nifi.administrative.yield.duration"] = "1 sec"
         self.properties["nifi.bored.yield.duration"] = "100 millis"
         self.properties["nifi.openssl.fips.support.enable"] = "false"
@@ -248,3 +250,13 @@ def create_debug_bundle(self) -> bool:
 
         (code, _) = self.exec_run(["test", "-f", "/tmp/debug.tar.gz"])
         return code == 0
+
+    def add_example_python_processors(self):
+        run_minifi_cmd = '/opt/minifi/minifi-current/bin/minifi.sh run' if not self.is_fhs else '/usr/bin/minifi'
+        minifi_python_dir_path = '/opt/minifi/minifi-current/minifi-python' if not self.is_fhs else '/var/lib/nifi-minifi-cpp/minifi-python'
+        minifi_python_examples = '/opt/minifi/minifi-current/minifi-python-examples' if not self.is_fhs else '/usr/share/doc/nifi-minifi-cpp/pythonprocessor-examples'
+        self.command = f'sh -c "cp -r {minifi_python_examples} {minifi_python_dir_path}/examples && {run_minifi_cmd}"'
+
+    def set_default_python_virtualenv_directory(self):
+        minifi_python_dir_path = '/opt/minifi/minifi-current/minifi-python' if not self.is_fhs else '/var/lib/nifi-minifi-cpp/minifi-python'
+        self.properties["nifi.python.processor.dir"] = minifi_python_dir_path

behave_framework/src/minifi_test_framework/steps/checking_steps.py

@@ -44,6 +44,7 @@ def step_impl(context: MinifiTestContext, container_name: str, content: str, dir
 
 
 @then('a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}')
+@then("a single file with the content '{content}' is placed in the '{directory}' directory in less than {duration}")
 def step_impl(context: MinifiTestContext, content: str, directory: str, duration: str):
     context.execute_steps(f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}')
 
@@ -171,6 +172,16 @@ def step_impl(context: MinifiTestContext, directory: str, timeout: str, contents
                               context=context)
 
 
+@then('files with at least these contents "{contents}" are placed in the "{directory}" directory in less than {timeout}')
+def step_impl(context: MinifiTestContext, directory: str, timeout: str, contents: str):
+    timeout_seconds = humanfriendly.parse_timespan(timeout)
+    new_contents = contents.replace("\\n", "\n")
+    contents_arr = new_contents.split(",")
+    assert wait_for_condition(condition=lambda: all([context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_file_with_content(directory, content) for content in contents_arr]),
+                              timeout_seconds=timeout_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited,
+                              context=context)
+
+
 @then("a file with the JSON content \"{content}\" is placed in the \"{directory}\" directory in less than {duration}")
 @then("a file with the JSON content '{content}' is placed in the '{directory}' directory in less than {duration}")
 def step_impl(context: MinifiTestContext, content: str, directory: str, duration: str):

behave_framework/src/minifi_test_framework/steps/flow_building_steps.py

@@ -70,16 +70,10 @@ def step_impl(context: MinifiTestContext, processor_type: str, property_name: st
     context.containers["nifi"].flow_definition.add_processor(processor)
 
 
-@given('a {processor_type} processor with the name "{processor_name}" in the "{minifi_container_name}" flow')
-def step_impl(context: MinifiTestContext, processor_type: str, processor_name: str, minifi_container_name: str):
-    processor = Processor(processor_type, processor_name)
-    context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_processor(processor)
-
-
 @given('a {processor_type} processor with the name "{processor_name}"')
 def step_impl(context: MinifiTestContext, processor_type: str, processor_name: str):
-    context.execute_steps(
-        f'given a {processor_type} processor with the name "{processor_name}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow')
+    processor = Processor(processor_type, processor_name)
+    context.get_or_create_default_minifi_container().flow_definition.add_processor(processor)
 
 
 @given("a {processor_type} processor in the \"{minifi_container_name}\" flow")
@@ -99,17 +93,6 @@ def step_impl(context: MinifiTestContext, processor_type: str):
     context.execute_steps(f'given a {processor_type} processor in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow')
 
 
-@given('the "{property_name}" property of the "{port_name}" port in the "{rpg_name}" remote process group is set to "{property_value}"')
-def step_impl(context: MinifiTestContext, property_name: str, port_name: str, rpg_name: str, property_value: str):
-    rpg = context.get_or_create_minifi_container(DEFAULT_MINIFI_CONTAINER_NAME).flow_definition.get_remote_process_group(rpg_name)
-    if not rpg:
-        raise ValueError(f"Remote Process Group with name {rpg_name} not found")
-    port = rpg.get_input_port(port_name)
-    if not port:
-        raise ValueError(f"Remote input port with name {port_name} not found in RPG {rpg_name}")
-    port.add_property(property_name, property_value)
-
-
 @given('the "{property_name}" property of the {processor_name} processor is set to "{property_value}" in the "{minifi_container_name}" flow')
 def step_impl(context: MinifiTestContext, property_name: str, processor_name: str, property_value: str, minifi_container_name: str):
     processor = context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(processor_name)
@@ -149,7 +132,6 @@ def step_impl(context: MinifiTestContext, funnel_name: str):
 
 
 @step('in the "{minifi_container_name}" flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}')
-@step('in the "{minifi_container_name}" flow the "{relationship_name}" relationship of the {source} node is connected to the {target}')
 def step_impl(context: MinifiTestContext, relationship_name: str, source: str, target: str, minifi_container_name: str):
     connection = Connection(source_name=source, source_relationship=relationship_name, target_name=target)
     context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_connection(connection)
@@ -162,17 +144,10 @@ def step_impl(context: MinifiTestContext, relationship_name: str, source: str, t
 
 
 @step('the "{relationship_name}" relationship of the {source} processor is connected to the {target}')
-@step('the "{relationship_name}" relationship of the {source} node is connected to the {target}')
 def step_impl(context: MinifiTestContext, relationship_name: str, source: str, target: str):
     context.execute_steps(f'given in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}')
 
 
-@step("the output port \"{port_name}\" is connected to the {destination_name} processor")
-def step_impl(context: MinifiTestContext, port_name: str, destination_name: str):
-    connection = Connection(source_name=port_name, source_relationship="undefined", target_name=destination_name)
-    context.get_or_create_minifi_container(DEFAULT_MINIFI_CONTAINER_NAME).flow_definition.add_connection(connection)
-
-
 @step('the Funnel with the name "{funnel_name}" is connected to the {target}')
 def step_impl(context: MinifiTestContext, funnel_name: str, target: str):
     connection = Connection(source_name=funnel_name, source_relationship="success", target_name=target)
@@ -370,50 +345,3 @@ def step_impl(context: MinifiTestContext, processor_name):
     add_ssl_context_service_for_minifi(context, "minifi_client", use_system_cert_store=True)
     processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name)
     processor.add_property('SSL Context Service', 'SSLContextService')
-
-
-@given("a RemoteProcessGroup node with name \"{rpg_name}\" is opened on \"{address}\" with transport protocol set to \"{transport_protocol}\"")
-def step_impl(context: MinifiTestContext, rpg_name: str, address: str, transport_protocol: str):
-    context.get_or_create_default_minifi_container().flow_definition.add_remote_process_group(address, rpg_name, transport_protocol)
-
-
-@given("a RemoteProcessGroup node with name \"{rpg_name}\" is opened on \"{address}\"")
-def step_impl(context: MinifiTestContext, rpg_name: str, address: str):
-    context.execute_steps(f"given a RemoteProcessGroup node with name \"{rpg_name}\" is opened on \"{address}\" with transport protocol set to \"RAW\"")
-
-
-@given("an input port with name \"{port_name}\" is created on the RemoteProcessGroup named \"{rpg_name}\"")
-def step_impl(context: MinifiTestContext, port_name: str, rpg_name: str):
-    context.get_or_create_default_minifi_container().flow_definition.add_input_port_to_rpg(rpg_name, port_name)
-
-
-@given("an input port using compression with name \"{port_name}\" is created on the RemoteProcessGroup named \"{rpg_name}\"")
-def step_impl(context: MinifiTestContext, port_name: str, rpg_name: str):
-    context.get_or_create_default_minifi_container().flow_definition.add_input_port_to_rpg(rpg_name, port_name, use_compression=True)
-
-
-@given("an output port with name \"{port_name}\" is created on the RemoteProcessGroup named \"{rpg_name}\"")
-def step_impl(context: MinifiTestContext, port_name: str, rpg_name: str):
-    context.get_or_create_default_minifi_container().flow_definition.add_output_port_to_rpg(rpg_name, port_name)
-
-
-@given("an output port using compression with name \"{port_name}\" is created on the RemoteProcessGroup named \"{rpg_name}\"")
-def step_impl(context: MinifiTestContext, port_name: str, rpg_name: str):
-    context.get_or_create_default_minifi_container().flow_definition.add_output_port_to_rpg(rpg_name, port_name, use_compression=True)
-
-
-@given("a NiFi flow is receiving data from the RemoteProcessGroup named \"{rpg_name}\" in an input port named \"{input_port_name}\" which has the same id as the port named \"{rpg_port_name}\"")
-def step_impl(context: MinifiTestContext, input_port_name: str, rpg_port_name: str, rpg_name: str):
-    input_port_id = context.get_or_create_default_minifi_container().flow_definition.get_input_port_id_of_rpg(rpg_name, rpg_port_name)
-    context.containers["nifi"].flow_definition.add_input_port(input_port_id, input_port_name)
-
-
-@given("a NiFi flow is sending data to an output port named \"{port_name}\" with the id of the port named \"{rpg_port_name}\" from the RemoteProcessGroup named \"{rpg_name}\"")
-def step_impl(context: MinifiTestContext, port_name: str, rpg_port_name: str, rpg_name: str):
-    output_port_id = context.get_or_create_default_minifi_container().flow_definition.get_output_port_id_of_rpg(rpg_name, rpg_port_name)
-    context.containers["nifi"].flow_definition.add_output_port(output_port_id, port_name)
-
-
-@given("the connection going to {destination} has \"drop empty\" set")
-def step_impl(context: MinifiTestContext, destination: str):
-    context.get_or_create_default_minifi_container().flow_definition.set_drop_empty_for_destination(destination)

docker/RunBehaveTests.sh

@@ -209,4 +209,5 @@ exec \
     "${docker_dir}/../extensions/lua/tests/features/" \
     "${docker_dir}/../extensions/civetweb/tests/features/" \
     "${docker_dir}/../extensions/mqtt/tests/features/" \
-    "${docker_dir}/../extensions/prometheus/tests/features/"
+    "${docker_dir}/../extensions/prometheus/tests/features/" \
+    "${docker_dir}/../extensions/python/tests/features/"