fix: Use of decoder 1.12.2

Now copies nextflow config into place
Fixed network name
Removing group compose deletes orphans
Sleeps for 10 seconds after running the group compose file
Fix -k behaviour for group files

Author: Alan Christie

requirements.txt

@@ -1,5 +1,5 @@
 docker-compose == 1.29.2
-im-data-manager-job-decoder == 1.12.1
+im-data-manager-job-decoder == 1.12.2
 munch == 2.5.0
 pyyaml == 5.4.1
 yamllint == 1.28.0

src/jote/compose.py

@@ -12,6 +12,7 @@
 import os
 import shutil
 import subprocess
+import time
 from typing import Any, Dict, Optional, Tuple
 
 # The 'simulated' instance directory,
@@ -226,10 +227,16 @@ def run(
 
         try:
             # Run the container
-            # and then cleanup
+            # and then cleanup.
+            # By using '-p' ('--project-name')
+            # we set the prefix for the network name and can use compose files
+            # from different directories. Without this the network name
+            # is prefixed by the directory the compose file is in.
             test = subprocess.run(
                 [
                     "docker-compose",
+                    "-p",
+                    "data-manager",
                     "up",
                     "--exit-code-from",
                     "job",
@@ -267,12 +274,18 @@ def run_group_compose_file(compose_file: str) -> bool:
         """Runs a group compose file in a detached state.
         The file is expected to be resident in the 'data-manager' directory."""
         try:
-            # Bring the compose file up...
+            # Bring the group-test compose file up.
+            # By using '-p' ('--project-name')
+            # we set the prefix for the network name and services from this container
+            # are visible to the test container. Without this the network name
+            # is prefixed by the directory the compose file is in.
             _ = subprocess.run(
                 [
                     "docker-compose",
                     "-f",
                     os.path.join("data-manager", compose_file),
+                    "-p",
+                    "data-manager",
                     "up",
                     "-d",
                 ],
@@ -282,6 +295,10 @@ def run_group_compose_file(compose_file: str) -> bool:
         except:  # pylint: disable=bare-except
             return False
 
+        # Wait for 10 seconds.
+        # Giving the container
+        time.sleep(10)
+
         return True
 
     @staticmethod
@@ -296,6 +313,7 @@ def stop_group_compose_file(compose_file: str) -> bool:
                     "-f",
                     os.path.join("data-manager", compose_file),
                     "down",
+                    "--remove-orphans",
                 ],
                 capture_output=False,
                 timeout=240,

src/jote/jote.py

@@ -475,14 +475,19 @@ def _check(
 
 
 def _run_nextflow(
-    command: str, project_path: str, timeout_minutes: int = DEFAULT_TEST_TIMEOUT_M
+    command: str,
+    project_path: str,
+    nextflow_config_file: str,
+    timeout_minutes: int = DEFAULT_TEST_TIMEOUT_M,
 ) -> Tuple[int, str, str]:
     """Runs nextflow in the project directory returning the exit code,
     stdout and stderr.
     """
     assert command
     assert project_path
 
+    print('# Executing the test ("nextflow")...')
+
     # The user cannot have a nextflow config in their home directory.
     # Nextflow looks here and any config will be merged with the test config.
     if _USR_HOME:
@@ -496,7 +501,17 @@ def _run_nextflow(
             print("! You cannot test Jobs and have your own config file")
             return 1, "", ""
 
-    print('# Executing the test ("nextflow")...')
+    # Is there a Nextflow config file defined for this test?
+    # It's a file in the 'data-manager' directory.
+    if nextflow_config_file:
+        print(
+            f'# Copying nextflow config file ("{nextflow_config_file}") to {project_path}'
+        )
+        shutil.copyfile(
+            os.path.join("data-manager", nextflow_config_file),
+            os.path.join(project_path, "nextflow.config"),
+        )
+
     print(f'# Execution directory is "{project_path}"')
 
     cwd = os.getcwd()
@@ -733,8 +748,16 @@ def _run_a_test(
             # Run nextflow directly
             assert job_command
             assert project_path
+
+            # Is there a nextflow config file for this test?
+            nextflow_config_file: str = ""
+            if "nextflow-config-file" in job_definition.tests[job_test_name]:
+                nextflow_config_file = job_definition.tests[job_test_name][
+                    "nextflow-config-file"
+                ]
+
             exit_code, out, err = _run_nextflow(
-                job_command, project_path, timeout_minutes
+                job_command, project_path, nextflow_config_file, timeout_minutes
             )
         else:
             print("! FAILURE")
@@ -971,7 +994,7 @@ def _run_grouped_tests(
 
                 # Always try and teardown the test compose
                 # between tests in a group.
-                if compose:
+                if compose and not args.keep_results:
                     compose.delete()
 
                 # And stop if any test has failed.