refactor: Adjusts the Compose logic

Use of entrypoint and command in compose
Sets group ID in compose

Author: Alan Christie

src/jote/compose.py

@@ -1,31 +1,42 @@
 """The Job Tester 'compose' module.
 
-This module is responsible for injecting a docker-compose file into the
+This module is responsible for injecting a 'docker-compose.yml' file into the
 repository of the Data Manager Job repository under test. It also
-executes docker-compose and can remove the test directory.
+created project and instance directories, and executes 'docker-compose up'
+to run the Job, and can remove the test directory.
+
+This module is designed to simulate the actions of the Data Manager
+and Job Operator that are running in the DM kubernetes deployment.
 """
 import copy
 import os
 import shutil
 import subprocess
 from typing import Any, Dict, Optional, Tuple
 
+# The 'simulated' instance directory,
+# created by the Data Manager prior to launching the corresponding Job.
+# Jobs know this directory because their container has this set via
+# the environment variable 'DM_INSTANCE_DIRECTORY'.
 INSTANCE_DIRECTORY: str = ".instance-88888888-8888-8888-8888-888888888888"
 
-# A default test execution timeout
+# A default test execution timeout (minutes)
 DEFAULT_TEST_TIMEOUT_M: int = 10
 
-# The user id containers will be started as
-_USER_ID: int = 8888
-
+# The docker-compose file template.
+# A multi-line string with variable mapping,
+# expanded and written to the test directory in 'create()'.
 _COMPOSE_CONTENT: str = """---
+# We use compose v2
+# because we're relying on 'mem_limit' and 'cpus',
+# which are ignored (moved to swarm) in v3.
 version: '2.4'
 services:
   job:
     image: {image}
     container_name: {job}-{test}-jote
-    user: '{uid}'
-    entrypoint: {working_directory}/{instance_directory}/command
+    user: '{uid}:{gid}'
+    entrypoint: {command}
     command: []
     working_dir: {working_directory}
     volumes:
@@ -35,8 +46,7 @@
     cpus: {cpus}.0
     environment:
     - DM_INSTANCE_DIRECTORY={instance_directory}
-{additional_environment}
-"""
+{additional_environment}"""
 
 _NF_CONFIG_CONTENT: str = """
 docker.enabled = true
@@ -84,6 +94,7 @@ def __init__(
         command: str,
         test_environment: Dict[str, str],
         user_id: Optional[int] = None,
+        group_id: Optional[int] = None,
     ):
 
         # Memory must have a Mi or Gi suffix.
@@ -105,6 +116,7 @@ def __init__(
         self._command: str = command
         self._test_environment = copy.deepcopy(test_environment)
         self._user_id: Optional[int] = user_id
+        self._group_id: Optional[int] = group_id
 
     def get_test_path(self) -> str:
         """Returns the path to the root directory for a given test."""
@@ -122,7 +134,7 @@ def create(self) -> str:
         full path to the test (project) directory.
         """
 
-        print("# Creating test environment...")
+        print("# Compose: Creating test environment...")
 
         # First, delete
         test_path: str = self.get_test_path()
@@ -132,7 +144,7 @@ def create(self) -> str:
         # Do we have the docker-compose version the user's installed?
         if not Compose._COMPOSE_VERSION:
             Compose._COMPOSE_VERSION = _get_docker_compose_version()
-            print(f"# docker-compose ({Compose._COMPOSE_VERSION})")
+            print(f"# Compose: docker-compose ({Compose._COMPOSE_VERSION})")
 
         # Make the test directory
         # (where the test is launched from)
@@ -143,25 +155,31 @@ def create(self) -> str:
         if not os.path.exists(inst_path):
             os.makedirs(inst_path)
 
-        # Run as a specific user ID?
+        # Run as a specific user/group ID?
         if self._user_id is not None:
             user_id = self._user_id
         else:
             user_id = os.getuid()
+        if self._group_id is not None:
+            group_id = self._group_id
+        else:
+            group_id = os.getgid()
 
         # Write the Docker compose content to a file in the test directory
         additional_environment: str = ""
         if self._test_environment:
             for e_name, e_value in self._test_environment.items():
                 additional_environment += f"    - {e_name}='{e_value}'\n"
         variables: Dict[str, Any] = {
+            "command": self._command,
             "test_path": project_path,
             "job": self._job,
             "test": self._test,
             "image": self._image,
             "memory_limit": self._memory,
             "cpus": self._cores,
             "uid": user_id,
+            "gid": group_id,
             "project_directory": self._project_directory,
             "working_directory": self._working_directory,
             "instance_directory": INSTANCE_DIRECTORY,
@@ -172,14 +190,6 @@ def create(self) -> str:
         with open(compose_path, "wt", encoding="UTF-8") as compose_file:
             compose_file.write(compose_content)
 
-        # Now write the command to the instance directory, as `command`.
-        command_path: str = f"{inst_path}/command"
-        with open(command_path, "wt", encoding="UTF-8") as command_file:
-            command_file.write("#!/bin/sh\numask 0002\n")
-            command_file.write(self._command + "\n")
-        # And set read/write/execute permission...
-        os.chmod(command_path, 0o775)
-
         # nextflow config?
         if self._image_type == "nextflow":
             # Write a nextflow config to the project path
@@ -189,7 +199,7 @@ def create(self) -> str:
             with open(nf_cfg_path, "wt", encoding="UTF-8") as nf_cfg_file:
                 nf_cfg_file.write(_NF_CONFIG_CONTENT)
 
-        print("# Created")
+        print("# Compose: Created")
 
         return project_path
 
@@ -204,8 +214,8 @@ def run(
 
         execution_directory: str = self.get_test_path()
 
-        print('# Executing the test ("docker-compose up")...')
-        print(f'# Execution directory is "{execution_directory}"')
+        print('# Compose: Executing the test ("docker-compose up")...')
+        print(f'# Compose: Execution directory is "{execution_directory}"')
 
         cwd = os.getcwd()
         os.chdir(execution_directory)
@@ -234,16 +244,16 @@ def run(
         finally:
             os.chdir(cwd)
 
-        print(f"# Executed (exit code {test.returncode})")
+        print(f"# Compose: Executed (exit code {test.returncode})")
 
         return test.returncode, test.stdout.decode("utf-8"), test.stderr.decode("utf-8")
 
     def delete(self) -> None:
         """Deletes a test directory created by 'create()'."""
-        print("# Deleting the test...")
+        print("# Compose: Deleting the test...")
 
         test_path: str = self.get_test_path()
         if os.path.exists(test_path):
             shutil.rmtree(test_path)
 
-        print("# Deleted")
+        print("# Compose: Deleted")