Uses of compose file v2.4

- Test images now given explicit names
- Support for image memory/cores
- Now displays compose stderr on failure

Author: Alan Christie

jote/compose.py

@@ -12,22 +12,20 @@
 _INSTANCE_DIRECTORY: str = '.instance-88888888-8888-8888-8888-888888888888'
 
 _COMPOSE_CONTENT: str = """---
-version: '3.8'
+version: '2.4'
 services:
   job:
     image: {image}
-    user: '{uid}:{gid}'
+    container_name: {job}-{test}-jote
+    user: '{uid}'
     command: {command}
     working_dir: {working_directory}
     environment:
     - DM_INSTANCE_DIRECTORY={instance_directory}
     volumes:
     - {test_path}:{project_directory}
-    deploy:
-      resources:
-        limits:
-          cpus: 1
-          memory: 1G
+    mem_limit: {memory_limit}
+    cpus: {cpus}.0
 """
 
 # A default, 30 minute timeout
@@ -64,17 +62,28 @@ def __init__(self, collection: str,
                  job: str,
                  test: str,
                  image: str,
+                 memory: str,
+                 cores: int,
                  project_directory: str,
                  working_directory: str,
                  command: str):
 
-        self._collection = collection
-        self._job = job
-        self._test = test
-        self._image = image
-        self._project_directory = project_directory
-        self._working_directory = working_directory
-        self._command = command
+        # Memory must have a Mi or Gi suffix.
+        # For docker-compose we translate to 'm' and 'g'
+        if memory.endswith('Mi'):
+            self._memory: str = f'{memory[:-2]}m'
+        elif memory.endswith('Gi'):
+            self._memory: str = f'{memory[:-2]}g'
+        assert self._memory
+
+        self._collection: str = collection
+        self._job: str = job
+        self._test: str = test
+        self._image: str = image
+        self._cores: int = cores
+        self._project_directory: str = project_directory
+        self._working_directory: str = working_directory
+        self._command: str = command
 
     def get_test_path(self) -> str:
         """Returns the path to the root directory for a given test.
@@ -113,15 +122,19 @@ def create(self) -> str:
         if not os.path.exists(inst_path):
             os.makedirs(inst_path)
 
-        # Write the Docker compose content to a file to the test directory
-        variables: Dict[str, Any] = {'test_path': project_path,
-                                     'image': self._image,
-                                     'uid': os.geteuid(),
-                                     'gid': os.getegid(),
-                                     'command': self._command,
-                                     'project_directory': self._project_directory,
-                                     'working_directory': self._working_directory,
-                                     'instance_directory': _INSTANCE_DIRECTORY}
+        # Write the Docker compose content to a file in the test directory
+        variables: Dict[str, Any] =\
+            {'test_path': project_path,
+             'job': self._job,
+             'test': self._test,
+             'image': self._image,
+             'memory_limit': self._memory,
+             'cpus': self._cores,
+             'uid': os.geteuid(),
+             'command': self._command,
+             'project_directory': self._project_directory,
+             'working_directory': self._working_directory,
+             'instance_directory': _INSTANCE_DIRECTORY}
         compose_content: str = _COMPOSE_CONTENT.format(**variables)
         compose_path: str = f'{test_path}/docker-compose.yml'
         with open(compose_path, 'wt', encoding='UTF-8') as compose_file:

jote/jote.py

@@ -271,6 +271,12 @@ def _test(args: argparse.Namespace,
     tests_failed: int = 0
 
     job_image: str = f'{job_definition.image.name}:{job_definition.image.tag}'
+    job_image_memory: str = job_definition.image['memory']
+    if job_image_memory is None:
+        job_image_memory = '1Gi'
+    job_image_cores: int = job_definition.image['cores']
+    if job_image_cores is None:
+        job_image_cores = 1
     job_project_directory: str = job_definition.image['project-directory']
     job_working_directory: str = job_definition.image['working-directory']
 
@@ -396,6 +402,8 @@ def _test(args: argparse.Namespace,
                                 job,
                                 job_test_name,
                                 job_image,
+                                job_image_memory,
+                                job_image_cores,
                                 job_project_directory,
                                 job_working_directory,
                                 job_command)
@@ -413,7 +421,7 @@ def _test(args: argparse.Namespace,
         if test_status and not args.dry_run:
             # Run the container
             assert t_compose
-            exit_code, out, _ = t_compose.run()
+            exit_code, out, err = t_compose.run()
 
             # Delete the test directory?
             # Not if there's an error
@@ -425,8 +433,10 @@ def _test(args: argparse.Namespace,
                 print('! FAILURE')
                 print(f'! exit_code={exit_code}'
                       f' expected_exit_code={expected_exit_code}')
-                print('! Container output follows...')
+                print('! Container stdout follows...')
                 print(out)
+                print('! Container stderr follows...')
+                print(err)
                 test_status = False
 
             if args.verbose:

requirements.txt

@@ -1,5 +1,5 @@
 docker-compose == 1.29.2
-im-data-manager-job-decoder == 1.3.0
+im-data-manager-job-decoder == 1.4.1
 munch == 2.5.0
 pyyaml == 5.4.1
 yamllint == 1.26.3