Merge pull request #600 from donny-wong/v2.7.0

V2.7.0

Author: donny-wong

.pre-commit-config.yaml

@@ -1,9 +1,9 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 24.10.0
+    rev: 25.1.0
     hooks:
     - id: black
   - repo: https://github.com/pycqa/flake8
-    rev: 7.1.1
+    rev: 7.1.2
     hooks:
     - id: flake8

Changelog.md

@@ -1,6 +1,20 @@
 # CHANGELOG
 All notable changes to this project will be documented here.
 
+## [v2.7.0]
+- Update python, pyta and jupyter testers to allow a requirements file (#580)
+- Update R tester to allow a renv.lock file (#581)
+- Improve display of Python package installation errors when creating environment (#585)
+- Update "setting up test environment" message with http response of status code 503 (#589)
+- Change rlimit resource settings to apply each worker individually (#587) 
+- Drop support for Python 3.8 (#590)
+- Use Python 3.13 in development (#590)
+- Update Docker configuration to install dependencies in a separate service (#590)
+- Improve error reporting with handled assertion errors (#591)
+- Add custom pytest markers to Python tester to record MarkUs metadata (#592)
+- Stop the autotester from running tests if there are errors in test settings (#593) 
+- Implement Redis backoff strategy (#594)
+
 ## [v2.6.0]
 - Update python versions in docker file (#568)
 - Update Github Actions config to use Python 3.11-3.13 and update action versions (#569)

README.md

@@ -196,6 +196,8 @@ redis_url: # url of the redis database. default is: redis://127.0.0.1:6379/0
 supervisor_url: # url used by the supervisor process. default is: '127.0.0.1:9001'
                 # This can also be set with the SUPERVISOR_URL environment variable.
 
+worker_log_dir: # an absolute path to a directory containing the worker's stdout and stderr logs.
+
 rlimit_settings: # RLIMIT settings (see details below)
   nproc: # for example, this setting sets the hard and soft limits for the number of processes available to 300
     - 300

client/.dockerfiles/Dockerfile

@@ -5,13 +5,7 @@ FROM ubuntu:$UBUNTU_VERSION
 RUN apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get -y install software-properties-common && \
     DEBIAN_FRONTEND=noninteractive add-apt-repository -y ppa:deadsnakes/ppa && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y python3.11 python3.11-venv
-
-COPY ./requirements.txt /requirements.txt
-
-RUN python3.11 -m venv /markus_venv && \
-    /markus_venv/bin/pip install wheel && \
-    /markus_venv/bin/pip install -r /requirements.txt
+    DEBIAN_FRONTEND=noninteractive apt-get install -y python3.13 python3.13-venv
 
 WORKDIR /app
 

client/.dockerfiles/entrypoint-dev-deps-updater.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+[ -f /markus_venv/bin/python3.13 ] || python3.13 -m venv /markus_venv
+
+/markus_venv/bin/pip install -q --upgrade pip
+/markus_venv/bin/pip install -q wheel
+printf "[MarkUs] Running pip install -q -r requirements.txt..."
+if /markus_venv/bin/pip install -q -r requirements.txt; then
+  printf " \e[32m✔\e[0m \n"
+fi
+
+# Execute the provided command
+exec "$@"

client/autotest_client/__init__.py

@@ -11,6 +11,9 @@
 import traceback
 import dotenv
 import redis
+from redis.retry import Retry
+from redis.exceptions import TimeoutError, ConnectionError
+from redis.backoff import FullJitterBackoff
 from datetime import datetime
 from contextlib import contextmanager
 
@@ -24,7 +27,12 @@
 SETTINGS_JOB_TIMEOUT = os.environ.get("SETTINGS_JOB_TIMEOUT", 600)
 REDIS_URL = os.environ["REDIS_URL"]
 
-REDIS_CONNECTION = redis.Redis.from_url(REDIS_URL)
+REDIS_CONNECTION = redis.Redis.from_url(
+    REDIS_URL,
+    retry=Retry(FullJitterBackoff(cap=10, base=1), 25),
+    retry_on_error=[ConnectionError, TimeoutError],
+    health_check_interval=1,
+)
 
 app = Flask(__name__)
 
@@ -224,7 +232,7 @@ def run_tests(settings_id, user):
     test_settings = json.loads(REDIS_CONNECTION.hget("autotest:settings", key=settings_id))
     env_status = test_settings.get("_env_status")
     if env_status == "setup":
-        raise Exception("Setting up test environment. Please try again later.")
+        abort(make_response(jsonify(message="Setting up test environment. Please try again later."), 503))
     elif env_status == "error":
         msg = "Settings Error"
         settings_error = test_settings.get("_error", "")

client/requirements.txt

@@ -1,10 +1,6 @@
-flask==2.2.5;python_version<"3.8"
-flask==3.0.3;python_version>="3.8"
-python-dotenv==0.21.1;python_version<"3.8"
-python-dotenv==1.0.1;python_version>="3.8"
-rq==2.0.0
-redis==5.2.0
-jsonschema==4.17.3;python_version<"3.8"
-jsonschema==4.23.0;python_version>="3.8"
-Werkzeug==2.2.3;python_version<"3.8"
-Werkzeug==3.1.3;python_version>="3.8"
+flask==3.1.0
+python-dotenv==1.0.1
+rq==2.1.0
+redis==5.2.1
+jsonschema==4.23.0
+Werkzeug==3.1.3

compose.yaml

@@ -1,36 +1,39 @@
 services:
-  server:
+  server: &server
     build:
       context: ./server
       dockerfile: ./.dockerfiles/Dockerfile
       args:
         UBUNTU_VERSION: '22.04'
         LOGIN_USER: 'docker'
         WORKSPACE: '/home/docker/.autotesting'
-    image: markus-autotest-server-dev:1.2.0
+    image: markus-autotest-server-dev:1.3.0
     volumes:
       - ./server:/app:cached
+      - venv_server:/home/docker/markus_venv
       - workspace:/home/docker/.autotesting:rw
     environment:
       - REDIS_URL=redis://redis:6379/
       - WORKSPACE=/home/docker/.autotesting
       - SUPERVISOR_URL=127.0.0.1:9001
       - AUTOTESTER_CONFIG=/app/.dockerfiles/docker-config.yml
       - STACK_ROOT=/home/docker/.autotesting/.stack
+      - WORKER_LOG_DIR=/home/docker/.autotesting/worker_log_dir
     depends_on:
       - postgres
       - redis
 
-  client:
+  client: &client
     build:
       context: ./client
       dockerfile: ./.dockerfiles/Dockerfile
       args:
         UBUNTU_VERSION: '22.04'
-    image: markus-autotest-client-dev:1.2.0
+    image: markus-autotest-client-dev:1.3.0
     container_name: 'autotest-client'
     volumes:
       - ./client:/app:cached
+      - venv_client:/markus_venv
     environment:
       - REDIS_URL=redis://redis:6379/
       - FLASK_HOST=0.0.0.0
@@ -43,6 +46,20 @@ services:
       - default
       - markus_dev
 
+  server-deps-updater:
+    <<: *server
+    entrypoint: "/app/.dockerfiles/entrypoint-dev-deps-updater.sh"
+    profiles: ["manual"]  # Prevent service from starting automatically
+    depends_on: []
+
+  client-deps-updater:
+    <<: *client
+    entrypoint: ".dockerfiles/entrypoint-dev-deps-updater.sh"
+    profiles: ["manual"]  # Prevent service from starting automatically
+    ports: []
+    depends_on: []
+    networks: []
+
   postgres:
     image: postgres:14
     volumes:
@@ -72,6 +89,8 @@ services:
 volumes:
   postgres_autotest:
   redis_autotest:
+  venv_client:
+  venv_server:
   workspace:
 
 networks:

server/.dockerfiles/Dockerfile

@@ -43,15 +43,12 @@ RUN chmod a+x /home/${LOGIN_USER}
 
 COPY . /app
 
-RUN python3.11 -m venv /markus_venv && \
-    /markus_venv/bin/pip install wheel && \
-    /markus_venv/bin/pip install -r /app/requirements.txt && \
-    find /app/autotest_server/testers -name requirements.system -exec {} \; && \
-    rm -rf /app/*
+RUN find /app/autotest_server/testers -name requirements.system -exec {} \;
 
 RUN echo "TZ=$( cat /etc/timezone )" >> /etc/R/Renviron.site
 
 RUN mkdir -p ${WORKSPACE} && chown ${LOGIN_USER} ${WORKSPACE}
+RUN mkdir -p /home/${LOGIN_USER}/markus_venv && chown ${LOGIN_USER} /home/${LOGIN_USER}/markus_venv
 
 WORKDIR /home/${LOGIN_USER}
 

server/.dockerfiles/cmd-dev.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
-/markus_venv/bin/python /app/install.py
-/markus_venv/bin/python /app/start_stop.py start -n
+markus_venv/bin/python /app/install.py
+markus_venv/bin/python /app/start_stop.py start -n