Merge in main

Author: stephen-riggs

.bumpclient.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.16.9"
+current_version = "0.18.1"
 commit = true
 tag = false
 

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.16.9"
+current_version = "0.18.1"
 commit = true
 tag = true
 
@@ -12,3 +12,28 @@ replace = '__version__ = "{new_version}"'
 filename = "pyproject.toml"
 search = 'version = "{current_version}"'
 replace = 'version = "{new_version}"'
+
+[[tool.bumpversion.files]]
+filename = "Helm/Chart.yaml"
+search = 'version: {current_version}'
+replace = 'version: {new_version}'
+
+[[tool.bumpversion.files]]
+filename = "Helm/charts/murfey-instrument-server-clem/Chart.yaml"
+search = 'version: {current_version}'
+replace = 'version: {new_version}'
+
+[[tool.bumpversion.files]]
+filename = "Helm/charts/murfey-instrument-server-tem/Chart.yaml"
+search = 'version: {current_version}'
+replace = 'version: {new_version}'
+
+[[tool.bumpversion.files]]
+filename = "Helm/charts/murfey-rsync/Chart.yaml"
+search = 'version: {current_version}'
+replace = 'version: {new_version}'
+
+[[tool.bumpversion.files]]
+filename = "Helm/charts/murfey-server/Chart.yaml"
+search = 'version: {current_version}'
+replace = 'version: {new_version}'

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
         python-version: ["3.9", "3.10", "3.11"]
     services:
       mariadb:
-        image: mariadb:11.6.2 # released 2024-05-06
+        image: mariadb:11.7.2 # released 2024-05-06
         # Pulls image from DockerHub
         # Docker images: https://hub.docker.com/_/mariadb
         # Previous version(s):
@@ -41,6 +41,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+
       - name: Use Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
@@ -67,13 +68,13 @@ jobs:
           docker run --detach --name rabbitmq -p 127.0.0.1:5672:5672 -p 127.0.0.1:15672:15672 test-rabbitmq
           docker container list -a
 
-      - name: Get ispyb database
+      - name: Get ISPyB database
         uses: actions/download-artifact@v4
         with:
           name: database
           path: database/
 
-      - name: Install package
+      - name: Install Murfey
         run: |
           set -eux
           pip install --disable-pip-version-check -e "."[cicd,client,server,developer]
@@ -84,7 +85,7 @@ jobs:
           mysql-version: "11.3"
           auto-start: false
 
-      - name: Set up test ipsyb database
+      - name: Set up test ISPyB database
         run: |
           set -eu
           cp ".github/workflows/config/my.cnf" .my.cnf
@@ -101,9 +102,14 @@ jobs:
                    schemas/ispyb/routines.sql \
                    grants/ispyb_processing.sql \
                    grants/ispyb_import.sql; do
-              echo Importing ${f}...
+
+              echo "Patching ${f} in SQL files to fix CLI escape issues..."
+              sed -i 's/\\-/-/g' "$f"
+
+              echo "Importing ${f}..."
               mariadb --defaults-file=.my.cnf < $f
           done
+
           mariadb --defaults-file=.my.cnf -e "CREATE USER ispyb_api@'%' IDENTIFIED BY 'password_1234'; GRANT ispyb_processing to ispyb_api@'%'; GRANT ispyb_import to ispyb_api@'%'; SET DEFAULT ROLE ispyb_processing FOR ispyb_api@'%';"
           mariadb --defaults-file=.my.cnf -e "CREATE USER ispyb_api_future@'%' IDENTIFIED BY 'password_4321'; GRANT SELECT ON ispybtest.* to ispyb_api_future@'%';"
           mariadb --defaults-file=.my.cnf -e "CREATE USER ispyb_api_sqlalchemy@'%' IDENTIFIED BY 'password_5678'; GRANT SELECT ON ispybtest.* to ispyb_api_sqlalchemy@'%'; GRANT INSERT ON ispybtest.* to ispyb_api_sqlalchemy@'%'; GRANT UPDATE ON ispybtest.* to ispyb_api_sqlalchemy@'%';"
@@ -112,18 +118,17 @@ jobs:
       - name: Check RabbitMQ is alive
         run: wget -t 10 -w 1 http://127.0.0.1:15672 -O -
 
-      - name: Run tests
+      - name: Run Murfey tests
         env:
           POSTGRES_HOST: localhost
           POSTGRES_PORT: 5432
           POSTGRES_DB: murfey_test_db
           POSTGRES_PASSWORD: psql_pwd
           POSTGRES_USER: psql_user
         run: |
-          export ISPYB_CREDENTIALS=".github/workflows/config/ispyb.cfg"
           PYTHONDEVMODE=1 pytest -v -ra --cov=murfey --cov-report=xml --cov-branch
 
-      - name: Upload to Codecov
+      - name: Upload test results to Codecov
         uses: codecov/codecov-action@v5
         with:
           name: ${{ matrix.python-version }}

.github/workflows/test.yml

@@ -3,11 +3,12 @@ name: Build and test
 on: [push, pull_request]
 
 env:
-  DATABASE_SCHEMA: 4.2.1 # released 2024-08-19
+  ISPYB_DATABASE_SCHEMA: 4.6.0
   # Installs from GitHub
   # Versions: https://github.com/DiamondLightSource/ispyb-database/tags
   # Previous version(s):
-  # 4.1.0
+  # 4.2.1 # released 2024-08-19
+  # 4.1.0 # released 2024-03-26
 
 permissions:
   contents: read
@@ -53,10 +54,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Download ISPyB DB schema v${{ env.DATABASE_SCHEMA }} for tests
+      - name: Download ISPyB DB schema v${{ env.ISPYB_DATABASE_SCHEMA }} for tests
         run: |
           mkdir database
-          wget -t 3 --waitretry=20 https://github.com/DiamondLightSource/ispyb-database/releases/download/v${{ env.DATABASE_SCHEMA }}/ispyb-database-${{ env.DATABASE_SCHEMA }}.tar.gz -O database/ispyb-database.tar.gz
+          wget -t 3 --waitretry=20 https://github.com/DiamondLightSource/ispyb-database/releases/download/v${{ env.ISPYB_DATABASE_SCHEMA }}/ispyb-database-${{ env.ISPYB_DATABASE_SCHEMA }}.tar.gz -O database/ispyb-database.tar.gz
       - name: Store database artifact
         uses: actions/upload-artifact@v4
         with:

.pre-commit-config.yaml

@@ -20,6 +20,7 @@ repos:
         name: Verifying JSON file syntax
       - id: check-yaml
         name: Verifying YAML file syntax
+        exclude: Helm/charts/.*?/templates/deployment\.yaml
         args: [--allow-multiple-documents]
       - id: check-toml
         name: Verifying TOML file syntax
@@ -103,4 +104,5 @@ repos:
     hooks:
       - id: prettier
         name: Formatting other file types according to Prettier
+        exclude: Helm/charts/.*?/templates/deployment\.yaml
         # types_or: [css, html, json, yaml]

Dockerfiles/murfey-instrument-server

@@ -0,0 +1,53 @@
+# Build command template
+# podman build --build-arg groupid=<groupid> --build-arg userid=<userid> --build-arg groupname=<groupname> --no-cache -f path/to/Dockerfiles/murfey-instrument-server -t murfey-instrument-server:<version> path/to/python-murfey
+
+# Set up the base image to build with
+FROM docker.io/library/python:3.12.10-slim-bookworm AS base
+
+# Install Vim in base image
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+        rsync \
+        vim \
+        && \
+    apt-get autoremove && \
+    rm -rf /var/lib/apt/lists/*
+
+
+# Build Murfey in a branch image
+FROM base as build
+COPY ./ /python-murfey/
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+        build-essential \
+        busybox \
+        net-tools \
+        libpq-dev \
+        && \
+    busybox --install && \
+    python -m venv /venv && \
+    /venv/bin/python -m pip install --upgrade \
+        pip \
+        build \
+        importlib-metadata && \
+    /venv/bin/python -m pip install /python-murfey[client,instrument-server]
+
+
+# Transfer completed Murfey build to base image
+FROM base
+
+# Define external build arguments
+ARG groupid
+ARG groupname
+ARG userid
+
+# Copy completed Murfey build across and set user and group permissions
+COPY --from=build /venv/ /venv/
+RUN groupadd -r -g "${groupid}" "${groupname}" && \
+    useradd -r -M "${groupname}" -u "${userid}" -g "${groupname}" && \
+    chown -R "${userid}":"${groupid}" /venv && \
+    chmod -R a+x /venv
+ENV PATH=/venv/bin:$PATH
+USER "${userid}":"${groupid}"

Dockerfiles/murfey-rsync

@@ -0,0 +1,14 @@
+# Template build command
+# podman build --build-arg groupid=<groupid> --build-arg userid=<userid> --build-arg groupname=<groupname> --no-cache -f path/to/Dockerfiles/murfey-rsync
+
+FROM docker.io/library/alpine:3.21
+# FROM alpine:3.14
+
+ARG groupid
+ARG groupname
+ARG userid
+
+# Add any system dependencies for the developer/build environment here
+RUN apk add --no-cache rsync && \
+    addgroup -S -g "${groupid}" "${groupname}" && \
+    adduser -S "${groupname}" -G "${groupname}" -u "${userid}" -s /bin/sh

Dockerfiles/murfey-server

@@ -0,0 +1,61 @@
+# Template build command
+# podman build --build-arg groupid=<groupid> --build-arg userid=<userid> --build-arg groupname=<groupname> --no-cache -f path/to/Dockerfiles/murfey-server -t murfey-server:<version> path/to/python-murfey
+
+# Set up the base image to build with
+FROM docker.io/library/python:3.12.10-slim-bookworm AS base
+
+# Install Vim and PostgreSQL dependencies in base image
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+        libpq-dev \
+        vim \
+        && \
+    apt-get autoremove && \
+    rm -rf /var/lib/apt/lists/*
+
+
+# Build Murfey and IMOD in a branch image
+FROM base as build
+COPY ./ /python-murfey/
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+        build-essential \
+        busybox \
+        curl \
+        net-tools \
+        && \
+    busybox --install && \
+    python -m venv /venv && \
+    /venv/bin/python -m pip install --upgrade \
+        pip \
+        build \
+        importlib-metadata \
+        psycopg2-binary \
+        && \
+    /venv/bin/python -m pip install /python-murfey[server] && \
+    curl https://bio3d.colorado.edu/imod/AMD64-RHEL5/imod_5.1.0_RHEL8-64_CUDA12.0.sh > imod_5.1.0_RHEL8-64_CUDA12.0.sh && \
+    chmod +x imod_5.1.0_RHEL8-64_CUDA12.0.sh && \
+    mkdir imod && \
+    ./imod_5.1.0_RHEL8-64_CUDA12.0.sh -dir imod -skip -y
+
+
+# Transfer completed builds to base image
+FROM base
+
+# Pass external build arguments to this stage
+ARG groupid
+ARG groupname
+ARG userid
+
+# Copy completed Murfey and IMOD builds across and set user and group permissions
+COPY --from=build /venv/ /venv/
+COPY --from=build /imod/ /imod/
+RUN groupadd -r -g "${groupid}" "${groupname}" && \
+    useradd -r -M "${groupname}" -u "${userid}" -g "${groupname}" && \
+    chown -R "${userid}":"${groupid}" /venv && \
+    chmod -R a+x /venv
+ENV PATH=/venv/bin:/imod/IMOD/bin:$PATH
+ENV IMOD_DIR=/imod/IMOD
+USER "${userid}":"${groupid}"

Helm/Chart.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+name: murfey-services
+description: Umbrella Helm chart for deploying the servers and daemons needed to enable Murfey to transfer and process data
+version: 0.18.1
+dependencies:
+  - name: murfey-instrument-server-clem
+  - name: murfey-instrument-server-tem
+  - name: murfey-server
+  - name: murfey-rsync

Helm/charts/murfey-instrument-server-clem/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+name: murfey-instrument-server-clem
+description: Helm chart for deploying a Murfey instrument server, which executes orders to detect, modify, and transfer files on the instrument PC, and notifies the backend server about transferred files
+version: 0.18.1