Fractal server 2.17 / ssh example

examples/ssh-based/docker-compose.yml

@@ -38,8 +38,8 @@ services:
       context: ./slurm_cluster
       dockerfile: Dockerfile
       args:
-        - FRACTAL_SERVER_GIT=
-        - FRACTAL_SERVER_RELEASE=2.15.0a4
+        - FRACTAL_SERVER_GIT=dev
+        - FRACTAL_SERVER_RELEASE=
         - SLURM_CPUS=3
         - SLURM_MEMORY=3000
     volumes:
@@ -65,8 +65,8 @@ services:
       context: ./server
       dockerfile: Dockerfile
       args:
-        - FRACTAL_SERVER_GIT=
-        - FRACTAL_SERVER_RELEASE=2.15.0a4
+        - FRACTAL_SERVER_GIT=dev
+        - FRACTAL_SERVER_RELEASE=
     depends_on:
       db:
         condition: service_healthy
@@ -90,7 +90,7 @@ services:
       context: ./web
       dockerfile: Dockerfile
       args:
-        - FRACTAL_WEB_VERSION=1.19.0-a0
+        - FRACTAL_WEB_GIT=dev
     ports:
       - 5173:5173
     networks:
@@ -101,7 +101,7 @@ services:
     build:
       context: ./server-config
       args:
-        - FRACTAL_CLIENT_VERSION=2.9.0
+        - FRACTAL_CLIENT_VERSION=2.10.0
     depends_on:
       server:
         condition: service_healthy

examples/ssh-based/makefile

@@ -7,12 +7,12 @@ build:
 build-no-cache:
 	docker compose --progress=plain build --no-cache
 
-clean:
+down:
 	docker compose down -v --timeout 1
 
-clean-with-volumes:
+down-with-volumes:
 	docker compose down --volumes -v --timeout 1
 
 
-all-from-scratch: clean-with-volumes build-no-cache
+all-from-scratch: down-with-volumes build-no-cache
 	docker compose up

examples/ssh-based/scripts/1_prepare_workflow_for_execution.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 
-# Exit immediately if a command exits with a non-zero status
 set -e
 
 # Set cache path to the local directory, remove it if it exists
@@ -15,7 +14,7 @@ echo "#!/bin/bash"> "$SUBMIT_SCRIPT"
 chmod +x "$SUBMIT_SCRIPT"
 BASE_LABEL=$(date +%s)
 
-for INDEX in {1..20}; do
+for INDEX in {1..10}; do
 
     LABEL="${BASE_LABEL}-${INDEX}"
     PROJECT_NAME="proj-$LABEL"

examples/ssh-based/scripts/workflow.json

@@ -58,7 +58,7 @@
       },
       "task": {
         "pkg_name": "fractal-tasks-core",
-        "version": "1.4.2",
+        "version": "1.5.4",
         "name": "Convert Cellvoyager to OME-Zarr"
       }
     },
@@ -81,7 +81,7 @@
       },
       "task": {
         "pkg_name": "fractal-tasks-core",
-        "version": "1.4.2",
+        "version": "1.5.4",
         "name": "Project Image (HCS Plate)"
       }
     }

examples/ssh-based/server-config/config.sh

@@ -1,22 +1,12 @@
 #!/bin/bash
 
-SSH_HOST="slurm"
-SSH_USER="test01"
-SSH_PRIVATE_KEY_FILE="/ssh_key_test01.key"
-
-SETTINGS_FILE=./ssh-settings.json
-echo "{" > "$SETTINGS_FILE"
-echo " \"ssh_host\": \"${SSH_HOST}\"," >> "$SETTINGS_FILE"
-echo " \"ssh_username\": \"${SSH_USER}\"," >> "$SETTINGS_FILE"
-echo " \"ssh_private_key_path\": \"${SSH_PRIVATE_KEY_FILE}\"," >> "$SETTINGS_FILE"
-echo " \"ssh_tasks_dir\": \"/data/remote/tasks/\"," >> "$SETTINGS_FILE"
-echo " \"ssh_jobs_dir\": \"/data/remote/jobs/\"" >> "$SETTINGS_FILE"
-echo "}" >> "$SETTINGS_FILE"
-cat "$SETTINGS_FILE"
+
 
 # Update user
 FRACTAL_USER_ID=$(fractal --batch user whoami)
-fractal user edit "$FRACTAL_USER_ID" --new-ssh-settings-json "$SETTINGS_FILE" --new-project-dir /data/zarrs/test01
+echo "FRACTAL_USER_ID: $FRACTAL_USER_ID"
+fractal user edit "$FRACTAL_USER_ID" --new-project-dir /data/zarrs/test01
+
 
 # Assuming that group 1 is the ALL
 ALL_GROUP_ID=1
@@ -32,7 +22,7 @@ fractal task collect fractal-tasks-core --package-extras fractal-tasks
 
 # Download test zarr data
 mkdir -p /data/zarrs
-cd /data/zarrs/
+cd /data/zarrs/ || exit 1
 wget --quiet https://zenodo.org/records/10424292/files/20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip
 unzip -q 20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip
 rm -r 20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip __MACOSX

examples/ssh-based/server/.fractal_server.env

@@ -1,20 +1,9 @@
 JWT_SECRET_KEY=somethingverysecret
+JWT_EXPIRE_SECONDS=84600
+
 FRACTAL_LOGGING_LEVEL=0
 FRACTAL_RUNNER_BACKEND=slurm_ssh
 
-FRACTAL_TASKS_DIR=/data/app/Tasks
-FRACTAL_RUNNER_WORKING_BASE_DIR=/data/app/Artifacts
-FRACTAL_SLURM_CONFIG_FILE=slurm_config.json
-FRACTAL_API_V1_MODE=exclude
-
-FRACTAL_SLURM_WORKER_PYTHON=/fractal-server-venv/bin/python3
-FRACTAL_TASKS_PYTHON_DEFAULT_VERSION="3.12"
-FRACTAL_TASKS_PYTHON_3_12=/usr/bin/python3.12
-
-FRACTAL_PIXI_CONFIG_FILE=pixi_config.json
-
-JWT_EXPIRE_SECONDS=84600
-
 POSTGRES_HOST=fractal-db
 POSTGRES_PORT=5433
 POSTGRES_USER=fractal

examples/ssh-based/server/Dockerfile

@@ -11,13 +11,10 @@ COPY get_fractal_server.sh ./
 RUN chmod +x ./get_fractal_server.sh
 RUN ./get_fractal_server.sh
 
-# RUN apt install wget -y
-RUN wget https://github.com/fractal-analytics-platform/fractal-server/raw/main/tests/v2/fractal_tasks_mock/dist/fractal_tasks_mock-0.0.1-py3-none-any.whl
-
 COPY setup_ssh_keys_and_run_server.sh ./
 COPY .fractal_server.env ./
-COPY slurm_config.json ./
-COPY pixi_config.json ./
+COPY resource.json ./
+COPY profile.json ./
 
 RUN mkdir -p /data/app/Tasks
 RUN mkdir -p /data/app/Artifacts

examples/ssh-based/server/profile.json

@@ -0,0 +1,8 @@
+{
+  "name": "Remote SLURM profile",
+  "resource_type": "slurm_ssh",
+  "username": "test01",
+  "ssh_key_path": "/ssh_key_test01.key",
+  "jobs_remote_dir": "/data/remote/jobs/",
+  "tasks_remote_dir": "/data/remote/tasks/"
+}

examples/ssh-based/server/resource.json

@@ -0,0 +1,44 @@
+{
+    "type": "slurm_ssh",
+    "name": "Remote SLURM cluster",
+    "host": "slurm",
+    "jobs_local_dir": "/data/app/Artifacts",
+    "jobs_runner_config": {
+        "default_slurm_config": {
+            "partition": "main",
+            "cpus_per_task": 1,
+            "mem": "100M"
+        },
+        "gpu_slurm_config": {},
+        "batching_config": {
+            "target_cpus_per_job": 1,
+            "max_cpus_per_job": 1,
+            "target_mem_per_job": 200,
+            "max_mem_per_job": 500,
+            "target_num_jobs": 2,
+            "max_num_jobs": 2
+        }
+    },
+    "jobs_slurm_python_worker": "/fractal-server-venv/bin/python3",
+    "jobs_poll_interval": 0,
+    "tasks_local_dir": "/data/app/Tasks",
+    "tasks_python_config": {
+        "default_version": "3.12",
+        "versions": {
+            "3.12": "/usr/bin/python3.12"
+        }
+    },
+    "tasks_pixi_config": {
+        "default_version": "0.47.0",
+        "versions": {
+            "0.47.0": "/pixi/0.47.0"
+        },
+        "SLURM_CONFIG": {
+            "partition": "main",
+            "cpus": 1,
+            "mem": "1G",
+            "time": "60"
+        }
+    },
+    "tasks_pip_cache_dir": null
+}

examples/ssh-based/server/setup_ssh_keys_and_run_server.sh

@@ -1,23 +1,23 @@
 #!/bin/bash
 
+
 SSH_HOST=slurm
 SSH_USER="test01"
 SSH_PRIVATE_KEY_FILE="/ssh_key_test01.key"
 SSH_PUBLIC_KEY_FILE="/ssh_key_test01.key.pub"
-
 # Add 'slurm' host to known ones
 ssh-keyscan slurm >> ~/.ssh/known_hosts
-
 # Generate key pair and copy ID to 'slurm'
 ssh-keygen -t rsa -f "$SSH_PRIVATE_KEY_FILE" -N ""
 sshpass -v -p test01 ssh-copy-id -f -i "$SSH_PUBLIC_KEY_FILE" "${SSH_USER}@${SSH_HOST}"
-
 # Run a simple command
 ssh -i "$SSH_PRIVATE_KEY_FILE" "${SSH_USER}@${SSH_HOST}" whoami
 
+
 fractalctl set-db
+fractalctl init-db-data --resource ./resource.json --profile ./profile.json
 gunicorn fractal_server.main:app \
-    --workers 4 \
+    --workers 3 \
     --timeout 100 \
     --bind 0.0.0.0:8000 \
     --access-logfile - \

examples/ssh-based/web/Dockerfile

@@ -1,23 +1,26 @@
 FROM node:20-alpine3.17
 
-ARG FRACTAL_WEB_VERSION
+RUN apk add git
 
+ARG FRACTAL_WEB_GIT
 
 ENV NODE_MAJOR_VERSION=20
 
-RUN wget -qO- "https://github.com/fractal-analytics-platform/fractal-web/releases/download/v${FRACTAL_WEB_VERSION}/node-${NODE_MAJOR_VERSION}-fractal-web-v${FRACTAL_WEB_VERSION}.tar.gz" | tar -xz
-
 ENV FRACTAL_SERVER_HOST=http://server:8000
 ENV PUBLIC_FRACTAL_VIZARR_VIEWER_URL=
 ENV PUBLIC_FRACTAL_ADMIN_SUPPORT_EMAIL=
 ENV PUBLIC_OAUTH_CLIENT_NAME=
 ENV AUTH_COOKIE_DOMAIN=
 ENV AUTH_COOKIE_SECURE=false
 ENV LOG_FILE=fractal-web.log
-ENV FRACTAL_API_V1_MODE=exclude
 ENV ORIGIN=http://localhost:5173
 ENV PORT=5173
 ENV LOG_LEVEL_CONSOLE=trace
 ENV FRACTAL_RUNNER_BACKEND=slurm_ssh
-# Running
+
+COPY get_fractal_web.sh ./
+RUN chmod +x get_fractal_web.sh
+RUN sh get_fractal_web.sh
+WORKDIR fractal-web
+
 CMD ["node", "build"]

examples/ssh-based/web/get_fractal_web.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+
+echo "FRACTAL_WEB_RELEASE=$FRACTAL_WEB_RELEASE"
+echo "FRACTAL_WEB_GIT=$FRACTAL_WEB_GIT"
+if [ -z "${FRACTAL_WEB_RELEASE}" ]; then
+    if [ -z "${FRACTAL_WEB_GIT}" ]; then
+        # Case 1: no release, no git
+        echo "Error: must set one of FRACTAL_WEB_RELEASE or FRACTAL_WEB_GIT." && exit 1
+    else
+        # Case 2: only git set
+        git clone -b "$FRACTAL_WEB_GIT" https://github.com/fractal-analytics-platform/fractal-web.git
+        cd fractal-web
+        npm install
+        npm run build
+    fi
+else
+    if [ -z "${FRACTAL_WEB_GIT}" ]; then
+        # Case 3: only release set
+        mkdir fractal-web
+        cd fractal-web
+        wget -qO- "https://github.com/fractal-analytics-platform/fractal-web/releases/download/v${FRACTAL_WEB_VERSION}/node-${NODE_MAJOR_VERSION}-fractal-web-v${FRACTAL_WEB_VERSION}.tar.gz" | tar -xz
+    else
+        # Case 4: both release and git set
+        echo "Error: cannot set both FRACTAL_WEB_RELEASE and FRACTAL_WEB_GIT." && exit 2
+    fi
+fi