diff --git a/examples/ssh-based/docker-compose.yml b/examples/ssh-based/docker-compose.yml index 3c77046..d30b185 100644 --- a/examples/ssh-based/docker-compose.yml +++ b/examples/ssh-based/docker-compose.yml @@ -38,8 +38,8 @@ services: context: ./slurm_cluster dockerfile: Dockerfile args: - - FRACTAL_SERVER_GIT= - - FRACTAL_SERVER_RELEASE=2.15.0a4 + - FRACTAL_SERVER_GIT=dev + - FRACTAL_SERVER_RELEASE= - SLURM_CPUS=3 - SLURM_MEMORY=3000 volumes: @@ -65,8 +65,8 @@ services: context: ./server dockerfile: Dockerfile args: - - FRACTAL_SERVER_GIT= - - FRACTAL_SERVER_RELEASE=2.15.0a4 + - FRACTAL_SERVER_GIT=dev + - FRACTAL_SERVER_RELEASE= depends_on: db: condition: service_healthy @@ -90,7 +90,7 @@ services: context: ./web dockerfile: Dockerfile args: - - FRACTAL_WEB_VERSION=1.19.0-a0 + - FRACTAL_WEB_GIT=dev ports: - 5173:5173 networks: @@ -101,7 +101,7 @@ services: build: context: ./server-config args: - - FRACTAL_CLIENT_VERSION=2.9.0 + - FRACTAL_CLIENT_VERSION=2.10.0 depends_on: server: condition: service_healthy diff --git a/examples/ssh-based/makefile b/examples/ssh-based/makefile index 108cdf4..ebb4343 100644 --- a/examples/ssh-based/makefile +++ b/examples/ssh-based/makefile @@ -7,12 +7,12 @@ build: build-no-cache: docker compose --progress=plain build --no-cache -clean: +down: docker compose down -v --timeout 1 -clean-with-volumes: +down-with-volumes: docker compose down --volumes -v --timeout 1 -all-from-scratch: clean-with-volumes build-no-cache +all-from-scratch: down-with-volumes build-no-cache docker compose up diff --git a/examples/ssh-based/scripts/1_prepare_workflow_for_execution.sh b/examples/ssh-based/scripts/1_prepare_workflow_for_execution.sh index 1818404..c279a02 100755 --- a/examples/ssh-based/scripts/1_prepare_workflow_for_execution.sh +++ b/examples/ssh-based/scripts/1_prepare_workflow_for_execution.sh @@ -1,6 +1,5 @@ #!/bin/bash -# Exit immediately if a command exits with a non-zero status set -e # Set cache path to the local directory, remove it if it exists @@ -15,7 +14,7 @@ echo "#!/bin/bash"> "$SUBMIT_SCRIPT" chmod +x "$SUBMIT_SCRIPT" BASE_LABEL=$(date +%s) -for INDEX in {1..20}; do +for INDEX in {1..10}; do LABEL="${BASE_LABEL}-${INDEX}" PROJECT_NAME="proj-$LABEL" diff --git a/examples/ssh-based/scripts/workflow.json b/examples/ssh-based/scripts/workflow.json index 06ce11a..e44ca44 100644 --- a/examples/ssh-based/scripts/workflow.json +++ b/examples/ssh-based/scripts/workflow.json @@ -58,7 +58,7 @@ }, "task": { "pkg_name": "fractal-tasks-core", - "version": "1.4.2", + "version": "1.5.4", "name": "Convert Cellvoyager to OME-Zarr" } }, @@ -81,7 +81,7 @@ }, "task": { "pkg_name": "fractal-tasks-core", - "version": "1.4.2", + "version": "1.5.4", "name": "Project Image (HCS Plate)" } } diff --git a/examples/ssh-based/server-config/config.sh b/examples/ssh-based/server-config/config.sh index 31c58bb..0f79cec 100755 --- a/examples/ssh-based/server-config/config.sh +++ b/examples/ssh-based/server-config/config.sh @@ -1,22 +1,12 @@ #!/bin/bash -SSH_HOST="slurm" -SSH_USER="test01" -SSH_PRIVATE_KEY_FILE="/ssh_key_test01.key" - -SETTINGS_FILE=./ssh-settings.json -echo "{" > "$SETTINGS_FILE" -echo " \"ssh_host\": \"${SSH_HOST}\"," >> "$SETTINGS_FILE" -echo " \"ssh_username\": \"${SSH_USER}\"," >> "$SETTINGS_FILE" -echo " \"ssh_private_key_path\": \"${SSH_PRIVATE_KEY_FILE}\"," >> "$SETTINGS_FILE" -echo " \"ssh_tasks_dir\": \"/data/remote/tasks/\"," >> "$SETTINGS_FILE" -echo " \"ssh_jobs_dir\": \"/data/remote/jobs/\"" >> "$SETTINGS_FILE" -echo "}" >> "$SETTINGS_FILE" -cat "$SETTINGS_FILE" + # Update user FRACTAL_USER_ID=$(fractal --batch user whoami) -fractal user edit "$FRACTAL_USER_ID" --new-ssh-settings-json "$SETTINGS_FILE" --new-project-dir /data/zarrs/test01 +echo "FRACTAL_USER_ID: $FRACTAL_USER_ID" +fractal user edit "$FRACTAL_USER_ID" --new-project-dir /data/zarrs/test01 + # Assuming that group 1 is the ALL ALL_GROUP_ID=1 @@ -32,7 +22,7 @@ fractal task collect fractal-tasks-core --package-extras fractal-tasks # Download test zarr data mkdir -p /data/zarrs -cd /data/zarrs/ +cd /data/zarrs/ || exit 1 wget --quiet https://zenodo.org/records/10424292/files/20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip unzip -q 20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip rm -r 20200812-CardiomyocyteDifferentiation14-Cycle1_mip.zarr.zip __MACOSX diff --git a/examples/ssh-based/server/.fractal_server.env b/examples/ssh-based/server/.fractal_server.env index df401df..e1b3f06 100644 --- a/examples/ssh-based/server/.fractal_server.env +++ b/examples/ssh-based/server/.fractal_server.env @@ -1,20 +1,9 @@ JWT_SECRET_KEY=somethingverysecret +JWT_EXPIRE_SECONDS=84600 + FRACTAL_LOGGING_LEVEL=0 FRACTAL_RUNNER_BACKEND=slurm_ssh -FRACTAL_TASKS_DIR=/data/app/Tasks -FRACTAL_RUNNER_WORKING_BASE_DIR=/data/app/Artifacts -FRACTAL_SLURM_CONFIG_FILE=slurm_config.json -FRACTAL_API_V1_MODE=exclude - -FRACTAL_SLURM_WORKER_PYTHON=/fractal-server-venv/bin/python3 -FRACTAL_TASKS_PYTHON_DEFAULT_VERSION="3.12" -FRACTAL_TASKS_PYTHON_3_12=/usr/bin/python3.12 - -FRACTAL_PIXI_CONFIG_FILE=pixi_config.json - -JWT_EXPIRE_SECONDS=84600 - POSTGRES_HOST=fractal-db POSTGRES_PORT=5433 POSTGRES_USER=fractal diff --git a/examples/ssh-based/server/Dockerfile b/examples/ssh-based/server/Dockerfile index 9a2a168..fa9722d 100644 --- a/examples/ssh-based/server/Dockerfile +++ b/examples/ssh-based/server/Dockerfile @@ -11,13 +11,10 @@ COPY get_fractal_server.sh ./ RUN chmod +x ./get_fractal_server.sh RUN ./get_fractal_server.sh -# RUN apt install wget -y -RUN wget https://github.com/fractal-analytics-platform/fractal-server/raw/main/tests/v2/fractal_tasks_mock/dist/fractal_tasks_mock-0.0.1-py3-none-any.whl - COPY setup_ssh_keys_and_run_server.sh ./ COPY .fractal_server.env ./ -COPY slurm_config.json ./ -COPY pixi_config.json ./ +COPY resource.json ./ +COPY profile.json ./ RUN mkdir -p /data/app/Tasks RUN mkdir -p /data/app/Artifacts diff --git a/examples/ssh-based/server/pixi_config.json b/examples/ssh-based/server/pixi_config.json deleted file mode 100644 index 3887b3b..0000000 --- a/examples/ssh-based/server/pixi_config.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "default_version": "0.47.0", - "versions": { - "0.47.0": "/pixi/0.47.0" - } -} diff --git a/examples/ssh-based/server/profile.json b/examples/ssh-based/server/profile.json new file mode 100644 index 0000000..9d52850 --- /dev/null +++ b/examples/ssh-based/server/profile.json @@ -0,0 +1,8 @@ +{ + "name": "Remote SLURM profile", + "resource_type": "slurm_ssh", + "username": "test01", + "ssh_key_path": "/ssh_key_test01.key", + "jobs_remote_dir": "/data/remote/jobs/", + "tasks_remote_dir": "/data/remote/tasks/" +} diff --git a/examples/ssh-based/server/resource.json b/examples/ssh-based/server/resource.json new file mode 100644 index 0000000..3a9c959 --- /dev/null +++ b/examples/ssh-based/server/resource.json @@ -0,0 +1,44 @@ +{ + "type": "slurm_ssh", + "name": "Remote SLURM cluster", + "host": "slurm", + "jobs_local_dir": "/data/app/Artifacts", + "jobs_runner_config": { + "default_slurm_config": { + "partition": "main", + "cpus_per_task": 1, + "mem": "100M" + }, + "gpu_slurm_config": {}, + "batching_config": { + "target_cpus_per_job": 1, + "max_cpus_per_job": 1, + "target_mem_per_job": 200, + "max_mem_per_job": 500, + "target_num_jobs": 2, + "max_num_jobs": 2 + } + }, + "jobs_slurm_python_worker": "/fractal-server-venv/bin/python3", + "jobs_poll_interval": 0, + "tasks_local_dir": "/data/app/Tasks", + "tasks_python_config": { + "default_version": "3.12", + "versions": { + "3.12": "/usr/bin/python3.12" + } + }, + "tasks_pixi_config": { + "default_version": "0.47.0", + "versions": { + "0.47.0": "/pixi/0.47.0" + }, + "SLURM_CONFIG": { + "partition": "main", + "cpus": 1, + "mem": "1G", + "time": "60" + } + }, + "tasks_pip_cache_dir": null +} diff --git a/examples/ssh-based/server/setup_ssh_keys_and_run_server.sh b/examples/ssh-based/server/setup_ssh_keys_and_run_server.sh index 0d4a5ed..e5ba11a 100755 --- a/examples/ssh-based/server/setup_ssh_keys_and_run_server.sh +++ b/examples/ssh-based/server/setup_ssh_keys_and_run_server.sh @@ -1,23 +1,23 @@ #!/bin/bash + SSH_HOST=slurm SSH_USER="test01" SSH_PRIVATE_KEY_FILE="/ssh_key_test01.key" SSH_PUBLIC_KEY_FILE="/ssh_key_test01.key.pub" - # Add 'slurm' host to known ones ssh-keyscan slurm >> ~/.ssh/known_hosts - # Generate key pair and copy ID to 'slurm' ssh-keygen -t rsa -f "$SSH_PRIVATE_KEY_FILE" -N "" sshpass -v -p test01 ssh-copy-id -f -i "$SSH_PUBLIC_KEY_FILE" "${SSH_USER}@${SSH_HOST}" - # Run a simple command ssh -i "$SSH_PRIVATE_KEY_FILE" "${SSH_USER}@${SSH_HOST}" whoami + fractalctl set-db +fractalctl init-db-data --resource ./resource.json --profile ./profile.json gunicorn fractal_server.main:app \ - --workers 4 \ + --workers 3 \ --timeout 100 \ --bind 0.0.0.0:8000 \ --access-logfile - \ diff --git a/examples/ssh-based/server/slurm_config.json b/examples/ssh-based/server/slurm_config.json deleted file mode 100644 index 7283805..0000000 --- a/examples/ssh-based/server/slurm_config.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "default_slurm_config": { - "partition": "main", - "cpus_per_task": 1, - "mem": "100M" - }, - "gpu_slurm_config": {}, - "batching_config": { - "target_cpus_per_job": 1, - "max_cpus_per_job": 1, - "target_mem_per_job": 200, - "max_mem_per_job": 500, - "target_num_jobs": 2, - "max_num_jobs": 2 - } -} diff --git a/examples/ssh-based/web/Dockerfile b/examples/ssh-based/web/Dockerfile index 5a2c44d..81b0323 100644 --- a/examples/ssh-based/web/Dockerfile +++ b/examples/ssh-based/web/Dockerfile @@ -1,12 +1,11 @@ FROM node:20-alpine3.17 -ARG FRACTAL_WEB_VERSION +RUN apk add git +ARG FRACTAL_WEB_GIT ENV NODE_MAJOR_VERSION=20 -RUN wget -qO- "https://github.com/fractal-analytics-platform/fractal-web/releases/download/v${FRACTAL_WEB_VERSION}/node-${NODE_MAJOR_VERSION}-fractal-web-v${FRACTAL_WEB_VERSION}.tar.gz" | tar -xz - ENV FRACTAL_SERVER_HOST=http://server:8000 ENV PUBLIC_FRACTAL_VIZARR_VIEWER_URL= ENV PUBLIC_FRACTAL_ADMIN_SUPPORT_EMAIL= @@ -14,10 +13,14 @@ ENV PUBLIC_OAUTH_CLIENT_NAME= ENV AUTH_COOKIE_DOMAIN= ENV AUTH_COOKIE_SECURE=false ENV LOG_FILE=fractal-web.log -ENV FRACTAL_API_V1_MODE=exclude ENV ORIGIN=http://localhost:5173 ENV PORT=5173 ENV LOG_LEVEL_CONSOLE=trace ENV FRACTAL_RUNNER_BACKEND=slurm_ssh -# Running + +COPY get_fractal_web.sh ./ +RUN chmod +x get_fractal_web.sh +RUN sh get_fractal_web.sh +WORKDIR fractal-web + CMD ["node", "build"] diff --git a/examples/ssh-based/web/get_fractal_web.sh b/examples/ssh-based/web/get_fractal_web.sh new file mode 100644 index 0000000..d2502c8 --- /dev/null +++ b/examples/ssh-based/web/get_fractal_web.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +set -e + +echo "FRACTAL_WEB_RELEASE=$FRACTAL_WEB_RELEASE" +echo "FRACTAL_WEB_GIT=$FRACTAL_WEB_GIT" +if [ -z "${FRACTAL_WEB_RELEASE}" ]; then + if [ -z "${FRACTAL_WEB_GIT}" ]; then + # Case 1: no release, no git + echo "Error: must set one of FRACTAL_WEB_RELEASE or FRACTAL_WEB_GIT." && exit 1 + else + # Case 2: only git set + git clone -b "$FRACTAL_WEB_GIT" https://github.com/fractal-analytics-platform/fractal-web.git + cd fractal-web + npm install + npm run build + fi +else + if [ -z "${FRACTAL_WEB_GIT}" ]; then + # Case 3: only release set + mkdir fractal-web + cd fractal-web + wget -qO- "https://github.com/fractal-analytics-platform/fractal-web/releases/download/v${FRACTAL_WEB_VERSION}/node-${NODE_MAJOR_VERSION}-fractal-web-v${FRACTAL_WEB_VERSION}.tar.gz" | tar -xz + else + # Case 4: both release and git set + echo "Error: cannot set both FRACTAL_WEB_RELEASE and FRACTAL_WEB_GIT." && exit 2 + fi +fi