diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index edd658d4..5013a700 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -99,6 +99,258 @@ jobs: FIRST_TAG=$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n 1) docker tag "$FIRST_TAG" openms-streamlit:test + - name: Save image as tar + run: docker save openms-streamlit:test -o /tmp/image.tar + + - name: Upload image artifact + uses: actions/upload-artifact@v4 + with: + name: openms-streamlit-${{ matrix.variant }}-image + path: /tmp/image.tar + retention-days: 1 + + test-apptainer: + # Apptainer/Singularity is the dominant container runtime on HPC clusters. + # It mounts the root filesystem read-only and runs as the host user's UID + # (not root inside the image). The entrypoint must tolerate both: this job + # exercises that contract by running the built image under apptainer and + # waiting for the streamlit /_stcore/health endpoint to come up. + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + variant: [full] + steps: + - uses: actions/checkout@v4 + + - name: Download image artifact + uses: actions/download-artifact@v4 + with: + name: openms-streamlit-${{ matrix.variant }}-image + path: /tmp + + - name: Install apptainer + uses: eWaterCycle/setup-apptainer@v2 + with: + apptainer-version: 1.3.4 + + - name: Build SIF from docker-archive + run: | + sudo apptainer build /tmp/openms.sif docker-archive:///tmp/image.tar + sudo chmod a+r /tmp/openms.sif + + - name: Prepare host bind dirs (mountpoint contract) + run: | + # Host paths we'll bind into the SIF. Asserting writability through + # singularity's bind machinery requires that the destination paths + # exist as real directories in the squashfs (otherwise singularity + # silently degrades the bind to read-only via underlay). + mkdir -p /tmp/host-workspaces /tmp/host-mounted-data + echo "from-host-pretest" > /tmp/host-mounted-data/sentinel.txt + + - name: Start apptainer instance (read-only root, host UID, with binds) + run: | + # Default apptainer semantics: read-only root, no --writable-tmpfs. + # This matches how users on HPC clusters run the SIF. + # Use `instance run` (apptainer 1.1+), not `instance start`: the SIF + # was built from docker-archive, which populates %runscript with the + # Docker ENTRYPOINT but leaves %startscript as the default no-op + # `exec "$@"`. `instance start` would launch an empty instance and + # streamlit would never bind 8501. + apptainer instance run \ + --bind /tmp/host-workspaces:/workspaces-streamlit-template:rw \ + --bind /tmp/host-mounted-data:/mounted-data:ro \ + /tmp/openms.sif openms-test + apptainer instance list + # Record where this run's logs will land so subsequent steps can tail + # them deterministically (path depends on hostname/user). + LOG_DIR=$(find "$HOME/.apptainer/instances/logs" -type d -name "$(whoami)" 2>/dev/null | head -n 1) + echo "APPTAINER_LOG_DIR=${LOG_DIR}" >> "$GITHUB_ENV" + ls -la "$LOG_DIR" || true + + - name: Wait for streamlit /_stcore/health + run: | + # Tail the entrypoint's stdout/stderr alongside the health probe so + # any startup failure surfaces directly in the CI log (the dedicated + # "Dump entrypoint logs on failure" step is post-mortem only and + # easy to miss in the GH Actions UI). + OUT="${APPTAINER_LOG_DIR}/openms-test.out" + ERR="${APPTAINER_LOG_DIR}/openms-test.err" + for i in $(seq 1 90); do + if curl -fsSo /dev/null --max-time 2 http://127.0.0.1:8501/_stcore/health; then + echo "Streamlit is ready after $i attempts" + exit 0 + fi + if [ $((i % 5)) -eq 0 ]; then + echo "--- attempt $i: instance log tail ---" + tail -n 20 "$OUT" 2>/dev/null || echo "(no $OUT yet)" + tail -n 10 "$ERR" 2>/dev/null || echo "(no $ERR yet)" + apptainer instance list || true + fi + sleep 2 + done + echo "TIMED OUT waiting for streamlit health endpoint" + echo "--- full entrypoint stdout ---" + cat "$OUT" 2>/dev/null || echo "(missing)" + echo "--- full entrypoint stderr ---" + cat "$ERR" 2>/dev/null || echo "(missing)" + exit 1 + + - name: Verify health endpoint returns 200 + run: curl -fsS http://127.0.0.1:8501/_stcore/health + + - name: Verify Redis is reachable inside container (full variant) + if: matrix.variant == 'full' + run: | + # In apptainer mode the entrypoint uses a unix socket (TCP 6379 on + # localhost is the host's, since net namespace is shared). The + # entrypoint writes the resolved URL to /tmp/openms-redis-url for + # out-of-band discovery, since `apptainer exec` spawns a fresh + # shell that doesn't inherit the daemon's exported env. + URL=$(apptainer exec instance://openms-test cat /tmp/openms-redis-url 2>/dev/null || true) + case "$URL" in + unix://*) + SOCK="${URL#unix://}" + echo "Redis URL is unix socket: $SOCK" + apptainer exec instance://openms-test redis-cli -s "$SOCK" ping | grep -i pong + ;; + *) + echo "Redis URL is TCP (or unset): ${URL:-default}" + apptainer exec instance://openms-test redis-cli ping | grep -i pong + ;; + esac + + - name: Verify bind mount is writable (workspaces) and readable (data) + run: | + # The whole point of pre-creating /workspaces-streamlit-template + # and /mounted-data in the image: singularity now has a real + # attach point and `:rw` actually sticks. Without the mkdir, + # `apptainer exec ... touch` here would fail with EROFS. + apptainer exec instance://openms-test sh -c \ + 'echo from-container > /workspaces-streamlit-template/probe.txt' + test -f /tmp/host-workspaces/probe.txt + grep -q from-container /tmp/host-workspaces/probe.txt + # Read-only data mount should also be visible inside the container. + apptainer exec instance://openms-test grep -q from-host-pretest /mounted-data/sentinel.txt + # The mounted-drive browser uses os.path.ismount() to gate + # rendering (existence is no longer enough now that the image + # pre-creates the dir). Assert the kernel reports both paths as + # real mount points so the detection function returns truthy. + apptainer exec instance://openms-test python3 -c " + import os, sys + for p in ('/mounted-data', '/workspaces-streamlit-template'): + assert os.path.ismount(p), f'{p} not reported as mount point' + print(f'ismount({p}) = True') + " + + - name: Dump entrypoint logs on failure + if: failure() + run: | + echo "--- apptainer instance list ---" + apptainer instance list || true + echo "--- apptainer instance logs ---" + find "$HOME/.apptainer" \( -name '*.out' -o -name '*.err' \) 2>/dev/null \ + | while read -r f; do echo "=== $f ==="; cat "$f"; done || true + + - name: Stop apptainer instance + if: always() + run: apptainer instance stop openms-test || true + + - name: Upload validated SIF artifact (push events only) + if: success() && github.event_name != 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: openms-streamlit-${{ matrix.variant }}-sif + path: /tmp/openms.sif + retention-days: 1 + if-no-files-found: error + + publish-apptainer: + # Publish the validated SIF (already health-checked above) to GHCR as an + # OCI artifact via ORAS, in a sibling package: ghcr.io///sif. + # Keeping it separate from the docker image package keeps tag lists clean + # and lets HPC users `apptainer pull oras://...` without the 5-15 min + # on-the-fly OCI->SIF conversion the docker:// path requires. + needs: test-apptainer + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + variant: [full, simple] + steps: + - name: Download validated SIF artifact + uses: actions/download-artifact@v4 + with: + name: openms-streamlit-${{ matrix.variant }}-sif + path: /tmp + + - name: Install apptainer + uses: eWaterCycle/setup-apptainer@v2 + with: + apptainer-version: 1.3.4 + + - name: Compute SIF tags + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/sif + tags: | + type=ref,event=branch,suffix=-${{ matrix.variant }} + type=ref,event=tag,suffix=-${{ matrix.variant }} + type=sha,prefix=,suffix=-${{ matrix.variant }} + type=raw,value=latest,enable=${{ matrix.variant == 'full' && github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Log in to GHCR for ORAS push + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # apptainer reads its auth from ~/.apptainer/remote.yaml, NOT from + # ~/.docker/config.json — so docker/login-action won't work here. + # Login and push must both run as the runner user (no sudo) so they + # share the same $HOME and therefore the same auth file. + echo "$GHCR_TOKEN" | apptainer registry login \ + --username "${{ github.actor }}" \ + --password-stdin \ + oras://ghcr.io + + - name: Push SIF to each computed tag + run: | + # `apptainer push` accepts ONE destination per invocation; iterate + # over the newline-separated tag list from docker/metadata-action. + # tr lowercase is belt-and-braces — metadata-action already + # lowercases, but GHCR is strict about case in OCI refs. + set -euo pipefail + while IFS= read -r tag; do + [ -z "$tag" ] && continue + tag_lc="$(echo "$tag" | tr '[:upper:]' '[:lower:]')" + echo "Pushing SIF to oras://${tag_lc}" + apptainer push /tmp/openms.sif "oras://${tag_lc}" + done <<< "${{ steps.meta.outputs.tags }}" + + test-nginx: + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + variant: [full] + steps: + - uses: actions/checkout@v4 + + - name: Download image artifact + uses: actions/download-artifact@v4 + with: + name: openms-streamlit-${{ matrix.variant }}-image + path: /tmp + + - name: Load image into local docker + run: docker load -i /tmp/image.tar + - name: Create kind cluster uses: helm/kind-action@v1 with: @@ -129,19 +381,24 @@ jobs: sleep "${i}0" done + - name: Discover overlay identity + run: | + SLUG=$(yq '.commonLabels.app' k8s/overlays/prod/kustomization.yaml) + echo "SLUG=$SLUG" >> "$GITHUB_ENV" + - name: Wait for Redis to be ready run: | - kubectl wait -n openms --for=condition=ready pod -l app=opendiakiosk,component=redis --timeout=60s + kubectl wait -n openms --for=condition=ready pod -l app=${SLUG},component=redis --timeout=60s - name: Verify Redis Service is reachable run: | - kubectl run redis-test -n openms --image=redis:7-alpine --rm -i --restart=Never -- redis-cli -h opendiakiosk-redis.openms.svc.cluster.local ping + kubectl run redis-test -n openms --image=redis:7-alpine --rm -i --restart=Never -- redis-cli -h ${SLUG}-redis.openms.svc.cluster.local ping - name: Verify all deployments are available run: | - kubectl wait -n openms --for=condition=available deployment -l app=opendiakiosk --timeout=120s || true - kubectl get pods -n openms -l app=opendiakiosk - kubectl get services -n openms -l app=opendiakiosk + kubectl wait -n openms --for=condition=available deployment -l app=${SLUG} --timeout=180s || true + kubectl get pods -n openms -l app=${SLUG} + kubectl get services -n openms -l app=${SLUG} - name: Curl both hostnames via nginx ingress run: | @@ -149,12 +406,12 @@ jobs: kubectl -n ingress-nginx port-forward "$NGINX_POD" 8080:80 & PF_PID=$! trap 'kill "$PF_PID" 2>/dev/null || true' EXIT - for i in 1 2 3 4 5; do + for i in $(seq 1 30); do sleep 2 if curl -fsSo /dev/null --max-time 2 http://127.0.0.1:8080/_stcore/health -H "Host: streamlit.openms.example.de"; then break fi - echo "port-forward not ready yet, retry $i" + echo "port-forward / app not ready yet, retry $i" done for host in streamlit.openms.example.de streamlit.openms.example.org; do curl -fsS --resolve "$host:8080:127.0.0.1" "http://$host:8080/_stcore/health" @@ -162,14 +419,24 @@ jobs: echo "$host -> 200 OK" done - traefik-integration: - needs: lint-manifests + test-traefik: + needs: build runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + variant: [full] steps: - uses: actions/checkout@v4 - - name: Build image (simple variant; routing is image-agnostic) - run: docker build -t openms-streamlit:test -f Dockerfile . + - name: Download image artifact + uses: actions/download-artifact@v4 + with: + name: openms-streamlit-${{ matrix.variant }}-image + path: /tmp + + - name: Load image into local docker + run: docker load -i /tmp/image.tar - name: Create kind cluster uses: helm/kind-action@v1 @@ -206,25 +473,39 @@ jobs: sleep "${i}0" done - - name: Wait for Redis and deployments to be ready + - name: Discover overlay identity + run: | + SLUG=$(yq '.commonLabels.app' k8s/overlays/prod/kustomization.yaml) + TRAEFIK_HOSTS=$(kubectl kustomize k8s/overlays/prod/ \ + | yq 'select(.kind == "IngressRoute") | .spec.routes[0].match' \ + | grep -oP "Host\(\`\K[^\`]+" | tr '\n' ' ') + echo "SLUG=$SLUG" >> "$GITHUB_ENV" + echo "TRAEFIK_HOSTS=$TRAEFIK_HOSTS" >> "$GITHUB_ENV" + + - name: Wait for Redis to be ready run: | - kubectl wait -n openms --for=condition=ready pod -l app=opendiakiosk,component=redis --timeout=60s - kubectl wait -n openms --for=condition=available deployment -l app=opendiakiosk --timeout=120s - kubectl get pods -n openms -l app=opendiakiosk + kubectl wait -n openms --for=condition=ready pod -l app=${SLUG},component=redis --timeout=60s + + - name: Verify all deployments are available + run: | + kubectl wait -n openms --for=condition=available deployment -l app=${SLUG} --timeout=180s || true + kubectl get pods -n openms -l app=${SLUG} + kubectl get services -n openms -l app=${SLUG} - name: Curl both hostnames via Traefik run: | kubectl -n traefik port-forward svc/traefik 8080:80 & PF_PID=$! trap 'kill "$PF_PID" 2>/dev/null || true' EXIT - for i in 1 2 3 4 5; do + FIRST_HOST=$(echo ${TRAEFIK_HOSTS} | awk '{print $1}') + for i in $(seq 1 30); do sleep 2 - if curl -fsSo /dev/null --max-time 2 http://127.0.0.1:8080/_stcore/health -H "Host: template.webapps.openms.de"; then + if curl -fsSo /dev/null --max-time 2 http://127.0.0.1:8080/_stcore/health -H "Host: ${FIRST_HOST}"; then break fi - echo "port-forward not ready yet, retry $i" + echo "port-forward / app not ready yet, retry $i" done - for host in template.webapps.openms.de template.webapps.openms.org; do + for host in ${TRAEFIK_HOSTS}; do curl -fsS --resolve "$host:8080:127.0.0.1" "http://$host:8080/_stcore/health" echo "" echo "$host -> 200 OK" diff --git a/.github/workflows/ghcr-cleanup.yml b/.github/workflows/ghcr-cleanup.yml index 6228b623..00aca969 100644 --- a/.github/workflows/ghcr-cleanup.yml +++ b/.github/workflows/ghcr-cleanup.yml @@ -51,3 +51,29 @@ jobs: tag-selection: untagged cut-off: 7d dry-run: ${{ github.event.inputs.dry-run || 'false' }} + + cleanup-sif-images: + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - name: Delete old commit-tagged SIFs (keep semver + main + latest) + uses: snok/container-retention-policy@v3.0.1 + with: + account: ${{ github.repository_owner }} + token: ${{ secrets.GITHUB_TOKEN }} + image-names: ${{ github.event.repository.name }}/sif + image-tags: "!v*-full !v*-simple !main-full !main-simple !latest" + tag-selection: tagged + cut-off: 30d + dry-run: ${{ github.event.inputs.dry-run || 'false' }} + + - name: Delete untagged SIF manifests + uses: snok/container-retention-policy@v3.0.1 + with: + account: ${{ github.repository_owner }} + token: ${{ secrets.GITHUB_TOKEN }} + image-names: ${{ github.event.repository.name }}/sif + tag-selection: untagged + cut-off: 7d + dry-run: ${{ github.event.inputs.dry-run || 'false' }} diff --git a/Dockerfile b/Dockerfile index 72e806a0..92b753a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,6 +66,13 @@ RUN wget -q \ && rm -f Miniforge3-Linux-x86_64.sh RUN mamba --version +# Make /root traversable so the entrypoint can `source +# /root/miniforge3/bin/activate ...` when the container runs as a non-root +# user (apptainer/singularity maps the host UID into the container; the +# default ubuntu /root is 0700 which would block path traversal). +x only, +# not +r, so the directory listing remains private. +RUN chmod o+x /root + # Setup mamba environment. RUN mamba create -n streamlit-env python=3.11 RUN echo "mamba activate streamlit-env" >> ~/.bashrc @@ -181,12 +188,26 @@ RUN rm -rf openms-build # Prepare and run streamlit app. FROM compile-openms AS run-app -# Install Redis server for job queue and nginx for load balancing +# Install Redis server for job queue and nginx for load balancing. +# Redis data lives under $RUNTIME_DIR at runtime (see entrypoint.sh) so no +# /var/lib/redis setup is needed - that path is not writable under Apptainer. RUN apt-get update && apt-get install -y --no-install-recommends redis-server nginx \ && rm -rf /var/lib/apt/lists/* -# Create Redis data directory -RUN mkdir -p /var/lib/redis && chown redis:redis /var/lib/redis +# Create Redis data directory. Default 0755 root-owned is enough: the docker +# entrypoint runs as root (can write regardless of mode), and the apptainer +# entrypoint relocates Redis state to /tmp/openms-runtime-* so this dir is +# never written under apptainer. +RUN mkdir -p /var/lib/redis + +# Pre-create bind-mount targets so apptainer/singularity has a real attach +# point. Docker auto-creates missing `-v` targets, but singularity uses a +# read-only underlay and silently ignores `:rw` when the target isn't a +# real directory in the SIF — writes then fail with EROFS even though the +# host bind path is writable. Pre-creating these directories costs one +# inode each and changes nothing in docker mode (the user's volume mount +# shadows them). +RUN mkdir -p /workspaces-streamlit-template /mounted-data # Create workdir and copy over all streamlit related files/folders. @@ -224,67 +245,10 @@ ENV REDIS_URL=redis://localhost:6379/0 # Set to >1 to enable nginx load balancer with multiple Streamlit instances ENV STREAMLIT_SERVER_COUNT=1 -# create entrypoint script to start cron, Redis, RQ workers, and Streamlit -RUN echo -e '#!/bin/bash\n\ -set -e\n\ -source /root/miniforge3/bin/activate streamlit-env\n\ -\n\ -# Start cron for workspace cleanup\n\ -service cron start\n\ -\n\ -# Start Redis server in background\n\ -echo "Starting Redis server..."\n\ -redis-server --daemonize yes --dir /var/lib/redis --appendonly no\n\ -\n\ -# Wait for Redis to be ready\n\ -until redis-cli ping > /dev/null 2>&1; do\n\ - echo "Waiting for Redis..."\n\ - sleep 1\n\ -done\n\ -echo "Redis is ready"\n\ -\n\ -# Start RQ worker(s) in background\n\ -WORKER_COUNT=${RQ_WORKER_COUNT:-1}\n\ -echo "Starting $WORKER_COUNT RQ worker(s)..."\n\ -for i in $(seq 1 $WORKER_COUNT); do\n\ - rq worker openms-workflows --url $REDIS_URL --name worker-$i &\n\ -done\n\ -\n\ -# Load balancer setup\n\ -SERVER_COUNT=${STREAMLIT_SERVER_COUNT:-1}\n\ -\n\ -if [ "$SERVER_COUNT" -gt 1 ]; then\n\ - echo "Starting $SERVER_COUNT Streamlit instances with nginx load balancer..."\n\ -\n\ - # Generate nginx upstream block\n\ - UPSTREAM_SERVERS=""\n\ - BASE_PORT=8510\n\ - for i in $(seq 0 $((SERVER_COUNT - 1))); do\n\ - PORT=$((BASE_PORT + i))\n\ - UPSTREAM_SERVERS="${UPSTREAM_SERVERS} server 127.0.0.1:${PORT};\\n"\n\ - done\n\ -\n\ - # Write nginx config\n\ - mkdir -p /etc/nginx\n\ - echo -e "worker_processes auto;\\npid /run/nginx.pid;\\n\\nevents {\\n worker_connections 1024;\\n}\\n\\nhttp {\\n client_max_body_size 0;\\n\\n map \\$cookie_stroute \\$route_key {\\n \\x22\\x22 \\$request_id;\\n default \\$cookie_stroute;\\n }\\n\\n upstream streamlit_backend {\\n hash \\$route_key consistent;\\n${UPSTREAM_SERVERS} }\\n\\n map \\$http_upgrade \\$connection_upgrade {\\n default upgrade;\\n \\x27\\x27 close;\\n }\\n\\n server {\\n listen 0.0.0.0:8501;\\n\\n location / {\\n proxy_pass http://streamlit_backend;\\n proxy_http_version 1.1;\\n proxy_set_header Upgrade \\$http_upgrade;\\n proxy_set_header Connection \\$connection_upgrade;\\n proxy_set_header Host \\$host;\\n proxy_set_header X-Real-IP \\$remote_addr;\\n proxy_set_header X-Forwarded-For \\$proxy_add_x_forwarded_for;\\n proxy_set_header X-Forwarded-Proto \\$scheme;\\n proxy_read_timeout 86400;\\n proxy_send_timeout 86400;\\n proxy_buffering off;\\n add_header Set-Cookie \\x22stroute=\\$route_key; Path=/; HttpOnly; SameSite=Lax\\x22 always;\\n }\\n }\\n}" > /etc/nginx/nginx.conf\n\ -\n\ - # Start Streamlit instances on internal ports\n\ - for i in $(seq 0 $((SERVER_COUNT - 1))); do\n\ - PORT=$((BASE_PORT + i))\n\ - echo "Starting Streamlit instance on port $PORT..."\n\ - streamlit run app.py --server.port $PORT --server.address 0.0.0.0 &\n\ - done\n\ -\n\ - sleep 2\n\ - echo "Starting nginx load balancer on port 8501..."\n\ - exec /usr/sbin/nginx -g "daemon off;"\n\ -else\n\ - # Single instance mode (default) - run Streamlit directly on port 8501\n\ - echo "Starting Streamlit app..."\n\ - exec streamlit run app.py --server.address 0.0.0.0\n\ -fi\n\ -' > /app/entrypoint.sh -# make the script executable +# Install the apptainer-compatible entrypoint that starts cron (when the root +# FS is writable), Redis, RQ workers, optional nginx load balancer, and the +# Streamlit server. The script falls back to /tmp paths under apptainer. +COPY docker/entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh # Patch Analytics diff --git a/README.md b/README.md index 8eb5c3d4..e7bc1b42 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,34 @@ This repository contains two Dockerfiles. and falls back to the standard upload UI. To use a different container path, change `local_data_dir` in `settings.json` before building. +## 🛰️ Run with Apptainer / Singularity (HPC) + +Apptainer (formerly Singularity) is the dominant container runtime on HPC +clusters. CI publishes prebuilt SIFs to GHCR via ORAS, so you can pull a +ready-to-run image with no on-the-fly OCI→SIF conversion and run it as your +user — no root, no `--writable-tmpfs` required: + +```bash +apptainer pull --name openms-streamlit-template.sif \ + oras://ghcr.io/openms/streamlit-template/sif:latest +apptainer run \ + --bind /path/to/data:/mounted-data:ro \ + --bind /path/to/workspaces:/workspaces-streamlit-template \ + openms-streamlit-template.sif +``` + +Available tags follow the same scheme as the Docker images: `latest`, +`main-full`, `main-simple`, `v*-full`, `v*-simple`, and per-commit SHAs. +If a tag hasn't been prebuilt yet (e.g. a PR branch), fall back to on-the-fly +conversion: `apptainer pull docker://ghcr.io/openms/streamlit-template:`. +Requires apptainer 1.1+ or singularity-ce 3.10+ for the `oras://` transport. + +The entrypoint auto-detects the read-only root filesystem (set by apptainer's +default isolation) and switches its runtime state — Redis data directory, +nginx config, PID files — to `/tmp/openms-runtime-$$`, which is always +writable inside an apptainer container. The workspace cleanup cron job is +skipped in this mode; rerun `clean-up-workspaces.py` manually if needed. + ## Documentation Documentation for **users** and **developers** is included as pages in [this template app](https://abi-services.cs.uni-tuebingen.de/streamlit-template/), indicated by the 📖 icon. diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000..65cbf8c4 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,224 @@ +#!/bin/bash +# Container entrypoint for the OpenMS streamlit template. +# +# Works with both Docker (writable root FS, runs as root) and +# Apptainer/Singularity (read-only root FS, runs as the host user's UID). +# On HPC clusters apptainer is the dominant runtime; this script makes the +# image usable there without --writable-tmpfs. +set -e + +# Force the app directory regardless of how the container was invoked. +# `apptainer instance start` does not always honor the Docker WORKDIR, so +# `streamlit run app.py` would otherwise resolve against the host's CWD. +cd /app + +# Breadcrumbs — surfaced via apptainer instance .out/.err on failure, harmless +# in docker mode. Cheap to keep around for ongoing apptainer support. +echo "entrypoint: uid=$(id -u) gid=$(id -g) cwd=$(pwd) host=$(hostname) tty=$(tty 2>/dev/null || echo none)" +echo "entrypoint: APPTAINER_NAME=${APPTAINER_NAME:-unset} SINGULARITY_NAME=${SINGULARITY_NAME:-unset} APPTAINER_CONTAINER=${APPTAINER_CONTAINER:-unset}" + +source /root/miniforge3/bin/activate streamlit-env +echo "entrypoint: conda env activated, streamlit=$(command -v streamlit || echo NOT_FOUND)" + +# ----------------------------------------------------------------------------- +# Apptainer / read-only root filesystem detection +# ----------------------------------------------------------------------------- +# Apptainer sets APPTAINER_NAME (and SINGULARITY_NAME for backwards compat). +# As a fallback we probe /var/run for writability: docker = writable, apptainer +# default = read-only. Either signal flips us into "read-only mode". +if [ -n "${APPTAINER_NAME:-}" ] || [ -n "${SINGULARITY_NAME:-}" ] \ + || [ -n "${APPTAINER_CONTAINER:-}" ] || [ -n "${SINGULARITY_CONTAINER:-}" ] \ + || ! ( : > /var/run/.openms-rw-probe ) 2>/dev/null; then + READONLY_ROOT=1 + echo "Detected read-only root filesystem (apptainer/singularity mode)" +else + READONLY_ROOT=0 + rm -f /var/run/.openms-rw-probe 2>/dev/null || true +fi + +# Pick state paths. In read-only mode we must use /tmp (always tmpfs in +# apptainer); in docker mode we keep the conventional /var paths so existing +# docker-compose / k8s deployments are unaffected. +if [ "$READONLY_ROOT" -eq 1 ]; then + RUNTIME_DIR="${OPENMS_RUNTIME_DIR:-/tmp/openms-runtime-$$}" + mkdir -p "$RUNTIME_DIR" + REDIS_DATA_DIR="$RUNTIME_DIR/redis" + REDIS_PID_FILE="$RUNTIME_DIR/redis.pid" + # Apptainer/singularity share the host's network namespace by default. If + # the host has anything listening on 6379 (a system redis-server, a docker + # container, a previous singularity instance that didn't clean up), our + # `redis-server --daemonize` silently fails with EADDRINUSE and the local + # redis-cli ping happily connects to the host's redis instead — which + # leaves stale `worker-1` records lying around and ultimately runs the + # workflow's mkdir outside our mount namespace (no bind → EROFS). A unix + # socket sidesteps the network stack entirely; the path is unambiguously + # ours. + REDIS_SOCKET="$RUNTIME_DIR/redis.sock" + REDIS_URL="unix://${REDIS_SOCKET}" + export REDIS_URL + NGINX_CONF_DIR="$RUNTIME_DIR/nginx" + NGINX_PID_FILE="$RUNTIME_DIR/nginx.pid" + mkdir -p "$REDIS_DATA_DIR" "$NGINX_CONF_DIR" + # Marker for out-of-band discovery (e.g. `apptainer exec ... redis-cli` + # from CI). The entrypoint's exported env doesn't propagate to fresh + # exec invocations, so write the resolved URL to a stable path. + echo "$REDIS_URL" > /tmp/openms-redis-url 2>/dev/null || true +else + RUNTIME_DIR="/var/run" + REDIS_DATA_DIR="/var/lib/redis" + REDIS_PID_FILE="/var/run/redis.pid" + REDIS_SOCKET="" + NGINX_CONF_DIR="/etc/nginx" + NGINX_PID_FILE="/run/nginx.pid" +fi + +# ----------------------------------------------------------------------------- +# Workspace cleanup cron (best-effort) +# ----------------------------------------------------------------------------- +# `service cron start` writes /var/run/crond.pid; it cannot work on a read-only +# root. The cleanup job is optional — workspaces just accumulate until the +# container is rebuilt, which is acceptable for HPC use cases where users +# manage their own workspace volumes. +if [ "$READONLY_ROOT" -eq 0 ]; then + service cron start || echo "WARN: cron failed to start; workspace cleanup disabled" +else + echo "Skipping cron (read-only root); run clean-up-workspaces.py manually if needed" +fi + +# ----------------------------------------------------------------------------- +# Redis + RQ workers (only present in the full image) +# ----------------------------------------------------------------------------- +# The simple image does not install redis-server. Skip the whole queue section +# when the binary is missing, so this entrypoint can be shared by both images. +if command -v redis-server >/dev/null 2>&1; then + if [ -n "$REDIS_SOCKET" ]; then + echo "Starting Redis server (data=$REDIS_DATA_DIR, socket=$REDIS_SOCKET)..." + # --port 0 disables the TCP listener entirely — we only accept the + # unix socket. This is the whole point of switching to a socket in + # apptainer mode: the host's network namespace (shared by default) + # cannot conflict with us, and there is no fall-through to a stray + # host redis-server. + redis-server --daemonize yes \ + --dir "$REDIS_DATA_DIR" \ + --pidfile "$REDIS_PID_FILE" \ + --unixsocket "$REDIS_SOCKET" \ + --unixsocketperm 700 \ + --port 0 \ + --appendonly no + REDIS_CLI_ARGS=(-s "$REDIS_SOCKET") + else + echo "Starting Redis server (data=$REDIS_DATA_DIR)..." + redis-server --daemonize yes \ + --dir "$REDIS_DATA_DIR" \ + --pidfile "$REDIS_PID_FILE" \ + --appendonly no + REDIS_CLI_ARGS=() + fi + + # Bounded wait so a broken redis-server (e.g. socket can't be created or + # an unexpected fork failure) fails the container fast instead of hanging + # forever and never serving /_stcore/health. + REDIS_STARTUP_RETRIES="${REDIS_STARTUP_RETRIES:-30}" + for i in $(seq 1 "$REDIS_STARTUP_RETRIES"); do + if redis-cli "${REDIS_CLI_ARGS[@]}" ping >/dev/null 2>&1; then + echo "Redis is ready" + break + fi + echo "Waiting for Redis... attempt $i/$REDIS_STARTUP_RETRIES" + sleep 1 + done + if ! redis-cli "${REDIS_CLI_ARGS[@]}" ping >/dev/null 2>&1; then + echo "ERROR: Redis failed to become ready within ${REDIS_STARTUP_RETRIES}s" >&2 + exit 1 + fi + + WORKER_COUNT="${RQ_WORKER_COUNT:-1}" + echo "Starting $WORKER_COUNT RQ worker(s)..." + for i in $(seq 1 "$WORKER_COUNT"); do + rq worker openms-workflows --url "$REDIS_URL" --name "worker-$i" & + done +fi + +# ----------------------------------------------------------------------------- +# Streamlit (single instance or behind nginx load balancer) +# ----------------------------------------------------------------------------- +SERVER_COUNT="${STREAMLIT_SERVER_COUNT:-1}" + +# Surface a misconfigured opt-in to load balancing — silently downgrading to a +# single instance has bitten users on the simple image variant where nginx +# isn't installed. +if [ "$SERVER_COUNT" -gt 1 ] && ! command -v nginx >/dev/null 2>&1; then + echo "WARN: STREAMLIT_SERVER_COUNT=$SERVER_COUNT requested but nginx is not installed (simple image?); falling back to a single instance" >&2 +fi + +if [ "$SERVER_COUNT" -gt 1 ] && command -v nginx >/dev/null 2>&1; then + echo "Starting $SERVER_COUNT Streamlit instances with nginx load balancer..." + + UPSTREAM_SERVERS="" + BASE_PORT=8510 + for i in $(seq 0 $((SERVER_COUNT - 1))); do + PORT=$((BASE_PORT + i)) + UPSTREAM_SERVERS="${UPSTREAM_SERVERS} server 127.0.0.1:${PORT}; +" + done + + NGINX_CONF_FILE="$NGINX_CONF_DIR/nginx.conf" + cat > "$NGINX_CONF_FILE" </dev/null || echo "Note: cron not started (read-only filesystem?). Scheduled workspace cleanup disabled." + +# Start Redis with an explicit runtime-generated config. This avoids the +# distro default /etc/redis/redis.conf which sets dir to /var/lib/redis, +# a path that is not writable on Apptainer's read-only rootfs. +echo "Starting Redis server..." +REDIS_CONF="$RUNTIME_DIR/redis/redis.conf" +cat > "$REDIS_CONF" < /dev/null 2>&1; do + echo "Waiting for Redis..." + sleep 1 +done +echo "Redis is ready" + +# Start RQ worker(s) in background +WORKER_COUNT=${RQ_WORKER_COUNT:-1} +echo "Starting $WORKER_COUNT RQ worker(s)..." +for i in $(seq 1 $WORKER_COUNT); do + rq worker openms-workflows --url "$REDIS_URL" --name "worker-$i" & +done + +# Load balancer setup +SERVER_COUNT=${STREAMLIT_SERVER_COUNT:-1} + +if [ "$SERVER_COUNT" -gt 1 ]; then + echo "Starting $SERVER_COUNT Streamlit instances with nginx load balancer..." + + BASE_PORT=8510 + UPSTREAM_LINES="" + for i in $(seq 0 $((SERVER_COUNT - 1))); do + PORT=$((BASE_PORT + i)) + UPSTREAM_LINES="${UPSTREAM_LINES} server 127.0.0.1:${PORT}; +" + done + + # Write nginx config into the writable runtime dir and point pidfile, + # logs, and all temp paths there too so nginx can run under Apptainer's + # read-only rootfs (which makes the default /run, /var/log/nginx, and + # /var/lib/nginx unwritable). Bash interpolates $RUNTIME_DIR and + # $UPSTREAM_LINES; nginx-side variables are escaped with \$ so they + # reach the file as literal $name for nginx to expand at request time. + NGINX_CONF="$RUNTIME_DIR/nginx/nginx.conf" + cat > "$NGINX_CONF" < Union[Path, None]: """Return the validated mount root from the ``local_data_dir`` setting. - The browser renders only when that path resolves to an existing - directory inside the container, i.e. when a host volume is actually - mounted there. + The browser renders only when ``local_data_dir`` is an actual mount + point inside the container — i.e. the operator passed ``-v`` / + ``--bind`` / ``volumeMount`` to attach host data. Existence alone is + no longer sufficient because the image now pre-creates the path so + apptainer/singularity binds have a real attach target; without + ``os.path.ismount`` the browser would render an empty tree for every + user who didn't mount anything. """ settings = st.session_state.get("settings") or {} raw = (settings.get("local_data_dir") or "").strip() @@ -39,7 +43,11 @@ def _mounted_data_root() -> Union[Path, None]: p = Path(raw).expanduser().resolve(strict=True) except (OSError, RuntimeError): return None - return p if p.is_dir() else None + if not p.is_dir(): + return None + if not os.path.ismount(p): + return None + return p class StreamlitUI: