Merge pull request #27 from andersinno/ansible

Add Ansible playbooks for deployment

Author: suutari-ai

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "passari-web-ui"]
 	path = passari-web-ui
 	url = ../passari-web-ui
+[submodule "ansible"]
+	path = ansible
+	url = ../passari-ansible

Dockerfile

@@ -62,15 +62,9 @@ RUN --mount=type=cache,sharing=locked,target=/var/cache/dnf \
 # Install pipx
 RUN pip install --root-user-action=ignore pipx
 
-
-###############################################################################
-FROM base AS wheels
+# Install packages needed to build Python dependencies
 RUN --mount=type=cache,sharing=locked,target=/var/cache/dnf \
     dnf install -y python3.12-devel swig  # For building M2Crypto
-COPY requirements-siptools.txt /
-RUN mkdir wheels
-WORKDIR /wheels
-RUN pip wheel -r ../requirements-siptools.txt
 
 ###############################################################################
 FROM base AS app
@@ -87,12 +81,6 @@ RUN pipx install pip-tools
 # Set a working directory for the application
 WORKDIR /app
 
-# Install siptools
-COPY --from=wheels /wheels /siptools-wheels
-COPY requirements-siptools.txt .
-RUN --mount=type=cache,sharing=locked,uid=1000,target=/home/appuser/.cache/pip \
-    pip install --user --no-index -f /siptools-wheels -r requirements-siptools.txt
-
 # Copy the sources
 #
 # Note: The dirs are intentionally created as root.  There should be no

Makefile

@@ -1,4 +1,10 @@
-.PHONY: up down start stop restart logs ps build requirements
+.PHONY: \
+	up down start stop restart logs ps build \
+	clean-wheels wheels \
+	ansible-update ansible-wheel-update \
+	ansible-requirements requirements
+
+ANSIBLE_DESTDIR ?= ansible/roles/passari_packages/files
 
 up:
 	docker-compose up
@@ -24,7 +30,28 @@ ps:
 build:
 	docker-compose build
 
-requirements: requirements.txt requirements-siptools.txt requirements-dev.txt
+clean-wheels:
+	rm -fr wheels
+
+wheels:
+	./build-wheels
+
+ansible-update: clean-wheels wheels ansible-wheel-update ansible-requirements
+
+ansible-wheel-update:
+	rm -fr $(ANSIBLE_DESTDIR)/wheels
+	mkdir -p $(ANSIBLE_DESTDIR)/wheels
+	cp -vf wheels/*.whl $(ANSIBLE_DESTDIR)/wheels/
+
+ansible-requirements: requirements.txt
+	rm -f $(ANSIBLE_DESTDIR)/requirements*.txt
+	cp -vf requirements.txt $(ANSIBLE_DESTDIR)/
+	./compile-requirements --hashes \
+		$(ANSIBLE_DESTDIR)/requirements-workflow.in
+	./compile-requirements --hashes \
+		$(ANSIBLE_DESTDIR)/requirements-web-ui.in
+
+requirements: requirements.txt requirements-dev.txt
 
 %.txt: %.in
 	./compile-requirements $<

ansible

@@ -0,0 +1,14 @@
+#!/bin/sh
+# Build Passari wheels
+
+set -e
+
+pip install build
+
+mkdir -p wheels
+
+for i in passari passari-workflow passari-web-ui; do
+    # The SOURCE_DATE_EPOCH variable makes the dates inside the wheel
+    # ZIP files reproducible so that the whl files are reproducible
+    SOURCE_DATE_EPOCH=0 python -m build --outdir ./wheels --wheel ./$i
+done

build-wheels

@@ -1,7 +1,29 @@
 #!/bin/bash
 set -e
-infile=${1:?Give a name of requirements*.in file as first argument}
-txtfile=${infile%.in}.txt
+
+called_as=("$0" "$@")
+pip_compile_args=(--quiet --strip-extras --no-emit-find-links --allow-unsafe)
+
+while [[ $1 == -* ]]; do
+    case "$1" in
+        -H|--hashes)
+            pip_compile_args+=(--generate-hashes --reuse-hashes)
+            ;;
+        *)
+            echo "Unknown option: $1" >&2
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+infile_path=${1:?Give a name of requirements*.in file as an argument}
+txtfile_path=${infile_path%.in}.txt
+
+# Change to the dir of the infile_path and use basenames for the files
+cd "$(dirname "$infile_path")"
+infile=${infile_path##*/}
+txtfile=${txtfile_path##*/}
 
 # If pip-compile is in PATH use it, otherwise run through pipx
 pip_compile=(pip-compile)
@@ -26,9 +48,9 @@ if grep -q "^-c" "$infile"; then
     done
 fi
 
-echo -n "Generating $txtfile from $infile..."
+echo -n "Generating $txtfile_path from $infile_path..."
 
-"${pip_compile[@]}" --quiet --strip-extras "$infile"
+"${pip_compile[@]}" "${pip_compile_args[@]}" "$infile"
 
 # Post-process the output file with sed:
 #
@@ -38,7 +60,7 @@ cwd_url="\(file://\)\?$PWD\(/\.\)\?"
 sed -i "
     s%^.* @ $cwd_url%.%;
     s%$cwd_url%.%g;
-    s|^\(# *\)pip-compile.*|\1$0${1:+ }$*|;
+    s|^\(# *\)pip-compile.*|\1${called_as[*]}|;
 " "$txtfile"
 
 echo "done"

compile-requirements

@@ -216,7 +216,9 @@ passlib==1.7.4
     #   -c ./requirements.txt.constraints
     #   flask-security-too
 pillow==11.1.0
-    # via passari
+    # via
+    #   -c ./requirements.txt.constraints
+    #   passari
 pluggy==1.5.0
     # via pytest
 propcache==0.2.1
@@ -337,4 +339,7 @@ yarl==1.18.3
     #   aiohttp
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools
+setuptools==75.8.1
+    # via
+    #   -c ./requirements.txt.constraints
+    #   sftpserver

requirements-dev.txt

@@ -2,7 +2,25 @@
 -e ./passari-workflow
 -e ./passari-web-ui
 
--c ./siptools-deps.txt
+# Siptools requirements
+#
+# These must be listed here, because passari uses dpres-siptools-ng and
+# dpres-mets-builder, and these are required them.  If those packages
+# are pushed to PyPI with proper requirements then this entire section
+# can be removed from here.
+dpres-siptools-ng @ https://github.com/andersinno/dpres-siptools-ng/releases/download/v1.0.0/dpres_siptools_ng-1.0.0-py3-none-any.whl
+dpres-mets-builder @ https://github.com/andersinno/dpres-mets-builder/releases/download/v1.0.0/dpres_mets_builder-1.0.0-py3-none-any.whl
+xml-helpers @ https://github.com/Digital-Preservation-Finland/xml-helpers/archive/refs/tags/v0.18.zip
+mets @ https://github.com/Digital-Preservation-Finland/mets/archive/refs/tags/v0.22.zip
+premis @ https://github.com/Digital-Preservation-Finland/premis/archive/refs/tags/v0.30.zip
+dpres-signature @ https://github.com/Digital-Preservation-Finland/dpres-signature/archive/refs/tags/v0.18.zip
+nisomix @ https://github.com/Digital-Preservation-Finland/nisomix/archive/refs/tags/v0.17.zip
+addml @ https://github.com/Digital-Preservation-Finland/addml/archive/refs/tags/v0.14.zip
+audiomd @ https://github.com/Digital-Preservation-Finland/audiomd/archive/refs/tags/v0.13.zip
+videomd @ https://github.com/Digital-Preservation-Finland/videomd/archive/refs/tags/v0.12.zip
+ffmpeg-python @ https://github.com/Digital-Preservation-Finland/ffmpeg-python/archive/refs/tags/0.2.0.zip
+opf-fido @ https://github.com/Digital-Preservation-Finland/fido/archive/refs/tags/v1.4.0-dpres5.zip
+file-scraper @ https://github.com/Digital-Preservation-Finland/file-scraper/archive/refs/tags/v0.78.zip
 
 # Limit upper version of some requirements, because the code doesn't
 # work with the newest versions (yet)
@@ -13,12 +31,19 @@ sqlalchemy<2
 rq-dashboard<0.6.2
 werkzeug<3
 
+# Install premis, because dpres-mets-builder requires it
+premis
+
 # Install pytz, because flask-security-too 3.4 requires flask-babelex
 # and flask-babelex uses pytz even though it doesn't list it as a
 # requirement.  NOTE: Newer flask-security-too (e.g. 5.5.2) doesn't seem
 # to require flask-babelex and pytz might become unnecessary.
 pytz
 
+# Install setuptools, because flask-security-too imports pkg_resources
+# which is provided by setuptools.  Note: pkg_resources is deprecated.
+setuptools
+
 # Pin psycopg2 to the version which is shipped by AlmaLinux 9 (from
 # package python3.12-psycopg2)
 psycopg2==2.9.6