diff --git a/.gitattributes b/.gitattributes index 8354139..fd11be8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,5 @@ -#ensure that needed psql.sh script (which is used within the docker container (which is based on unix system)) is correctly set tp LF -/sql/pg_restore.sh text eol=lf +#ensure that *.sh scripts (which are used on docker containers (unix) or directly on unix) are correctly set to LF lineendings +*.sh text eol=lf #ensure that data dump is not modified by git /sql/data binary \ No newline at end of file diff --git a/.github/workflows/docker-publish-db-hardrestore.yml b/.github/workflows/docker-publish-db-hardrestore.yml new file mode 100644 index 0000000..0de19d0 --- /dev/null +++ b/.github/workflows/docker-publish-db-hardrestore.yml @@ -0,0 +1,71 @@ +name: Build and Push Docker image for Database Hardrestore + +on: + workflow_dispatch: + +permissions: + id-token: write + contents: write + packages: write + attestations: write + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: prepare docker metadata + uses: docker/metadata-action@v5 + id: meta + with: + images: | + ghcr.io/${{ github.repository }}/db-hardrestore + tags: | + type=sha + type=ref,event=tag + type=semver,pattern={{version}} + + - name: Build and push image + id: push + uses: docker/build-push-action@v6 + with: + context: sql/ + file: sql/Dockerfile-hardrestore-without-extensions + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Get first tag + id: firsttag + uses: actions/github-script@v8 + env: + ALLTAGS: ${{ steps.meta.outputs.tags }} + with: + script: | + const firsttag = process.env.ALLTAGS.split('\n')[0] + core.setOutput('firsttag', firsttag) + + - name: Generate SBOM + uses: anchore/sbom-action@v0 + with: + image: ${{ steps.firsttag.outputs.firsttag }} + format: 'cyclonedx-json' + output-file: 'sbom.cyclonedx.json' + + - name: Attest + uses: actions/attest-sbom@v3 + id: attest + with: + subject-name: ghcr.io/${{ github.repository }}/db-hardrestore + subject-digest: ${{ steps.push.outputs.digest }} + sbom-path: 'sbom.cyclonedx.json' diff --git a/docker-compose.yml b/docker-compose.yml index 66a4df2..c63c3ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,7 @@ services: - postgres_data:/var/lib/postgresql/data # Persistent data healthcheck: - test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\""] + test: ["CMD-SHELL", "pg_isready --user=\"$$POSTGRES_USER\" --dbname=\"$$POSTGRES_DB\""] # specifying user & db will not fail the healthcheck if they don't exist but will add a logentry interval: 60s timeout: 10s retries: 3 diff --git a/sql/Dockerfile-hardrestore-without-extensions b/sql/Dockerfile-hardrestore-without-extensions new file mode 100644 index 0000000..c30dece --- /dev/null +++ b/sql/Dockerfile-hardrestore-without-extensions @@ -0,0 +1,7 @@ +FROM alpine/psql:15.5 + +WORKDIR /db_restore/ +COPY data /db_restore/ +COPY --chmod=777 pg_hardrestore_by_connection_uri.sh /db_restore/ + +ENTRYPOINT ["/db_restore/pg_hardrestore_by_connection_uri.sh"] \ No newline at end of file diff --git a/sql/README.md b/sql/README.md new file mode 100644 index 0000000..8717169 --- /dev/null +++ b/sql/README.md @@ -0,0 +1,20 @@ +# Dataset and scripts for docker compose + +When running the docker compose dev setup, it will mount some scripts and the `data`-file into the database container. The postgres db in the container will then set itself up with the default dataset. That dataset is compatible for use with the GeospatialAnalyzer backend when the backend is configured with the default `topic.json`. The data is restored from the `data`-file in this repository which is created with [`pg_dump`](https://www.postgresql.org/docs/15/app-pgdump.html) and uses the custom postgres backup file format. + +# Dockerfile-hardrestore-without-extensions - Hardrestore database utility image + +This directory contains a Dockerfile to generate a database utility container image. It uses the same default dataset used in the docker compose dev setup. It is especially useful for initializing / restoring a database for use with the geospatialanalyzer in an orchestrated and isolated environment (e.g. kubernetes cluster). + +This image does not contain any database itself. It takes a [postgres connection string](https://www.postgresql.org/docs/15/libpq-connect.html#LIBPQ-CONNSTRING) pointing to an external database server and restores the default data dump to that database. It will first _wipe clean all relations from the specified database that are also contained in the dump_. Just deploy the image as a container to an environment were it can reach the postgres database and set the `POSTGRES_DB_URI` environment variable with the connection details. + +The database specified has to contain the `postgis` and the `postgis_raster` extensions. This image can be used to bring testing data to an existing or by other means automatically created database. It is used for this goal for testing kubernetes deployments with managed postgres clusters, cf. [the kubernetes deployment configuration for the geospatialanalyzer backend](https://github.com/geobakery/gsa-deployment/blob/main/README.md#database-initialization-or-restoration). + +Build command (from the parent folder; remove `sql` from both paths if run in this folder): +```bash +docker buildx build -f sql/Dockerfile-hardrestore-without-extensions -t sql/ +``` + +## Building custom utility images + +To supply a custom dataset fork the repository and switch out the `data` file in this directory before building the utility Dockerfile. You can use any file compatbile with [`pg_restore`](https://www.postgresql.org/docs/15/app-pgrestore.html). \ No newline at end of file diff --git a/sql/pg_hardrestore_by_connection_uri.sh b/sql/pg_hardrestore_by_connection_uri.sh new file mode 100644 index 0000000..0be9b2c --- /dev/null +++ b/sql/pg_hardrestore_by_connection_uri.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -e + +waiting_for() +{ + if [ $i -eq 6 ]; then + >&2 echo "$1 still not ready. Aborting." + exit 1 + fi + + >&2 echo "Waiting for $1 ..." + sleep 5 + i=$(($i+1)) +} + +i=0 +until pg_isready --dbname="$POSTGRES_DB_URI"; do + waiting_for "Database (pg_ready)" +done + +i=0 +until psql --dbname="$POSTGRES_DB_URI" -c "\dx" | grep -w postgis; do + waiting_for "postgis extension" +done + +pg_restore --dbname="$POSTGRES_DB_URI" --no-owner --clean --if-exists --verbose ./data \ No newline at end of file diff --git a/sql/pg_restore.sh b/sql/pg_restore.sh index db8e66d..638373e 100644 --- a/sql/pg_restore.sh +++ b/sql/pg_restore.sh @@ -4,8 +4,9 @@ set -e echo "Starting database restore from custom format dump..." # Restore the database dump -pg_restore -U "postgres" -d "postgres" -v '/docker-entrypoint-initdb.d/data' 2>&1 || { +# use docker image env vars, c.f. https://hub.docker.com/_/postgres#initialization-scripts +pg_restore --username="$POSTGRES_USER" --dbname="$POSTGRES_DB" --no-owner -v '/docker-entrypoint-initdb.d/data' 2>&1 || { echo "Note: Some warnings are expected during restore (e.g., extensions already exist)" } -echo "Database restore completed!" \ No newline at end of file +echo "Database restore completed!"