Merge pull request #157 from cipherstash/feat/optionally-trigger-eql-install-on-container-start

feat: optionally trigger EQL install on container start

Author: auxesis

README.md

@@ -208,7 +208,27 @@ TODO: Add instructions for running Proxy locally
 
 ### Setting up the database schema
 
-TODO: Add instructions for setting up the database schema
+Under the hood, Proxy uses [CipherStash Encrypt Query Language](https://github.com/cipherstash/encrypt-query-language/) to index and search encrypted data.
+
+When you start the Proxy container, you can install EQL by setting the `CS_DATABASE__INSTALL_EQL` environment variable:
+
+```bash
+CS_DATABASE__INSTALL_EQL=true
+```
+
+This will install the version of EQL bundled with the Proxy container.
+The version of EQL bundled with the Proxy container is tested to work with that version of Proxy.
+
+If you are following the [getting started](#getting-started) guide above, EQL is automatically installed for you.
+You can also install EQL by running [the installation script](https://github.com/cipherstash/encrypt-query-language/releases) as a database migration in your application.
+
+Once you have installed EQL, you can see what version is installed by querying the database:
+
+```sql
+SELECT cs_eql_version();
+```
+
+This will output the version of EQL installed.
 
 #### Creating columns with the right types
 

docker-compose.yml

@@ -39,6 +39,7 @@ services:
       - CS_DATABASE__HOST=postgres
       - CS_DATABASE__PORT=5432
       - CS_PROMETHEUS__ENABLED=${CS_PROMETHEUS__ENABLED:-true}
+      - CS_DATABASE__INSTALL_EQL=true # install EQL into the PostgreSQL database we start above
     networks:
       - cipherstash
 

docker-entrypoint.sh

@@ -1,6 +1,34 @@
 #!/usr/bin/env bash
 set -eu
 
+DATABASE_URL="postgresql://${CS_DATABASE__USERNAME}:${CS_DATABASE__PASSWORD}@${CS_DATABASE__HOST}:${CS_DATABASE__PORT}/${CS_DATABASE__NAME}"
+
+postgres_ready () {
+  psql ${DATABASE_URL} -c "SELECT 1" > /dev/null 2>&1
+}
+
+wait_for_postgres_or_exit() {
+  host=${CS_DATABASE__HOST}
+  port=${CS_DATABASE__PORT}
+  max_retries=20
+  interval=0.5
+  attempt=1
+  echo "Testing presence of PostgreSQL at ${host}:${port} with a maximum of ${max_retries} retries"
+
+  until postgres_ready
+  do
+    if [ $attempt -lt $max_retries ]; then
+      echo "Waiting for ${host}:${port}"
+      sleep $interval
+      attempt=$(expr $attempt + 1)
+    else
+      echo "Unable to connect to ${host}:${port} after ${max_retries} attempts"
+      exit 64
+    fi
+  done
+  echo "Connected to ${host}:${port} after ${attempt} attempts"
+}
+
 : "${CS_DATABASE__AWS_BUNDLE_PATH:=./aws-rds-global-bundle.pem}"
 
 # Optionally pull in the AWS RDS global certificate bundle. This is required
@@ -30,4 +58,31 @@ case "${CS_DATABASE__INSTALL_AWS_RDS_CERT_BUNDLE:-}" in
     ;;
 esac
 
+# Optionally install EQL in the target database
+case "${CS_DATABASE__INSTALL_EQL:-}" in
+  "true") ;&
+  "yes") ;&
+  "1")
+    >&2 echo "Installing EQL in target PostgreSQL database..."
+
+    if [ ! -f "/opt/cipherstash-eql.sql" ]; then
+      >&2 echo "error: unable to find EQL installer at: /opt/cipherstash-eql.sql"
+      exit 1
+    fi
+
+    # Wait for postgres to become available
+    wait_for_postgres_or_exit
+
+    # Attempt to install EQL
+    psql --file=/opt/cipherstash-eql.sql --quiet $DATABASE_URL > /dev/null 2>&1
+    if [ $? != 0 ]; then
+      >&2 echo "error: unable to install EQL in target PostgreSQL database!"
+      exit 2
+    fi
+    ;;
+  *)
+    >&2 echo "Not installing EQL in target PostgreSQL database."
+    ;;
+esac
+
 exec cipherstash-proxy "$@"

mise.toml

@@ -52,21 +52,8 @@ run = """
 {% set target = arch() ~ "-unknown-linux-gnu" | replace(from="arm64", to="aarch64") | replace(from="x64", to="x86_64") %}
 {% set docker_platform = "linux/" ~ arch() | replace(from="x64", to="amd64") %}
 
-{# If we are on macos, cross-compile for Linux, so we can run the binary in a Docker container. #}
-{# Only supports Apple Silicon. #}
-{% if os() == "macos" %}
-if ! which {{ target }}-gcc ; then
-  brew install MaterializeInc/crosstools/aarch64-unknown-linux-gnu
-fi
-export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER={{ target }}-gcc
-{% endif %}
-
-# cross-compile
-rustup target add --toolchain stable {{ target }}
-
-cargo build --locked --target {{ target }} --release --package cipherstash-proxy
-
-cp {{config_root}}/target/{{ target }}/release/cipherstash-proxy {{config_root}}/
+# build a binary
+mise run build:binary --target {{target}}
 
 # build a new container
 mise run build:docker --platform {{docker_platform}}
@@ -514,8 +501,29 @@ mise run build:docker --platform {{option(name="platform",default=default_platfo
 [tasks."build:binary"]
 description = "Build a releasable binary for cipherstash-proxy"
 run = """
-cargo build --locked --release --package cipherstash-proxy
-cp -v target/release/cipherstash-proxy .
+#!/bin/bash
+{% set default_target_arch = arch() | replace(from="arm64", to="aarch64") | replace(from="x64", to="x86_64") %}
+{% set default_target_os = os() | replace(from="linux", to="unknown-linux-gnu") | replace(from="macos", to="apple-darwin") %}
+{% set default_target = default_target_arch ~ "-" ~ default_target_os %}
+{% set target = option(name="target", default=default_target) %}
+
+{# If we are on macos and are cross-compiling for Linux, set up a linker and toolchain. #}
+{# Only supports cross-compiling to Linux/ARM64. #}
+{% if os() == "macos" %}
+if [[ "{{option(name="target", default=default_target)}}" =~ "linux" ]]; then
+  if ! which {{ target }}-gcc ; then
+    brew install MaterializeInc/crosstools/aarch64-unknown-linux-gnu
+  fi
+  export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER={{ target }}-gcc
+fi
+{% endif %}
+
+# cross-compile
+rustup target add --toolchain stable {{ target }}
+
+cargo build --locked --target {{ target }} --release --package cipherstash-proxy
+
+cp -v {{config_root}}/target/{{ target }}/release/cipherstash-proxy {{config_root}}/
 """
 
 [tasks."build:docker"]

proxy.Dockerfile

@@ -6,6 +6,7 @@ RUN apt update && apt install -y ca-certificates postgresql-client curl
 
 # Copy binary
 COPY cipherstash-proxy /usr/local/bin/cipherstash-proxy
+# Copy entrypoint, for handling Proxy startup
 COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
 # Copy EQL install scripts

tests/docker-compose.yml

@@ -57,10 +57,7 @@ services:
       - CS_DEFAULT_KEYSET_ID=${CS_DEFAULT_KEYSET_ID}
       - CS_CLIENT_KEY=${CS_CLIENT_KEY}
       - CS_CLIENT_ID=${CS_CLIENT_ID}
-
       - CS_PROMETHEUS__ENABLED=${CS_PROMETHEUS__ENABLED:-true}
-
-
     networks:
       - postgres