Dockerize featureService

Author: c-w

.dockerignore

@@ -0,0 +1,6 @@
+node_modules/
+test/
+tools/
+.gitignore
+LICENSE
+README.md

Dockerfile

@@ -0,0 +1,26 @@
+FROM node:6.9
+
+RUN apt-get update && apt-get -qq install -y postgresql-client curl gzip
+
+WORKDIR /app
+ADD package.json /app/package.json
+RUN npm install
+
+ADD server.js /app/server.js
+ADD config.js /app/config.js
+ADD controllers /app/controllers
+ADD services /app/services
+ADD ddl /app/ddl
+ADD docker-entrypoint.sh /app/docker-entrypoint.sh
+
+CMD /app/docker-entrypoint.sh
+
+ENV PORT=80
+EXPOSE 80
+
+ENV FEATURES_DB_USER=""
+ENV FEATURES_DB_PASSWORD=""
+ENV FEATURES_DB_HOST=""
+ENV FEATURES_DB_PORT="5432"
+ENV FEATURES_DB_NAME="features"
+ENV FEATURES_DB_DUMP_URL="https://fortiscentral.blob.core.windows.net/locations/dump.fc.gz"

README.md

@@ -1,50 +1,32 @@
 # featureService #
 
-## Development setup ##
+## Setup ##
 
-### System dependencies ###
+In order to run this project on your machine, you will need to set up:
 
-In order to install on your local machine, you will need to install:
+- [Postgres on Azure](https://azure.microsoft.com/en-us/services/postgresql/)
+- [Docker](https://docs.docker.com/docker-for-windows/)
 
-- Postgres + postgis ([instructions for Ubuntu 16.04](http://www.gis-blog.com/how-to-install-postgis-2-3-on-ubuntu-16-04-lts/))
-- Node ([instructions for Ubuntu 16.04](https://www.digitalocean.com/community/tutorials/how-to-install-node-js-on-ubuntu-16-04))
-
-### Database setup ###
-
-Once the system dependencies are installed, go open a bash shell on the project directory and set up the database:
-
-```sh
-cat << EOF | sudo -u postgres psql
-CREATE DATABASE features;
-CREATE USER frontend WITH login password 'your_password_here';
-CREATE USER ops WITH login password 'your_other_password_here';
-GRANT ops TO postgres;
-GRANT frontend TO postgres;
-EOF
-sudo -u postgres psql -d features < schema.sql
-```
-
-### Application setup ###
-
-First, make sure the environment variables for the service are set:
-
-```sh
-export FEATURES_CONNECTION_STRING='postgres://frontend:[email protected]/features'
-export PORT=3035
-```
-
-Now you're ready to install and start the service:
+Once the system dependencies are installed, you can run the project via Docker:
 
 ```sh
-npm install
-node server.js
+docker build -t featureservice .
+
+docker run \
+  -p 8080:80 \
+  -e FEATURES_DB_USER="----CHANGEME----" \
+  -e FEATURES_DB_PASSWORD="----CHANGEME----" \
+  -e FEATURES_DB_HOST="----CHANGEME----" \
+  -t featureservice
 ```
 
-## Production setup ##
+The first time that you run this command, it will take a while as your Postgres
+on Azure instance is getting populated with over 2GB of global geo-spatial
+features.
 
-You can run the script `scripts/install.sh` to set up a production machine with the featureService and all its dependencies. The script will:
+After starting the service, you will be able to call the featureService, for
+example via the following requests:
 
-- Install postgres and postgis
-- Populate the postgres features database
-- Install the featureService
-- Autostart the featureService on port 80
+- http://localhost:8080/features/name/bogota
+- http://localhost:8080/features/point/18.678/15.123
+- http://localhost:8080/features/bbox/12.3/22.3/12.4/22.4

ddl/indices.sql

@@ -0,0 +1,7 @@
+CREATE INDEX features_hull_index ON features USING gist (hull);
+CREATE INDEX features_name_lower_index ON features(lower(name));
+CREATE INDEX features_namespace_index ON features(lower(split_part(id, '-', 1)));
+CREATE INDEX features_layer_lower_index ON features(lower(layer));
+
+CREATE INDEX visits_start_index ON visits (start);
+CREATE INDEX visits_userid_index ON visits (user_id);

schema.sql ddl/schema.sqlschema.sql renamed to ddl/schema.sql

@@ -23,11 +23,6 @@ CREATE TABLE features
   CONSTRAINT enforce_srid_hull CHECK (st_srid(hull) = 4326)
 );
 
-CREATE INDEX features_hull_index ON features USING gist (hull);
-CREATE INDEX features_name_lower_index ON features(lower(name));
-CREATE INDEX features_namespace_index ON features(lower(split_part(id, '-', 1)));
-CREATE INDEX features_layer_lower_index ON features(lower(layer));
-
 GRANT SELECT, UPDATE, INSERT, DELETE ON features TO frontend;
 
 ------------------------------------------------------------------------------
@@ -50,6 +45,3 @@ CREATE TABLE visits
 );
 
 GRANT SELECT, UPDATE, INSERT, DELETE ON visits TO frontend;
-
-CREATE INDEX visits_start_index ON visits (start);
-CREATE INDEX visits_userid_index ON visits (user_id);

docker-entrypoint.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+run_sql() {
+  PGSSLMODE="require" \
+  PGPASSWORD="${FEATURES_DB_PASSWORD}" \
+  psql \
+    --username="${FEATURES_DB_USER}" \
+    --port="${FEATURES_DB_PORT}" \
+    --host="${FEATURES_DB_HOST}" \
+    --dbname="$1" \
+}
+
+load_features_dump() {
+  PGSSLMODE="require" \
+  PGPASSWORD="${FEATURES_DB_PASSWORD}" \
+  pg_restore \
+    --username="${FEATURES_DB_USER}" \
+    --port="${FEATURES_DB_PORT}" \
+    --host="${FEATURES_DB_HOST}" \
+    --dbname="${FEATURES_DB_NAME}" \
+    --jobs="$(grep -c ^processor /proc/cpuinfo)" \
+    "$1"
+}
+
+features_database_exists() {
+  echo "SELECT 'yes' FROM pg_database WHERE datname='${FEATURES_DB_NAME}';" \
+  | run_sql 'postgres' \
+  | grep -q 'yes'
+}
+
+log() {
+  echo "[$(date)] $1"
+}
+
+fail() {
+  echo "$1" >&2
+  exit 1
+}
+
+if [ -z "${FEATURES_DB_USER}" ]; then fail "Need to provide FEATURES_DB_USER environment variable"; fi
+if [ -z "${FEATURES_DB_PASSWORD}" ]; then fail "Need to provide FEATURES_DB_PASSWORD environment variable"; fi
+if [ -z "${FEATURES_DB_HOST}" ]; then fail "Need to provide FEATURES_DB_HOST environment variable"; fi
+
+if ! features_database_exists; then
+  dump_file="/tmp/db.fc.gz"
+
+  log "Setting up database..."
+  echo "CREATE DATABASE ${FEATURES_DB_NAME};" | run_sql 'postgres'
+  echo "CREATE USER ops WITH login password 'changeme';" | run_sql 'postgres'
+  echo "CREATE USER frontend WITH login password 'changeme';" | run_sql 'postgres'
+  echo "ALTER USER ops WITH password '${FEATURES_DB_PASSWORD}';" | run_sql 'postgres'
+  echo "ALTER USER frontend WITH password '${FEATURES_DB_PASSWORD}';" | run_sql 'postgres'
+  echo "GRANT ops TO ${FEATURES_DB_USER%@*};" | run_sql 'postgres'
+  echo "GRANT frontend TO ${FEATURES_DB_USER%@*};" | run_sql 'postgres'
+  log "...done, database is now set up"
+
+  log "Setting up schema..."
+  < /app/ddl/schema.sql run_sql "${FEATURES_DB_NAME}"
+  log "...done, schema is now set up"
+
+  log "Fetching database dump..."
+  curl --silent "${FEATURES_DB_DUMP_URL}" > "${dump_file}"
+  log "...done, database dump is now available"
+
+  log "Ingesting database dump..."
+  load_features_dump "${dump_file}"
+  rm "${dump_file}"
+  log "...done, database dump is now ingested"
+
+  log "Setting up indices..."
+  < /app/ddl/indices.sql run_sql "${FEATURES_DB_NAME}"
+  log "...done, indices are now set up"
+
+  log "Improving query planner..."
+  echo "ANALYZE;" | run_sql "${FEATURES_DB_NAME}"
+  log "...done, query planner is now ready"
+fi
+
+FEATURES_CONNECTION_STRING="postgres://frontend:${FEATURES_DB_PASSWORD}@${FEATURES_DB_HOST}:${FEATURES_DB_PORT}/${FEATURES_DB_NAME}?ssl=true" \
+npm start

package.json

@@ -26,6 +26,7 @@
     "uuid": "^3.0.1"
   },
   "scripts": {
+    "start": "node server.js",
     "test": "PORT=3035 FEATURES_CONNECTION_STRING='postgres://frontend:[euro4sure]@127.0.0.1/geofeatures' mocha"
   },
   "devDependencies": {