Add scripts to control dev environment

Author: lossyrob

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 88
+extend-ignore = E203, W503, E731, E722
+per-file-ignores = __init__.py:F401

README.md

@@ -20,7 +20,7 @@ poetry build pypgstac
 pip install dist/pypgstac-[version]-py3-none-any.whl
 ```
 
-# Migrations
+## Migrations
 To install the latest version of pgstac on an empty database, you can directly load the primary source code.
 ```
 psql -f pgstac.sql
@@ -33,20 +33,20 @@ For each new version of PGStac, two migrations should be added to pypgstac/pypgs
  - pgstac.[version].sql (equivalent to `cat sql/*.sql > migration.sql && echo "insert into migrations (versions) VALUES ('[version]')" >> migration.sql)
  - pgstac.[version].[fromversion].sql (Migration to move from existing version to new version, can be created by hand or using the makemigration.sh tool below)
 
-## Running Migrations
+### Running Migrations
 Migrations can be installed by either directly running the appropriate migration sql file for from and target PGStac versions:
 `psql -f pypgstac/pypgstac/migrations/pgstac.0.1.8.sql`
 
 Or by using pypgstac:
 `pypgstac migrate`
 
-## Creating Migrations Using Schema Diff
+### Creating Migrations Using Schema Diff
 To create a migration from a previous version of pgstac you can calculate the migration from the running instance of pgstac using the makemigration.sh command. This will use docker to copy the schema of the existing database and the new sql into new docker databases and create/test the migration between the two.
 ```
 makemigration.sh postgresql://myuser:mypassword@myhost:myport/mydatabase
 ```
 
-# Bulk Data Loading
+## Bulk Data Loading
 A python utility is included which allows to load data from any source openable by smart-open using python in a memory efficient streaming manner using PostgreSQL copy. There are options for collections and items and can be used either as a command line or a library.
 
 To load an ndjson of items directly using copy (will fail on any duplicate ids but is the fastest option to load new data you know will not conflict)
@@ -63,3 +63,41 @@ To upsert any records, adding anything new and replacing anything with the same
 ```
 pypgstac load items --method upsert
 ```
+
+## Development
+
+PGStac uses a dockerized development environment. You can set this up using:
+
+```bash
+scripts/setup
+```
+
+To bring up the development database:
+```
+scripts/server
+```
+
+To run tests, use:
+```bash
+scripts/test
+```
+
+To rebuild docker images:
+```bash
+scripts/update
+```
+
+To drop into a console, use
+```bash
+scripts/console
+```
+
+To drop into a psql console on the database container, use:
+```bash
+scripts/console --db
+```
+
+To run migrations on the development database, use
+```bash
+scripts/migrate
+```

mypy.ini

@@ -0,0 +1,4 @@
+[mypy]
+ignore_missing_imports = True
+disallow_untyped_defs = True
+namespace_packages = True

scripts/bin/format

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+if [[ "${CI}" ]]; then
+    set -x
+fi
+
+function usage() {
+    echo -n \
+        "Usage: $(basename "$0")
+Format code.
+
+This scripts is meant to be run inside the dev container.
+
+"
+}
+
+if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
+    echo "Formatting pypgstac..."
+    black pypgstac/pypgstac
+    black pypgstac/tests
+fi

scripts/bin/test

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -e
+
+if [[ "${CI}" ]]; then
+    set -x
+fi
+
+function usage() {
+    echo -n \
+        "Usage: $(basename "$0")
+Runs tests for the project.
+
+This scripts is meant to be run inside the dev container.
+
+"
+}
+
+if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
+
+    echo "Running mypy..."
+    mypy pypgstac/pypgstac pypgstac/tests
+
+    echo "Running black..."
+    black --check pypgstac/pypgstac pypgstac/tests
+
+    echo "Running flake8..."
+    flake8 pypgstac/pypgstac pypgstac/tests
+
+    echo "Running unit tests..."
+    python -m unittest discover pypgstac/tests
+
+fi

scripts/cibuild

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+if [[ "${CI}" ]]; then
+    set -x
+fi
+
+function usage() {
+    echo -n \
+        "Usage: $(basename "$0")
+CI build for this project.
+"
+}
+
+if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
+    scripts/setup
+    scripts/test
+fi

scripts/cipublish

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -e
+
+if [[ "${CI}" ]]; then
+    set -x
+fi
+
+function usage() {
+    echo -n \
+        "Usage: $(basename "$0") [--db]
+Start a console in the dev container
+
+--db: Instead, start a psql console in the database container.
+"
+}
+
+while [[ "$#" > 0 ]]; do case $1 in
+    --db)
+        DB_CONSOLE=1
+        shift
+        ;;
+    *)
+        usage "Unknown parameter passed: $1"
+        shift
+        shift
+        ;;
+    esac; done
+
+if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
+
+    if [[ "${DB_CONSOLE}" ]]; then
+        scripts/server --detach
+
+        docker-compose \
+            -f docker-compose.yml \
+            run --rm \
+            database \
+            psql postgres://username:password@database:5432/postgis
+
+        exit 0
+    fi
+
+    # Run database migrations
+    docker-compose \
+        -f docker-compose.yml \
+        run --rm dev \
+        /bin/bash
+
+fi

scripts/console

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+if [[ "${CI}" ]]; then
+    set -x
+fi
+
+function usage() {
+    echo -n \
+        "Usage: $(basename "$0")
+Format code in this project
+
+"
+}
+
+if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
+    docker-compose  \
+        run --rm \
+        dev scripts/bin/format;
+fi

scripts/format

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+if [[ "${CI}" ]]; then
+    set -x
+fi
+
+function usage() {
+    echo -n \
+        "Usage: $(basename "$0")
+Run migrations against the development database.
+"
+}
+
+if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
+
+    # Run database migrations
+    docker-compose \
+        -f docker-compose.yml \
+        run --rm dev \
+        bash -c "pypgstac pgready && pypgstac migrate"
+
+fi