Merge pull request #18 from stac-utils/tests/rde/ingest-issue

Add dockerized development environment; pypgstac testing; CI - fix issue with ingest

Author: lossyrob

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 88
+extend-ignore = E203, W503, E731, E722
+per-file-ignores = __init__.py:F401

.github/workflows/continuous-integration.yml

@@ -0,0 +1,16 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  test:
+    name: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Execute linters and test suites
+        run: ./scripts/cibuild

.github/workflows/release.yml

@@ -0,0 +1,30 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  release:
+    name: release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python 3.x
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+
+      - name: Install release dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+
+      - name: Build and publish package
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_STACUTILS_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_STACUTILS_PASSWORD }}
+        run: |
+          scripts/cipublish

.gitignore

@@ -1,2 +1,6 @@
 .envrc
 pypgstac/dist
+*.pyc
+*.egg-info
+*.eggs
+venv

Dockerfile.dev

@@ -0,0 +1,17 @@
+FROM python:3.8-slim
+
+ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt
+
+RUN mkdir -p /opt/src/pypgstac
+
+WORKDIR /opt/src/pypgstac
+
+COPY pypgstac/requirements-dev.txt /opt/src/pypgstac/requirements-dev.txt
+RUN pip install -r requirements-dev.txt
+
+COPY pypgstac /opt/src/pypgstac
+RUN pip install .
+
+ENV PYTHONPATH=/opt/src/pypgstac:${PYTHONPATH}
+
+WORKDIR /opt/src

README.md

@@ -20,7 +20,7 @@ poetry build pypgstac
 pip install dist/pypgstac-[version]-py3-none-any.whl
 ```
 
-# Migrations
+## Migrations
 To install the latest version of pgstac on an empty database, you can directly load the primary source code.
 ```
 psql -f pgstac.sql
@@ -33,20 +33,20 @@ For each new version of PGStac, two migrations should be added to pypgstac/pypgs
  - pgstac.[version].sql (equivalent to `cat sql/*.sql > migration.sql && echo "insert into migrations (versions) VALUES ('[version]')" >> migration.sql)
  - pgstac.[version].[fromversion].sql (Migration to move from existing version to new version, can be created by hand or using the makemigration.sh tool below)
 
-## Running Migrations
+### Running Migrations
 Migrations can be installed by either directly running the appropriate migration sql file for from and target PGStac versions:
 `psql -f pypgstac/pypgstac/migrations/pgstac.0.1.8.sql`
 
 Or by using pypgstac:
 `pypgstac migrate`
 
-## Creating Migrations Using Schema Diff
+### Creating Migrations Using Schema Diff
 To create a migration from a previous version of pgstac you can calculate the migration from the running instance of pgstac using the makemigration.sh command. This will use docker to copy the schema of the existing database and the new sql into new docker databases and create/test the migration between the two.
 ```
 makemigration.sh postgresql://myuser:mypassword@myhost:myport/mydatabase
 ```
 
-# Bulk Data Loading
+## Bulk Data Loading
 A python utility is included which allows to load data from any source openable by smart-open using python in a memory efficient streaming manner using PostgreSQL copy. There are options for collections and items and can be used either as a command line or a library.
 
 To load an ndjson of items directly using copy (will fail on any duplicate ids but is the fastest option to load new data you know will not conflict)
@@ -63,3 +63,41 @@ To upsert any records, adding anything new and replacing anything with the same
 ```
 pypgstac load items --method upsert
 ```
+
+## Development
+
+PGStac uses a dockerized development environment. You can set this up using:
+
+```bash
+scripts/setup
+```
+
+To bring up the development database:
+```
+scripts/server
+```
+
+To run tests, use:
+```bash
+scripts/test
+```
+
+To rebuild docker images:
+```bash
+scripts/update
+```
+
+To drop into a console, use
+```bash
+scripts/console
+```
+
+To drop into a psql console on the database container, use:
+```bash
+scripts/console --db
+```
+
+To run migrations on the development database, use
+```bash
+scripts/migrate
+```

docker-compose.yml

@@ -0,0 +1,33 @@
+services:
+  dev:
+    container_name: pgstac-dev
+    image: pgstac-dev
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    depends_on:
+      - database
+    volumes:
+      - ./:/opt/src
+    environment:
+      - PGUSER=username
+      - PGPASSWORD=password
+      - PGHOST=database
+      - PGDATABASE=postgis
+  database:
+    container_name: pgstac-db
+    image: pgstac-db
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      - POSTGRES_USER=username
+      - POSTGRES_PASSWORD=password
+      - POSTGRES_DB=postgis
+    ports:
+      - "5432:5432"
+    volumes:
+      - pgstac-pgdata:/var/lib/postgresql/data
+      - ./:/opt/src
+volumes:
+  pgstac-pgdata:

mypy.ini

@@ -0,0 +1,4 @@
+[mypy]
+ignore_missing_imports = True
+disallow_untyped_defs = True
+namespace_packages = True

pgstac.sql

@@ -3,4 +3,5 @@ BEGIN;
 \i sql/002_collections.sql
 \i sql/003_items.sql
 \i sql/004_search.sql
+\i sql/999_version.sql
 COMMIT;

pypgstac/pypgstac/__init__.py

@@ -1 +1 @@
-__version__ = '0.2.7'
+__version__ = "0.2.7"