Merge pull request #430 from mapswipe/dev

Code Style, Tutorial Text, Firebase Contributor Count

Author: Hagellach37

.gitignore

@@ -114,13 +114,10 @@ ENV/
 # serviceAccountKey for Google Platform access
 *serviceAccountKey.json
 
-# don't upload config files and api keys etc. for python_workers
-mapswipe_workers/config/configuration.json
-
 # don´t upload data dir
 data/
-
 mapswipe-data/
+postgres-data/
 
 # don't upload app config for project managers dashboard
 app.js
@@ -135,3 +132,5 @@ nginx.conf
 
 .DS_Store
 
+# Custom setup script
+setup-custom.sh

.travis.yml

@@ -3,62 +3,56 @@
 #
 # For technical details, see http://travis-ci.org/
 #
-branches:
-  except:
-    - docs
-
 dist: bionic
-addons:
-  apt:
-    sources:
-      - ppa:ubuntugis/ppa
-    update: true
-    packages:
-      - gdal-bin
-      - python-gdal
-      - python3-gdal
-      - libgdal-dev
-
 language: python
 python:
   - "3.6"
-virtualenv:
-  system_site_packages: true
-
 services:
   - docker
+addons:
+  ssh_known_hosts:
+    - 35.190.148.87  # Development server
+env:
+  - ANSIBLE_CONFIG=ansible/ansible.cfg
 
 before_install:
-  - echo "$SERVICE_ACCOUNT_KEY" > mapswipe_workers/serviceAccountKey.json
-  - echo "MOCK_SERVICE_ACCOUNT_KEY" > postgres/serviceAccountKey.json
+  - openssl aes-256-cbc -K $encrypted_444bb228fd3f_key -iv $encrypted_444bb228fd3f_iv -in travis/ci-mapswipe-firebase-adminsdk-80fzw-21ddcd1649.json.enc -out mapswipe_workers/serviceAccountKey.json -d
+  # Create a mock file for wal-g setup
+  - touch postgres/serviceAccountKey.json
 
 install:
   - pip install --upgrade pip setuptools
-  - pip install flake8 black
+  - pip install black==19.10b0 flake8==3.8.3 isort==5.5.2
+  # Install ansible for automated deployment
+  - sudo apt-get --quiet update
+  - sudo apt-get --yes install software-properties-common
+  - sudo apt-add-repository --yes --update ppa:ansible/ansible
+  - sudo apt-get --yes install ansible
 
 script:
+  - black --check mapswipe_workers
+  - flake8 --count --config mapswipe_workers/setup.cfg mapswipe_workers
+  - isort --check --settings-file mapswipe_workers/setup.cfg mapswipe_workers
   - docker-compose up --build --detach firebase_deploy
+  - docker-compose up --build --detach postgres
   - docker-compose run mapswipe_workers python -m unittest discover --verbose --start-directory tests/unittests/
-  #- docker-compose run mapswipe_workers python -m unittest discover --verbose --start-directory tests/integration/
-  # - TODO: Run black and flake8
-
-# before_deploy:
-#   # SSH setup to deploy to server after build.
-#   - apk add openssh-client
-#   - mkdir -p ~/.ssh
-#   - chmod 700 ~/.ssh
-#   - eval "$(ssh-agent -s)"
-#   - export KEY_FILE=~/.ssh/api_server_ssh_key
-#   - echo "$DEPLOYMENT_SERVER_PRIVATE_KEY" | tr -d '\r' > $KEY_FILE
-#   - chmod 600 $KEY_FILE
-#   - ssh-add $KEY_FILE > /dev/null
-#   - ssh-keyscan -H "$DEPLOYMENT_SERVER_IP" >> ~/.ssh/known_hosts
-#   - chmod 644 ~/.ssh/known_hosts
-#   - ssh -i $KEY_FILE -o StrictHostKeyChecking=no "mapswipe@$DEPLOYMENT_SERVER_IP"
-
-# deploy:
-#   provider: script
-#   script: bash scripts/deploy.sh
-#   on:
-#     branch: dev
-
+  - docker-compose run mapswipe_workers python -m unittest discover --verbose --start-directory tests/integration/
+
+after_success:
+  # Deploy to development server using Ansible
+  # SSH setup to deploy to server after build.
+  - apk add openssh-client
+  - eval "$(ssh-agent -s)"  #start the ssh agent
+  - mkdir -p $HOME/.ssh
+  # - chmod 700 $HOME/.ssh
+  - cp travis/ssh_config $HOME/.ssh/config
+  - openssl aes-256-cbc -K $encrypted_c17fa81e6490_key -iv $encrypted_c17fa81e6490_iv -in travis/ssh-private-key.enc -out $HOME/.ssh/private-key -d
+  - chmod 600 $HOME/.ssh/private-key
+  - ssh-add $HOME/.ssh/private-key
+  - chmod +x deploy.sh
+
+deploy:
+  provider: script
+  script: bash deploy.sh
+  on:
+    branch: dev

ansible/ansible.cfg

@@ -0,0 +1,5 @@
+[defaults]
+INVENTORY = inventory.yml
+
+[ssh_connection]
+pipelining = True  # Reuse connection. Make execution faster.

ansible/inventory.yml

@@ -0,0 +1,12 @@
+all:
+  hosts:
+    development:
+      ansible_host: 35.190.148.87
+      ansible_user: mapswipe
+      # Uncomment following line when not using ~/.ssh/config
+      # ansible_ssh_private_key_file: $HOME/.ssh/private-key
+    production:
+      ansible_host: 35.231.228.82
+      ansible_user: mapswipe
+      # Uncomment following line when not using ~/.ssh/config
+      # ansible_ssh_private_key_file: $HOME/.ssh/private-key

ansible/playbook.yml

@@ -0,0 +1,18 @@
+---
+- name: Deploy MapSwipe Backend
+  hosts: development
+
+  tasks:
+    - name: Fetch latest code base from GitHub.
+      git:
+        repo: 'https://github.com/mapswipe/python-mapswipe-workers.git'
+        dest: $HOME/python-mapswipe-workers
+        version: dev
+    - name: Rebuild Docker container.
+      command: docker-compose build postgres firebase_deploy mapswipe_workers manager_dashboard nginx api
+      args:
+        chdir: $HOME/python-mapswipe-workers
+    - name: Restart Docker container.
+      command: docker-compose up -d --force-recreate postgres firebase_deploy mapswipe_workers manager_dashboard nginx api
+      args:
+        chdir: $HOME/python-mapswipe-workers

deploy.sh

@@ -1,88 +1,4 @@
-git pull
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-python3 test_config.py
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose build --no-cache postgres
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose build --no-cache firebase_deploy
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose build --no-cache mapswipe_workers
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose build --no-cache manager_dashboard
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose build --no-cache nginx
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose build --no-cache api
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose up -d --force-recreate postgres firebase_deploy mapswipe_workers manager_dashboard nginx api
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker logs firebase_deploy
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker ps -a
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
+#!/bin/bash
+# Deploy to dev server.
+# This script is called by Travis.
+ansible-playbook -i ansible/inventory.yml ansible/playbook.yml

docs/source/conf.py

@@ -13,9 +13,9 @@
 
 # -- Project information -----------------------------------------------------
 
-project = 'MapSwipe Back-End'
-copyright = '2019, MapSwipe'
-author = 'MapSwipe'
+project = "MapSwipe Back-End"
+copyright = "2019, MapSwipe"
+author = "MapSwipe"
 
 
 # -- General configuration ---------------------------------------------------
@@ -24,14 +24,14 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.napoleon',
-    'sphinx_markdown_tables',
-    'm2r',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
+    "sphinx_markdown_tables",
+    "m2r",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # For conversion from markdown to html
 # Add a source file parser for markdwon
@@ -42,7 +42,7 @@
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
-source_suffix = ['.rst', '.md']
+source_suffix = [".rst", ".md"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -55,12 +55,12 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 
 # -- Extension configuration -------------------------------------------------

docs/source/contributing.md

@@ -21,12 +21,16 @@ git push -u origin featureA
 
 ## Style Guide
 
-This project uses [black](https://github.com/psf/black) and [flake8](https://gitlab.com/pycqa/flake8) to achieve a unified style.
+This project uses [black](https://github.com/psf/black), [flake8](https://gitlab.com/pycqa/flake8) and [isort](https://github.com/PyCQA/isort) to achive a constisted style across the project.
+The configuration of flake8 and isort is stored in `setup.cfg`.
 
 Use [pre-commit](https://pre-commit.com/) to run `black` and `flake8` prior to any git commit. `pre-commit`, `black` and `flake8` should already be installed in your virtual environment since they are listed in `requirements.txt`. To setup pre-commit simply run:
 
 ```
 pre-commit install
 ```
 
-From now on `black` and `flake8` should run automatically whenever `git commit` is executed.
+From now on `black`, `flake8` and `isort` should run automatically whenever `git commit` is executed.
+
+When running those tools manually please make sure the right version is used. The version can be looked up in `.pre-commit-config.yaml`.
+To update to newer version please make sure to change version numbers in `.pre-commit-config.yaml`, `.travis.yml` and `requirements.txt`.

docs/source/dev_setup.md

@@ -136,13 +136,29 @@ For more information refer to the official [Reference on Cloud Function for Fire
 
 ## Travis Setup
 
-Configuration for travis setup is utilizing environment variables.
+A Travis instance is used to build MapSwipe Workers and run tests.
+There exists a Firebase instance only for Travis.
+For the configuration of Travis following environment variables are used:
 
-One difference to production or development setup is that the service account key as json is stored in the environment variables `FIREBASE_CONFIG` as text. To make this work special characters as to be escaped first. This command will simply escape every character:
+- FIREBASE_API_KEY
+- FIREBASE_DB
+- FIREBASE_TOKEN
+- POSTGRES_DB
+- POSTGRES_HOST
+- POSTGRES_PASSWORD
+- POSTGRES_PASSWORD
+- POSTGRES_USER
+- WALG_GS_PREFIX: empty
+- GOOGLE_APPLICATION_CREDENTIALS: mapswipe_workers/serviceAccountKey.json
 
-```bash
-sed -e 's/./\\&/g; 1{$s/^$/""/}; 1!s/^/"/; $!s/$/"/' serviceAccountKey.json
-```
+Those variables can be definied directly in the repository settings of Travis. For more inofmration refer to: https://docs.travis-ci.com/user/environment-variables/#defining-variables-in-repository-settings
+
+Additionaly a Service Account Key in JSON format is encrypted and added to the GitHub repository using the travis CLI. Once Travis runs it will decrypt the Service Account Key. Read more on that process in the Travis docs: https://docs.travis-ci.com/user/encrypting-files/
+
+Once a Travis build succeeds Travis executes an Ansible Playbook to deploy MapSwipe Workers to an already installed and configured server.
+For this to work an SSH-Key (with access rights to the server) is also encrypted and added to the GitHub repository. Travis will decrypt the key and Ansible will use it to execute commands defined in the Playbook on the server.
+
+All files encrypted for Travis (Service Account Key, SSH-Key) are stored in the `travis` directory.
 
 
 ## Database Backup

docs/source/index.rst

@@ -33,13 +33,16 @@ Welcome to MapSwipe Back-End's documentation!
    configuration
    installation
    cli
-   dev_setup
-   testing_and_travis
    debugging
    backup
 
 
+.. toctree::
+   :maxdepth: 2
+   :caption: Development:
 
+   dev_setup
+   testing_and_travis
 
 
 Indices and tables