Merge pull request #422 from mapswipe/continuous_deployment

Continuous deployment

Author: Matthias-Schaub

.travis.yml

@@ -9,16 +9,25 @@ python:
   - "3.6"
 services:
   - docker
+addons:
+  ssh_known_hosts:
+    - 35.190.148.87  # Development server
+env:
+  - ANSIBLE_CONFIG=ansible/ansible.cfg
 
 before_install:
-  - openssl aes-256-cbc -K $encrypted_444bb228fd3f_key -iv $encrypted_444bb228fd3f_iv -in .ci-mapswipe-firebase-adminsdk-80fzw-21ddcd1649.json.enc -out mapswipe_workers/serviceAccountKey.json -d
-
+  - openssl aes-256-cbc -K $encrypted_444bb228fd3f_key -iv $encrypted_444bb228fd3f_iv -in travis/ci-mapswipe-firebase-adminsdk-80fzw-21ddcd1649.json.enc -out mapswipe_workers/serviceAccountKey.json -d
   # Create a mock file for wal-g setup
   - touch postgres/serviceAccountKey.json
 
 install:
   - pip install --upgrade pip setuptools
   - pip install black==19.10b0 flake8==3.8.3 isort==5.5.2
+  # Install ansible for automated deployment
+  - sudo apt-get --quiet update
+  - sudo apt-get --yes install software-properties-common
+  - sudo apt-add-repository --yes --update ppa:ansible/ansible
+  - sudo apt-get --yes install ansible
 
 script:
   - black --check mapswipe_workers
@@ -29,22 +38,21 @@ script:
   - docker-compose run mapswipe_workers python -m unittest discover --verbose --start-directory tests/unittests/
   - docker-compose run mapswipe_workers python -m unittest discover --verbose --start-directory tests/integration/
 
-# before_deploy:
-#   # SSH setup to deploy to server after build.
-#   - apk add openssh-client
-#   - mkdir -p ~/.ssh
-#   - chmod 700 ~/.ssh
-#   - eval "$(ssh-agent -s)"
-#   - export KEY_FILE=~/.ssh/api_server_ssh_key
-#   - echo "$DEPLOYMENT_SERVER_PRIVATE_KEY" | tr -d '\r' > $KEY_FILE
-#   - chmod 600 $KEY_FILE
-#   - ssh-add $KEY_FILE > /dev/null
-#   - ssh-keyscan -H "$DEPLOYMENT_SERVER_IP" >> ~/.ssh/known_hosts
-#   - chmod 644 ~/.ssh/known_hosts
-#   - ssh -i $KEY_FILE -o StrictHostKeyChecking=no "mapswipe@$DEPLOYMENT_SERVER_IP"
+after_success:
+  # Deploy to development server using Ansible
+  # SSH setup to deploy to server after build.
+  - apk add openssh-client
+  - eval "$(ssh-agent -s)"  #start the ssh agent
+  - mkdir -p $HOME/.ssh
+  # - chmod 700 $HOME/.ssh
+  - cp travis/ssh_config $HOME/.ssh/config
+  - openssl aes-256-cbc -K $encrypted_c17fa81e6490_key -iv $encrypted_c17fa81e6490_iv -in travis/ssh-private-key.enc -out $HOME/.ssh/private-key -d
+  - chmod 600 $HOME/.ssh/private-key
+  - ssh-add $HOME/.ssh/private-key
+  - chmod +x deploy.sh
 
-# deploy:
-#   provider: script
-#   script: bash scripts/deploy.sh
-#   on:
-#     branch: dev
+deploy:
+  provider: script
+  script: bash deploy.sh
+  on:
+    branch: dev

ansible/ansible.cfg

@@ -0,0 +1,5 @@
+[defaults]
+INVENTORY = inventory.yml
+
+[ssh_connection]
+pipelining = True  # Reuse connection. Make execution faster.

ansible/inventory.yml

@@ -0,0 +1,12 @@
+all:
+  hosts:
+    development:
+      ansible_host: 35.190.148.87
+      ansible_user: mapswipe
+      # Uncomment following line when not using ~/.ssh/config
+      # ansible_ssh_private_key_file: $HOME/.ssh/private-key
+    production:
+      ansible_host: 35.231.228.82
+      ansible_user: mapswipe
+      # Uncomment following line when not using ~/.ssh/config
+      # ansible_ssh_private_key_file: $HOME/.ssh/private-key

ansible/playbook.yml

@@ -0,0 +1,12 @@
+---
+- name: Deploy MapSwipe Backend
+  hosts: development
+
+  tasks:
+    - name: Fetch latest code base from GitHub.
+      git:
+        repo: 'https://github.com/mapswipe/python-mapswipe-workers.git'
+        dest: $HOME/python-mapswipe-workers
+        version: dev
+    - name: Rebuild and restart Docker container usind Docker-Compose.
+      script: $HOME/python-mapswipe-worker/recreate_container.sh

deploy.sh

@@ -1,88 +1,4 @@
-git pull
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-python3 test_config.py
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose build --no-cache postgres
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose build --no-cache firebase_deploy
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose build --no-cache mapswipe_workers
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose build --no-cache manager_dashboard
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose build --no-cache nginx
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose build --no-cache api
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker-compose up -d --force-recreate postgres firebase_deploy mapswipe_workers manager_dashboard nginx api
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker logs firebase_deploy
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
-docker ps -a
-if [[ $? = 0 ]]; then
-    echo "success"
-else
-    echo "failure: $?"
-    exit
-fi
-
+#!/bin/bash
+# Deploy to dev server.
+# This script is called by Travis.
+ansible-playbook -i ansible/inventory.yml ansible/playbook.yml

docs/source/dev_setup.md

@@ -141,7 +141,7 @@ There exists a Firebase instance only for Travis.
 For the configuration of Travis following environment variables are used:
 
 - FIREBASE_API_KEY
-- FIREBASE_DB: test-mapswipe
+- FIREBASE_DB
 - FIREBASE_TOKEN
 - POSTGRES_DB
 - POSTGRES_HOST
@@ -153,7 +153,12 @@ For the configuration of Travis following environment variables are used:
 
 Those variables can be definied directly in the repository settings of Travis. For more inofmration refer to: https://docs.travis-ci.com/user/environment-variables/#defining-variables-in-repository-settings
 
-Additionaly a Service Account Key in JSON format is encrypted and added to the GitHub repository using the travis CLI. Read more on that process in the Travis docs: https://docs.travis-ci.com/user/encrypting-files/
+Additionaly a Service Account Key in JSON format is encrypted and added to the GitHub repository using the travis CLI. Once Travis runs it will decrypt the Service Account Key. Read more on that process in the Travis docs: https://docs.travis-ci.com/user/encrypting-files/
+
+Once a Travis build succeeds Travis executes an Ansible Playbook to deploy MapSwipe Workers to an already installed and configured server.
+For this to work an SSH-Key (with access rights to the server) is also encrypted and added to the GitHub repository. Travis will decrypt the key and Ansible will use it to execute commands defined in the Playbook on the server.
+
+All files encrypted for Travis (Service Account Key, SSH-Key) are stored in the `travis` directory.
 
 
 ## Database Backup

docs/source/tags

@@ -0,0 +1,6 @@
+!_TAG_FILE_FORMAT	2	/extended format; --format=1 will not append ;" to lines/
+!_TAG_FILE_SORTED	1	/0=unsorted, 1=sorted, 2=foldcase/
+!_TAG_PROGRAM_AUTHOR	Darren Hiebert	/[email protected]/
+!_TAG_PROGRAM_NAME	Exuberant Ctags	//
+!_TAG_PROGRAM_URL	http://ctags.sourceforge.net	/official site/
+!_TAG_PROGRAM_VERSION	5.8	//

recreate_container.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# This script builds and starts all Docker container for running the Mapswipe ecosystem.
+# It is run either manually or by an Ansible Playbook after a successful Travis build.
+
+docker-compose build --no-cache postgres firebase_deploy mapswipe_workers manager_dashboard nginx api
+if [[ $? = 0 ]]; then
+    echo "success"
+else
+    echo "failure: $?"
+    exit
+fi
+
+docker-compose up -d --force-recreate postgres firebase_deploy mapswipe_workers manager_dashboard nginx api
+if [[ $? = 0 ]]; then
+    echo "success"
+else
+    echo "failure: $?"
+    exit
+fi