Skip to content

Commit 1911294

Browse files
Hagellach37Hagellach37
authored
Merge pull request #465 from mapswipe/dev
WIP: Release 2.3.0
2 parents 2198bcd + d385eb8 commit 1911294

35 files changed

+987
-474
lines changed

‎.github/workflows/actions.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: MapSwipe Workers
2+
3+
on: [push]
4+
5+
jobs:
6+
build:
7+
8+
runs-on: ubuntu-latest
9+
strategy:
10+
matrix:
11+
python-version: [3.8]
12+
13+
steps:
14+
- uses: actions/checkout@v2
15+
- name: Set up Python ${{ matrix.python-version }}
16+
uses: actions/setup-python@v2
17+
with:
18+
python-version: ${{ matrix.python-version }}
19+
- name: Install dependencies
20+
working-directory: ./mapswipe_workers
21+
run: |
22+
python -m pip install --upgrade pip
23+
pip install flake8 black==19.10b0 isort
24+
- name: Code style
25+
working-directory: ./mapswipe_workers
26+
run: |
27+
black --check mapswipe_workers
28+
flake8 --count --config setup.cfg mapswipe_workers/
29+
isort --check --settings-file setup.cfg mapswipe_workers/
30+
- name: Setup Postgres Database Container
31+
env:
32+
POSTGRES_PASSWORD: test
33+
POSTGRES_USER: test
34+
POSTGRES_DB: test
35+
run: |
36+
# Create a mock file for wal-g setup
37+
touch postgres/serviceAccountKey.json
38+
docker-compose up --build --detach postgres
39+
- name: Deploy Firebase Rules and Functions
40+
env:
41+
FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
42+
FIREBASE_DB: ${{ secrets.FIREBASE_DB }}
43+
run: |
44+
# Create a mock file for wal-g setup
45+
docker-compose up --build firebase_deploy
46+
- name: Decrypt Service Account Key File
47+
working-directory: ./
48+
run: |
49+
openssl enc -aes-256-cbc -d -K "$OPENSSL_KEY" -iv "$OPENSSL_IV" -in ci-mapswipe-firebase-adminsdk-80fzw-ebce84bd5b.json.enc -out mapswipe_workers/serviceAccountKey.json
50+
env:
51+
OPENSSL_PASSPHRASE: ${{ secrets.OPENSSL_PASSPHRASE }}
52+
OPENSSL_KEY: ${{ secrets.OPENSSL_KEY }}
53+
OPENSSL_IV: ${{ secrets.OPENSSL_IV }}
54+
- name: Run Tests
55+
working-directory: ./mapswipe_workers
56+
env:
57+
FIREBASE_API_KEY: ${{ secrets.FIREBASE_API_KEY }}
58+
FIREBASE_DB: ${{ secrets.FIREBASE_DB }}
59+
POSTGRES_PASSWORD: test
60+
POSTGRES_USER: test
61+
POSTGRES_DB: test
62+
run: |
63+
docker-compose run mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/
64+
docker-compose run mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/integration/

‎.travis.yml‎

Lines changed: 0 additions & 58 deletions
This file was deleted.

‎ansible/playbook.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@
99
dest: $HOME/python-mapswipe-workers
1010
version: dev
1111
- name: Rebuild Docker container.
12-
command: docker-compose build postgres firebase_deploy mapswipe_workers manager_dashboard nginx api
12+
command: docker-compose build postgres firebase_deploy mapswipe_workers_creation mapswipe_workers_stats manager_dashboard nginx api
1313
args:
1414
chdir: $HOME/python-mapswipe-workers
1515
- name: Restart Docker container.
16-
command: docker-compose up -d --force-recreate postgres firebase_deploy mapswipe_workers manager_dashboard nginx api
16+
command: docker-compose up -d --force-recreate postgres firebase_deploy mapswipe_workers_creation mapswipe_workers_stats manager_dashboard nginx api
1717
args:
1818
chdir: $HOME/python-mapswipe-workers

‎ci-mapswipe-firebase-adminsdk-80fzw-ebce84bd5b.json.enc‎

2.28 KB
Binary file not shown.

‎docker-compose.yaml‎

Lines changed: 71 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,76 @@ services:
4949
networks:
5050
- manager_dashboard
5151

52-
mapswipe_workers:
53-
container_name: mapswipe_workers
52+
mapswipe_workers_creation:
53+
container_name: mapswipe_workers_creation
54+
build:
55+
context: mapswipe_workers/
56+
environment:
57+
FIREBASE_DB: '${FIREBASE_DB}'
58+
FIREBASE_API_KEY: '${FIREBASE_API_KEY}'
59+
FIREBASE_TOKEN: '${FIREBASE_TOKEN}'
60+
GOOGLE_APPLICATION_CREDENTIALS: 'serviceAccountKey.json'
61+
POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
62+
POSTGRES_USER: '${POSTGRES_USER}'
63+
POSTGRES_DB: '${POSTGRES_DB}'
64+
POSTGRES_HOST: 'postgres'
65+
POSTGRES_PORT: 5432
66+
PGDATA: '/var/lib/postgresql/mapswipe'
67+
IMAGE_BING_API_KEY: '${IMAGE_BING_API_KEY}'
68+
IMAGE_ESRI_API_KEY: '${IMAGE_ESRI_API_KEY}'
69+
IMAGE_ESRI_BETA_API_KEY: '${IMAGE_ESRI_BETA_API_KEY}'
70+
IMAGE_MAPBOX_API_KEY: '${IMAGE_MAPBOX_API_KEY}'
71+
IMAGE_MAXAR_PREMIUM_API_KEY: '${IMAGE_MAXAR_PREMIUM_API_KEY}'
72+
IMAGE_MAXAR_STANDARD_API_KEY: '${IMAGE_MAXAR_STANDARD_API_KEY}'
73+
SLACK_TOKEN: '${SLACK_TOKEN}'
74+
SLACK_CHANNEL: '${SLACK_CHANNEL}'
75+
SENTRY_DSN: '${SENTRY_DSN}'
76+
depends_on:
77+
- postgres
78+
command: mapswipe_workers --verbose run --analysis_type=creation --schedule --time_interval=5
79+
volumes:
80+
- ./mapswipe-data:/root/.local/share/mapswipe_workers
81+
restart: "no"
82+
networks:
83+
- mapswipe_workers
84+
- postgres
85+
86+
mapswipe_workers_stats:
87+
container_name: mapswipe_workers_stats
88+
build:
89+
context: mapswipe_workers/
90+
environment:
91+
FIREBASE_DB: '${FIREBASE_DB}'
92+
FIREBASE_API_KEY: '${FIREBASE_API_KEY}'
93+
FIREBASE_TOKEN: '${FIREBASE_TOKEN}'
94+
GOOGLE_APPLICATION_CREDENTIALS: 'serviceAccountKey.json'
95+
POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
96+
POSTGRES_USER: '${POSTGRES_USER}'
97+
POSTGRES_DB: '${POSTGRES_DB}'
98+
POSTGRES_HOST: 'postgres'
99+
POSTGRES_PORT: 5432
100+
PGDATA: '/var/lib/postgresql/mapswipe'
101+
IMAGE_BING_API_KEY: '${IMAGE_BING_API_KEY}'
102+
IMAGE_ESRI_API_KEY: '${IMAGE_ESRI_API_KEY}'
103+
IMAGE_ESRI_BETA_API_KEY: '${IMAGE_ESRI_BETA_API_KEY}'
104+
IMAGE_MAPBOX_API_KEY: '${IMAGE_MAPBOX_API_KEY}'
105+
IMAGE_MAXAR_PREMIUM_API_KEY: '${IMAGE_MAXAR_PREMIUM_API_KEY}'
106+
IMAGE_MAXAR_STANDARD_API_KEY: '${IMAGE_MAXAR_STANDARD_API_KEY}'
107+
SLACK_TOKEN: '${SLACK_TOKEN}'
108+
SLACK_CHANNEL: '${SLACK_CHANNEL}'
109+
SENTRY_DSN: '${SENTRY_DSN}'
110+
depends_on:
111+
- postgres
112+
command: mapswipe_workers --verbose run --analysis_type=generate-stats --schedule --time_interval=60
113+
volumes:
114+
- ./mapswipe-data:/root/.local/share/mapswipe_workers
115+
restart: "no"
116+
networks:
117+
- mapswipe_workers
118+
- postgres
119+
120+
mapswipe_workers_firebase_to_postgres:
121+
container_name: mapswipe_workers_firebase_to_postgres
54122
build:
55123
context: mapswipe_workers/
56124
environment:
@@ -75,7 +143,7 @@ services:
75143
SENTRY_DSN: '${SENTRY_DSN}'
76144
depends_on:
77145
- postgres
78-
command: mapswipe_workers --verbose run --schedule
146+
command: mapswipe_workers --verbose run --analysis_type=firebase-to-postgres --schedule --time_interval=2
79147
volumes:
80148
- ./mapswipe-data:/root/.local/share/mapswipe_workers
81149
restart: "no"

‎docs/source/deployment_overview.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,9 @@ This is done by using Travis script deployment (https://docs.travis-ci.com/user/
2525
To be able to connect to the MapSwipe server the Travis instance uses an encrypted SSH private key (Which can be found in the directory `travis/`).
2626

2727
In the `deploy.sh` script an Ansible Playbook is run (https://docs.ansible.com/ansible/latest/index.html). Ansible is an automation tool which utilizes a SSH connection (`ansible/ansible.cfg`) to run commands defined in the Playbook (`ansible/playbook.yml`) on hosts defined in the Inventory (`ansible/inventory.yml`).
28+
29+
## Continuous Deployment with Github Actions
30+
We use an encrypted service account key file. The file has been generated with this commands:
31+
32+
`openssl enc -aes-256-cbc -e -p -nosalt -in=ci-mapswipe-firebase-adminsdk-80fzw-ebce84bd5b.json -out=ci-mapswipe-firebase-adminsdk-80fzw-ebce84bd5b.json.enc`
33+

0 commit comments

Comments
 (0)