Skip to content

Commit 7836995

Browse files
sukrit007sukrit007
committed
Merge pull request #56 from totem/develop
0.5.0 Release
2 parents 7996349 + eb8cc37 commit 7836995

File tree

11 files changed

+226
-91
lines changed

11 files changed

+226
-91
lines changed

‎.dockerignore‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
.idea
2+
docs
3+
postman
4+
tests
5+
.git*
6+
.travis.yml
7+
apiary.apib
8+
*.iml
9+
LICENSE
10+
local-celery.py
11+
local-server.py
12+
README.md
13+
totem.yml

‎.travis.yml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,12 @@ services:
99

1010
# command to install dependencies
1111
install:
12+
- "pip install --upgrade pip"
1213
- "pip install -r requirements.txt"
1314
- "pip install -r dev-requirements.txt"
14-
- "curl -L https://github.com/coreos/etcd/releases/download/v0.4.6/etcd-v0.4.6-linux-amd64.tar.gz -o etcd-v0.4.6-linux-amd64.tar.gz"
15-
- "tar xzvf etcd-v0.4.6-linux-amd64.tar.gz"
16-
- "./etcd-v0.4.6-linux-amd64/etcd &"
15+
- "curl -L https://github.com/coreos/etcd/releases/download/v2.2.5/etcd-v2.2.5-linux-amd64.tar.gz -o etcd-linux-amd64.tar.gz"
16+
- "tar xzvf etcd-linux-amd64.tar.gz"
17+
- "./etcd-v2.2.5-linux-amd64/etcd &"
1718

1819
# commands to run tests and style check
1920
script:

‎Dockerfile‎

Lines changed: 48 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,64 @@
1-
FROM totem/python-base:2.7-trusty-b3
1+
FROM python:2.7.11-alpine
22

33
ENV DEBIAN_FRONTEND noninteractive
4+
ENV ETCDCTL_VERSION v2.2.5
5+
ENV DUMB_INIT_VERSION 1.0.1
46

5-
RUN apt-get update --fix-missing && apt-get install -y \
7+
RUN apk add --no-cache --update \
8+
pcre \
69
gettext \
7-
libyaml-dev \
8-
&& apt-get clean \
9-
&& rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/*
10+
curl \
1011

11-
# Etcdctl
12-
ENV ETCDCTL_VERSION v2.2.5
13-
RUN curl -L https://github.com/coreos/etcd/releases/download/$ETCDCTL_VERSION/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz -o /tmp/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz && \
14-
cd /tmp && gzip -dc etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz | tar -xof - && \
15-
cp -f /tmp/etcd-$ETCDCTL_VERSION-linux-amd64/etcdctl /usr/local/bin && \
16-
rm -rf /tmp/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz && \
17-
rm -rf /tmp/etcd-$ETCDCTL_VERSION-linux-amd64
18-
19-
# Supervisor and App dependencies
20-
RUN pip install supervisor==3.1.2 supervisor-stdout
21-
ADD requirements.txt /opt/requirements.txt
22-
RUN pip install -r /opt/requirements.txt
23-
24-
# Supervisor Config
25-
RUN mkdir -p /var/log/supervisor
12+
# Etcdctl
13+
&& curl -L https://github.com/coreos/etcd/releases/download/$ETCDCTL_VERSION/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz -o /tmp/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz \
14+
&& cd /tmp && gzip -dc etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz | tar -xof - \
15+
&& cp -f /tmp/etcd-$ETCDCTL_VERSION-linux-amd64/etcdctl /usr/local/bin \
16+
17+
# Dumb Init
18+
&& wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMB_INIT_VERSION}/dumb-init_${DUMB_INIT_VERSION}_amd64 \
19+
&& chmod +x /usr/bin/dumb-init \
20+
21+
# Cleanup
22+
&& rm -rf /tmp/*
23+
24+
# Application dependencies
25+
ADD requirements.txt /opt/cluster-orchestrator/requirements.txt
26+
RUN apk add --no-cache --update --virtual build-dependencies \
27+
musl-dev \
28+
linux-headers \
29+
build-base \
30+
pcre-dev \
31+
libffi-dev \
32+
openssl-dev \
33+
34+
# Python depdencies
35+
&& pip install --ignore-installed --no-cache-dir \
36+
supervisor==3.2.3 \
37+
supervisor-stdout \
38+
-r /opt/cluster-orchestrator/requirements.txt \
39+
40+
# Supervisor (Post Setup)
41+
&& mkdir -p /var/log/supervisor \
42+
&& ln -sf /etc/supervisor/supervisord.conf /etc/supervisord.conf \
43+
44+
# Cleanup
45+
&& apk del build-dependencies \
46+
&& find /usr/local \
47+
\( -type d -a -name test -o -name tests \) -exec echo rm -rf '{}' + \
48+
-o \( -type f -a -name '*.pyc' -o -name '*.pyo' \) -exec echo rm -f '{}' +
49+
50+
51+
# Supervisor Scripts
2652
ADD bin/supervisord-wrapper.sh /usr/sbin/supervisord-wrapper.sh
27-
RUN chmod +x /usr/sbin/supervisord-wrapper.sh && \
28-
ln -sf /etc/supervisor/supervisord.conf /etc/supervisord.conf
53+
RUN chmod +x /usr/sbin/supervisord-wrapper.sh
2954

3055
# Etc Config
3156
ADD etc /etc
3257

3358
ADD . /opt/cluster-orchestrator
34-
RUN pip install -r /opt/cluster-orchestrator/requirements.txt
3559

3660
EXPOSE 9400
3761

3862
WORKDIR /opt/cluster-orchestrator
3963

40-
ENTRYPOINT ["/usr/sbin/supervisord-wrapper.sh"]
64+
CMD ["/usr/bin/dumb-init", "/usr/sbin/supervisord-wrapper.sh"]

‎bin/supervisord-wrapper.sh‎

Lines changed: 68 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,45 +1,73 @@
1-
#!/bin/bash -le
1+
#!/bin/sh -e
22

33
HOST_IP="${HOST_IP:-$(/sbin/ip route|awk '/default/ { print $3 }')}"
44

5-
cat <<END>> /etc/profile.d/cluster-deployer-env.sh
6-
7-
export ETCD_HOST='${ETCD_HOST:-$HOST_IP}'
8-
export ETCD_PORT='${ETCD_PORT:-4001}'
9-
export ETCD_TOTEM_BASE='${ETCD_TOTEM_BASE:-/totem}'
10-
export ETCD_YODA_BASE='${ETCD_YODA_BASE:-/yoda}'
11-
export CELERY_GEVENT_EXECUTORS='${CELERY_GEVENT_EXECUTORS:-1}'
12-
export CELERY_GEVENT_CONCURRENCY='${CELERY_GEVENT_CONCURRENCY:-50}'
13-
export API_EXECUTORS='${API_EXECUTORS:-2}'
14-
export GITHUB_TOKEN='${GITHUB_TOKEN}'
15-
export CLUSTER_NAME='${CLUSTER_NAME:-local}'
16-
export TOTEM_ENV='${TOTEM_ENV:-local}'
17-
export QUAY_ORGANIZATION='${QUAY_ORGANIZATION:-totem}'
18-
export QUAY_PREFIX='${QUAY_PREFIX:-totem-}'
5+
export ETCD_HOST="${ETCD_HOST:-$HOST_IP}"
6+
export ETCD_PORT="${ETCD_PORT:-4001}"
7+
export ETCD_URL="${ETCD_URL:-http://$ETCD_HOST:$ETCD_PORT}"
8+
export ETCDCTL="${ETCDCTL:-etcdctl --peers $ETCD_URL}"
9+
export ETCD_TOTEM_BASE="${ETCD_TOTEM_BASE:-/totem}"
10+
export ETCD_YODA_BASE="${ETCD_YODA_BASE:-/yoda}"
11+
export CELERY_GEVENT_EXECUTORS="${CELERY_GEVENT_EXECUTORS:-1}"
12+
export CELERY_GEVENT_CONCURRENCY="${CELERY_GEVENT_CONCURRENCY:-50}"
13+
export API_EXECUTORS="${API_EXECUTORS:-2}"
14+
export GITHUB_TOKEN="${GITHUB_TOKEN}"
15+
export CLUSTER_NAME="${CLUSTER_NAME:-local}"
16+
export TOTEM_ENV="${TOTEM_ENV:-local}"
17+
export QUAY_ORGANIZATION="${QUAY_ORGANIZATION:-totem}"
18+
export QUAY_PREFIX="${QUAY_PREFIX:-totem-}"
1919
export C_FORCE_ROOT=true
20-
export AMQP_HOST='${AMQP_HOST:-$HOST_IP}'
21-
export AMQP_PORT='${AMQP_PORT:-5672}'
22-
export AMQP_USERNAME='${AMQP_USERNAME:-guest}'
23-
export AMQP_PASSWORD='${AMQP_PASSWORD:-guest}'
24-
export MONGODB_USERNAME='${MONGODB_USERNAME:-}'
25-
export MONGODB_PASSWORD='${MONGODB_PASSWORD:-}'
26-
export MONGODB_HOST='${MONGODB_HOST:-$HOST_IP}'
27-
export MONGODB_PORT='${MONGODB_PORT:-27017}'
28-
export MONGODB_DB='${MONGODB_DB}'
29-
export MONGODB_AUTH_DB='${MONGODB_AUTH_DB}'
30-
export BROKER_URL='${BROKER_URL}'
31-
export CLUSTER_DEPLOYER_URL='${CLUSTER_DEPLOYER_URL:-http://$HOST_IP:9000}'
32-
export ENCRYPTION_PASSPHRASE='${ENCRYPTION_PASSPHRASE:-changeit}'
33-
export ENCRYPTION_S3_BUCKET='${ENCRYPTION_S3_BUCKET:-not-set}'
34-
export ENCRYPTION_STORE='${ENCRYPTION_PROVIDER:-s3}'
35-
export HIPCHAT_ENABLED='${HIPCHAT_ENABLED:-false}'
36-
export HIPCHAT_TOKEN='${HIPCHAT_TOKEN}'
37-
export HIPCHAT_ROOM='${HIPCHAT_ROOM:-not-set}'
38-
export GITHUB_NOTIFICATION_ENABLED='${GITHUB_NOTIFICATION_ENABLED:-false}'
39-
export LOG_IDENTIFIER='${LOG_IDENTIFIER:-cluster-orchestrator}'
40-
export LOG_ROOT_LEVEL='${LOG_ROOT_LEVEL}'
41-
END
42-
43-
/bin/bash -le -c " envsubst < /etc/supervisor/conf.d/supervisord.conf.template > /etc/supervisor/conf.d/supervisord.conf; \
44-
/usr/local/bin/supervisord -c /etc/supervisor/supervisord.conf"
20+
export AMQP_HOST="${AMQP_HOST:-$HOST_IP}"
21+
export AMQP_PORT="${AMQP_PORT:-5672}"
22+
export AMQP_USERNAME="${AMQP_USERNAME:-guest}"
23+
export AMQP_PASSWORD="${AMQP_PASSWORD:-guest}"
24+
export MONGODB_USERNAME="${MONGODB_USERNAME:-}"
25+
export MONGODB_PASSWORD="${MONGODB_PASSWORD:-}"
26+
export MONGODB_HOST="${MONGODB_HOST:-$HOST_IP}"
27+
export MONGODB_PORT="${MONGODB_PORT:-27017}"
28+
export MONGODB_DB="${MONGODB_DB}"
29+
export MONGODB_AUTH_DB="${MONGODB_AUTH_DB}"
30+
export BROKER_URL="${BROKER_URL}"
31+
export CLUSTER_DEPLOYER_URL="${CLUSTER_DEPLOYER_URL:-http://$HOST_IP:9000}"
32+
export ENCRYPTION_PASSPHRASE="${ENCRYPTION_PASSPHRASE:-changeit}"
33+
export ENCRYPTION_S3_BUCKET="${ENCRYPTION_S3_BUCKET:-not-set}"
34+
export ENCRYPTION_STORE="${ENCRYPTION_PROVIDER:-s3}"
35+
export HIPCHAT_ENABLED="${HIPCHAT_ENABLED:-false}"
36+
export HIPCHAT_TOKEN="${HIPCHAT_TOKEN}"
37+
export HIPCHAT_ROOM="${HIPCHAT_ROOM:-not-set}"
38+
export GITHUB_NOTIFICATION_ENABLED="${GITHUB_NOTIFICATION_ENABLED:-false}"
39+
export LOG_IDENTIFIER="${LOG_IDENTIFIER:-cluster-orchestrator}"
40+
export LOG_ROOT_LEVEL="${LOG_ROOT_LEVEL}"
41+
42+
until $ETCDCTL cluster-health; do
43+
>&2 echo "Etcdctl cluster not healthy - sleeping"
44+
sleep 10
45+
done
46+
47+
# Cleanup for local testing
48+
if ls celerybeat* 2>/dev/null; then
49+
rm celerybeat*
50+
fi
51+
52+
if [ "$DISCOVER_RABBITMQ" == "true" ]; then
53+
export AMQP_HOST="$($ETCDCTL ls $ETCD_TOTEM_BASE/rabbitmq/nodes | xargs -n 1 $ETCDCTL get | xargs echo -n | tr ' ' ',')"
54+
until [ ! -z "$AMQP_HOST" ]; do
55+
>&2 echo "Rabbitmq could not be discovered - sleeping"
56+
sleep 10
57+
export AMQP_HOST="$($ETCDCTL ls $ETCD_TOTEM_BASE/rabbitmq/nodes | xargs -n 1 $ETCDCTL get | xargs echo -n | tr ' ' ',')"
58+
done
59+
fi
60+
61+
if [ "$DISCOVER_MONGO" == "true" ]; then
62+
export MONGODB_SERVERS="$($ETCDCTL ls $ETCD_TOTEM_BASE/mongo/nodes | xargs -n 1 $ETCDCTL get | xargs echo -n | tr ' ' ',')"
63+
until [ ! -z "$MONGODB_SERVERS" ]; do
64+
>&2 echo "Mongo servers could not be discovered - sleeping"
65+
sleep 10
66+
export MONGODB_SERVERS="$($ETCDCTL ls $ETCD_TOTEM_BASE/mongo/nodes | xargs -n 1 $ETCDCTL get | xargs echo -n | tr ' ' ',')"
67+
done
68+
fi
69+
70+
envsubst < /etc/supervisor/conf.d/supervisord.conf.template > /etc/supervisor/conf.d/supervisord.conf
71+
72+
/usr/local/bin/supervisord -c /etc/supervisor/supervisord.conf
4573

‎conf/appconfig.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -226,12 +226,16 @@
226226
MONGODB_PASSWORD = os.getenv('MONGODB_PASSWORD', '')
227227
MONGODB_HOST = os.getenv('MONGODB_HOST', '127.0.0.1')
228228
MONGODB_PORT = int(os.getenv('MONGODB_PORT', '27017'))
229+
230+
MONGODB_SERVERS = os.getenv('MONGODB_SERVERS', '{}:{}'.format(
231+
MONGODB_HOST, MONGODB_PORT))
229232
MONGODB_DB = os.getenv('MONGODB_DB') or 'totem-{}'.format(TOTEM_ENV)
230-
MONGODB_AUTH_DB = os.getenv('MONGODB_AUTH_DB') or MONGODB_DB
233+
MONGODB_AUTH_DB = os.getenv('MONGODB_AUTH_DB') or 'admin'
231234
MONGODB_AUTH = '{0}:{1}@'.format(MONGODB_USERNAME, MONGODB_PASSWORD) \
232235
if MONGODB_USERNAME else ''
233-
MONGODB_DEFAULT_URL = 'mongodb://{0}{1}:{2}/{3}'.format(
234-
MONGODB_AUTH, MONGODB_HOST, MONGODB_PORT, MONGODB_AUTH_DB)
236+
237+
MONGODB_DEFAULT_URL = 'mongodb://{0}{1}/{2}'.format(
238+
MONGODB_AUTH, MONGODB_SERVERS, MONGODB_AUTH_DB)
235239
MONGODB_URL = os.getenv('MONGODB_URL') or MONGODB_DEFAULT_URL
236240

237241
MONGODB_JOB_COLLECTION = os.getenv('MONGODB_JOB_COLLECTION') or \

‎conf/celeryconfig.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,11 @@
1616
AMQP_PASSWORD = os.getenv('AMQP_PASSWORD', 'guest')
1717
AMQP_HOST = os.getenv('AMQP_HOST', 'localhost')
1818
AMQP_PORT = int(os.getenv('AMQP_PORT', '5672'))
19-
DEFAULT_BROKER_URL = 'amqp://%s:%s@%s:%s' % (AMQP_USERNAME, AMQP_PASSWORD,
20-
AMQP_HOST, AMQP_PORT)
19+
DEFAULT_BROKER_URL = [
20+
'amqp://{}:{}@{}:{}'.format(
21+
AMQP_USERNAME, AMQP_PASSWORD, host.strip(), AMQP_PORT
22+
) for host in AMQP_HOST.split(',') if host.strip()
23+
]
2124
BROKER_URL = os.getenv('BROKER_URL') or DEFAULT_BROKER_URL
2225
BROKER_HEARTBEAT = int(os.getenv('BROKER_HEARTBEAT', '20'))
2326
BROKER_TRANSPORT_OPTIONS = {'confirm_publish': True}

‎dev-requirements.txt‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
nose==1.3.4
2-
mock==1.0.1
3-
flake8
1+
nose==1.3.7
2+
mock==2.0.0
3+
flake8==2.5.4
44
Sphinx==1.2.3
55

66
# For freezing time
@@ -9,4 +9,5 @@ https://github.com/spulec/freezegun/archive/master.tar.gz
99
# Code Coverage
1010
coveralls
1111

12-
flower==0.7.2
12+
flower==0.9.0
13+

‎docker-compose.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
version: '2'
2+
3+
networks:
4+
local:
5+
driver: 'bridge'
6+
7+
services:
8+
orchestrator:
9+
build: .
10+
networks:
11+
local:
12+
aliases:
13+
- orchestrator.local
14+
environment:
15+
ETCD_HOST: 'etcd.local'
16+
AMQP_HOST: 'rabbitmq.local'
17+
MONGODB_SERVERS: 'mongo.local:27017'
18+
DISCOVER_MONGO: 'false'
19+
DISCOVER_RABBITMQ: 'false'
20+
21+
ports:
22+
- 9400
23+
24+
depends_on:
25+
- etcd
26+
- rabbitmq
27+
- mongo
28+
restart: always
29+
volumes:
30+
- ".:/opt/cluster-orchestrator"
31+
32+
rabbitmq:
33+
image: 'rabbitmq:3.6'
34+
networks:
35+
local:
36+
aliases:
37+
- rabbitmq.local
38+
39+
mongo:
40+
image: 'mongo:2.6'
41+
networks:
42+
local:
43+
aliases:
44+
- mongo.local
45+
46+
etcd:
47+
image: 'quay.io/coreos/etcd'
48+
command: [
49+
'-name', 'etcd0',
50+
'-advertise-client-urls','http://etcd.local:2379,http://etcd.local:4001',
51+
'-listen-client-urls', 'http://0.0.0.0:2379,http://0.0.0.0:4001',
52+
'-initial-advertise-peer-urls', 'http://etcd.local:2380',
53+
'-listen-peer-urls', 'http://0.0.0.0:2380',
54+
'-initial-cluster-token', 'etcd-cluster-1',
55+
'-initial-cluster', 'etcd0=http://etcd.local:2380',
56+
'-initial-cluster-state', 'new',
57+
]
58+
59+
networks:
60+
local:
61+
aliases:
62+
- etcd.local
63+
volumes:
64+
- /usr/share/ca-certificates/:/etc/ssl/certs

‎orchestrator/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
from celery.signals import setup_logging
33
import orchestrator.logger
44

5-
__version__ = '0.4.3'
5+
__version__ = '0.5.0'
66
__author__ = 'sukrit'
77

88
orchestrator.logger.init_logging()

‎requirements.txt‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
11
Flask==0.10.1
2-
Flask-Cors==1.9.0
3-
gevent==1.0.2
4-
uWSGI==2.0.12
2+
Flask-Cors==2.1.2
3+
gevent==1.1.1
4+
# Uncomment below , when 2.0.13 is released See https://github.com/gliderlabs/docker-alpine/issues/158#issuecomment-205401343
5+
# uWSGI==2.0.13
6+
https://github.com/unbit/uwsgi/archive/uwsgi-2.0.zip#egg=uwsgi
57
python-etcd==0.3.3
6-
future==0.15.0
8+
future==0.15.2
79
PyYAML==3.11
810
boto==2.34.0
9-
pymongo==3.0.3
11+
pymongo==3.2.2
1012
Jinja2==2.7.3
11-
requests[security]==2.7.0
12-
urllib3==1.11
13-
celery[mongodb]==3.1.20
14-
https://github.com/totem/flask-hyperschema/archive/v0.1.1.tar.gz
13+
requests[security]==2.9.1
14+
urllib3==1.15
15+
celery[mongodb]==3.1.23
16+
https://github.com/totem/flask-hyperschema/archive/0.2.0.tar.gz
1517
https://github.com/totem/totem-encrypt/archive/master.tar.gz
1618
https://github.com/dlitz/pycrypto/archive/v2.7a1.tar.gz

0 commit comments

Comments
 (0)