Skip to content

Commit fc86060

Browse files
dgageotdgageot
committed
Allow multiple managers
Create an instance group for the managers. First manager to publish it's IP address becomes the leader and then publishes the worker/manager tokens used by the other nodes to join the swarm. Signed-off-by: David Gageot <[email protected]>
1 parent dd0c958 commit fc86060

File tree

8 files changed

+117
-65
lines changed

8 files changed

+117
-65
lines changed

‎gcp/README.md

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,11 @@ make revoke
2828
+ Use Google Log driver
2929
+ Monitoring
3030
+ Configure project
31-
+ Multiple managers
3231
+ SSH keys
3332
+ Additional swarm properties
3433
+ Publish the templates
3534
+ See how the Cloud Shell fits in the big picture
3635
+ DTR/DDC
3736
+ Auto-enable all the Apis we need
38-
+ Share manager's url through the runtime config to be able to create workers sooner
3937
+ Have each worker increment a counter to be able to wait from outside
40-
+ Manager pool
4138
+ Diagnostics

‎gcp/configuration/docker.yaml

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ imports:
44
- path: templates/disk-image.py
55
- path: templates/swarm.py
66
- path: templates/manager.py
7+
- path: templates/managers.py
78
- path: templates/worker.py
89
- path: templates/workers.py
910
- path: templates/network.py
@@ -17,10 +18,3 @@ resources:
1718
type: runtimeconfig.v1beta1.config
1819
properties:
1920
config: swarm-config
20-
21-
- name: swarm-token
22-
type: runtimeconfig.v1beta1.variable
23-
properties:
24-
parent: $(ref.swarm-config.name)
25-
variable: token
26-
value: ""

‎gcp/configuration/templates/manager.py

Lines changed: 68 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -14,59 +14,83 @@ def GenerateConfig(context):
1414
set -x
1515
1616
service docker start
17-
docker swarm init --advertise-addr ens4:2377 --listen-addr ens4:2377
17+
docker swarm init --advertise-addr eth0:2377 --listen-addr eth0:2377
1818
19-
TOKEN=$(docker swarm join-token worker -q)
2019
PROJECT=$(curl -s http://metadata.google.internal/computeMetadata/v1/project/project-id -H "Metadata-Flavor: Google")
2120
ACCESS_TOKEN=$(curl -s http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token -H "Metadata-Flavor: Google" | jq -r ".access_token")
2221
23-
curl -s -X PUT -H "Content-Type: application/json" -d "{\"text\":\"${TOKEN}\"}" https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables/token -H "Authorization":"Bearer ${ACCESS_TOKEN}"
24-
"""
22+
LEADER_IP=$(curl -s http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip -H "Metadata-Flavor: Google")
23+
curl -f -s -X POST -H "Content-Type: application/json" -d "{'name':'projects/${PROJECT}/configs/swarm-config/variables/leader-ip','text':'${LEADER_IP}'}" https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables -H "Authorization":"Bearer ${ACCESS_TOKEN}"
2524
26-
outputs = [{
27-
'name': 'internalIP',
28-
'value': '$(ref.' + context.env['name'] + '.networkInterfaces[0].networkIP)'
29-
}]
25+
if [ $? -eq 0 ]; then
26+
echo "I'm a leader"
27+
28+
WORKER_TOKEN=$(docker swarm join-token worker -q)
29+
curl -s -X POST -H "Content-Type: application/json" -d "{'name':'projects/${PROJECT}/configs/swarm-config/variables/worker-token','text':'${WORKER_TOKEN}'}" https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables -H "Authorization":"Bearer ${ACCESS_TOKEN}"
30+
31+
MANAGER_TOKEN=$(docker swarm join-token manager -q)
32+
curl -s -X POST -H "Content-Type: application/json" -d "{'name':'projects/${PROJECT}/configs/swarm-config/variables/manager-token','text':'${MANAGER_TOKEN}'}" https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables -H "Authorization":"Bearer ${ACCESS_TOKEN}"
33+
else
34+
echo "I'm not a leader"
35+
36+
docker swarm leave --force
37+
38+
for i in $(seq 1 300); do
39+
LEADER_IP=$(curl -sSL "https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables/leader-ip" -H "Authorization":"Bearer ${ACCESS_TOKEN}" | jq -r ".text // empty")
40+
if [ ! -z "${LEADER_IP}" ]; then
41+
TOKEN=$(curl -sSL "https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables/manager-token" -H "Authorization":"Bearer ${ACCESS_TOKEN}" | jq -r ".text // empty")
42+
docker swarm join --token "${TOKEN}" "${LEADER_IP}" --advertise-addr eth0:2377 --listen-addr eth0:2377
43+
break
44+
fi
45+
46+
sleep 1
47+
done
48+
fi
49+
"""
3050

3151
resources = [{
3252
'name': context.env['name'],
33-
'type': 'compute.v1.instance',
53+
'type': 'compute.v1.instanceTemplate',
3454
'properties': {
35-
'zone': zone,
36-
'tags': {
37-
'items': ['swarm', 'swarm-manager']
38-
},
39-
'machineType': '/'.join(['projects', project,
40-
'zones', zone,
41-
'machineTypes', machineType]),
42-
'disks': [{
43-
'deviceName': 'boot',
44-
'type': 'PERSISTENT',
45-
'boot': True,
46-
'autoDelete': True,
47-
'initializeParams': {
48-
'sourceImage': image
49-
}
50-
}],
51-
'networkInterfaces': [{
52-
'network': network,
53-
'accessConfigs': [{
54-
'name': 'External NAT',
55-
'type': 'ONE_TO_ONE_NAT'
55+
'properties': {
56+
'zone': zone,
57+
'machineType': machineType,
58+
'tags': {
59+
'items': ['swarm', 'swarm-manager']
60+
},
61+
'disks': [{
62+
'deviceName': 'boot',
63+
'type': 'PERSISTENT',
64+
'boot': True,
65+
'autoDelete': True,
66+
'initializeParams': {
67+
'sourceImage': image
68+
}
69+
}],
70+
'networkInterfaces': [{
71+
'network': network,
72+
'accessConfigs': [{
73+
'name': 'External NAT',
74+
'type': 'ONE_TO_ONE_NAT'
75+
}]
76+
}],
77+
'metadata': {
78+
'items': [{
79+
'key': 'startup-script',
80+
'value': script
81+
}]
82+
},
83+
'scheduling': {
84+
'preemptible': False,
85+
'onHostMaintenance': 'TERMINATE',
86+
'automaticRestart': False
87+
},
88+
'serviceAccounts': [{
89+
'scopes': [
90+
'https://www.googleapis.com/auth/cloudruntimeconfig'
91+
]
5692
}]
57-
}],
58-
'metadata': {
59-
'items': [{
60-
'key': 'startup-script',
61-
'value': script
62-
}]
63-
},
64-
'serviceAccounts': [{
65-
'scopes': [
66-
'https://www.googleapis.com/auth/cloudruntimeconfig'
67-
]
68-
}]
93+
}
6994
}
7095
}]
71-
72-
return {'resources': resources, 'outputs': outputs}
96+
return {'resources': resources}

‎gcp/configuration/templates/managers.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Copyright 2016 Docker Inc. All rights reserved.
2+
3+
"""Manager Instance Group."""
4+
5+
def GenerateConfig(context):
6+
project = context.env['project']
7+
zone = context.properties['zone']
8+
size = context.properties['size']
9+
template = context.properties['template']
10+
11+
resources = [{
12+
'name': context.env['name'],
13+
'type': 'compute.v1.instanceGroupManager',
14+
'properties': {
15+
'zone': zone,
16+
'instanceTemplate': '/'.join(['projects', project,
17+
'global',
18+
'instanceTemplates', template]),
19+
'baseInstanceName': context.env['name'],
20+
'targetSize': size,
21+
'autoHealingPolicies': [{
22+
'initialDelaySec': 300
23+
}]
24+
}
25+
}]
26+
return {'resources': resources}

‎gcp/configuration/templates/swarm.py

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@
44

55
def GenerateConfig(context):
66
zone = context.properties['zone']
7+
managerCount = context.properties['managerCount']
78
managerMachineType = context.properties['managerMachineType']
9+
workerCount = context.properties['workerCount']
810
workerMachineType = context.properties['workerMachineType']
911
preemptible = context.properties['preemptible']
10-
size = context.properties['size']
1112

1213
resources = [{
1314
'name': 'docker',
@@ -21,6 +22,14 @@ def GenerateConfig(context):
2122
'image': '$(ref.docker.selfLink)',
2223
'network': 'swarm-network'
2324
}
25+
}, {
26+
'name': 'managers',
27+
'type': 'templates/managers.py',
28+
'properties': {
29+
'zone': zone,
30+
'template': '$(ref.manager.name)',
31+
'size': managerCount
32+
}
2433
}, {
2534
'name': 'worker',
2635
'type': 'templates/worker.py',
@@ -30,15 +39,14 @@ def GenerateConfig(context):
3039
'preemptible': preemptible,
3140
'image': '$(ref.docker.selfLink)',
3241
'network': 'swarm-network',
33-
'managerIP': '$(ref.manager.internalIP)'
3442
}
3543
}, {
3644
'name': 'workers',
3745
'type': 'templates/workers.py',
3846
'properties': {
3947
'zone': zone,
4048
'template': '$(ref.worker.name)',
41-
'size': size
49+
'size': workerCount
4250
}
4351
}, {
4452
'name': 'swarm-network',

‎gcp/configuration/templates/swarm.py.schema

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,12 @@ imports:
66
- path: swarm.py
77

88
properties:
9-
size:
9+
managerCount:
10+
type: integer
11+
default: 1
12+
description: Number of Managers
13+
14+
workerCount:
1015
type: integer
1116
default: 2
1217
description: Number of Workers

‎gcp/configuration/templates/worker.py

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,12 @@
22

33
"""Worker Instance Template."""
44

5-
COMPUTE_URL_BASE = 'https://www.googleapis.com/compute/v1'
6-
75
def GenerateConfig(context):
86
project = context.env['project']
97
zone = context.properties['zone']
108
machineType = context.properties['machineType']
119
preemptible = context.properties['preemptible']
1210
image = context.properties['image']
13-
managerIP = context.properties['managerIP']
1411
network = '$(ref.' + context.properties['network'] + '.selfLink)'
1512

1613
script = r"""
@@ -23,9 +20,10 @@ def GenerateConfig(context):
2320
ACCESS_TOKEN=$(curl -s http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token -H "Metadata-Flavor: Google" | jq -r ".access_token")
2421
2522
for i in $(seq 1 300); do
26-
TOKEN=$(curl -sSL "https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables/token" -H "Authorization":"Bearer ${ACCESS_TOKEN}" | jq -r ".text")
27-
if [ "${TOKEN}" != "" ]; then
28-
docker swarm join --token "${TOKEN}" """ + managerIP + r""" --advertise-addr ens4:2377 --listen-addr ens4:2377
23+
LEADER_IP=$(curl -sSL "https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables/leader-ip" -H "Authorization":"Bearer ${ACCESS_TOKEN}" | jq -r ".text // empty")
24+
if [ ! -z "${LEADER_IP}" ]; then
25+
TOKEN=$(curl -sSL "https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables/worker-token" -H "Authorization":"Bearer ${ACCESS_TOKEN}" | jq -r ".text // empty")
26+
docker swarm join --token "${TOKEN}" "${LEADER_IP}" --advertise-addr eth0:2377 --listen-addr eth0:2377
2927
break
3028
fi
3129

‎gcp/configuration/templates/workers.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Copyright 2016 Docker Inc. All rights reserved.
22

3-
"""Worker Instance Group Manager."""
3+
"""Worker Instance Group."""
44

55
def GenerateConfig(context):
66
project = context.env['project']

0 commit comments

Comments
 (0)