Merge pull request docker-archive#446 from dgageot/improved-swarm

FrenchBen · web-flow · commit 3241eabc3d2d · 2016-11-22T13:36:01.000-08:00
Improved Swarm Templates for GCP
diff --git a/gcp/Makefile b/gcp/Makefile
@@ -1,16 +1,23 @@
 # Copyright 2016 Docker Inc. All rights reserved.
 
-auth:
-	docker run -it --name gcloud-config google/cloud-sdk gcloud init
+GCLOUD_IMAGE_TAG = sha256:3a4e572df0c716196b3ffdeacaba3ebdd36f8cdfdac6d5ee4bd50cd405366736
+GCLOUD = docker run --rm -ti -v $(CURDIR)/configuration:/configuration --volumes-from gcloud-config google/cloud-sdk gcloud
+DEPLOYMENTS = $(GCLOUD) deployment-manager deployments
+
+all: auth create
+
+auth: revoke
+	docker run -it --name gcloud-config google/cloud-sdk@$(GCLOUD_IMAGE_TAG) true
+	$(GCLOUD) init
 
 revoke:
-	docker rm -f gcloud-config
+	docker rm gcloud-config || true
 
 create:
-	docker run --rm -ti -v $(CURDIR)/configuration:/configuration --volumes-from gcloud-config google/cloud-sdk gcloud deployment-manager deployments create docker --config configuration/docker.yaml
+	$(DEPLOYMENTS) create docker --config configuration/docker.yaml
 
 describe:
-	docker run --rm -ti -v $(CURDIR)/configuration:/configuration --volumes-from gcloud-config google/cloud-sdk gcloud deployment-manager deployments describe docker
+	$(DEPLOYMENTS) describe docker
 
 delete:
-	docker run --rm -ti -v $(CURDIR)/configuration:/configuration --volumes-from gcloud-config google/cloud-sdk gcloud deployment-manager deployments delete docker -q
+	$(DEPLOYMENTS) delete docker -q
diff --git a/gcp/README.md b/gcp/README.md
@@ -25,15 +25,12 @@ make revoke
  + External load balancer
  + Better UX
  + Use Moby
- + Logs
+ + Use Google Log driver
  + Monitoring
- + Configure project
- + Multiple managers
- + Different machine types for workers and managers
  + SSH keys
  + Additional swarm properties
  + Publish the templates
  + See how the Cloud Shell fits in the big picture
  + DTR/DDC
- + List the Apis we need
- + Auto-enable all the Apis we need
+ + Have each worker increment a counter to be able to wait from outside
+ + Diagnostics
diff --git a/gcp/configuration/docker.yaml b/gcp/configuration/docker.yaml
@@ -1,8 +1,10 @@
 # Copyright 2016 Docker Inc. All rights reserved.
 
 imports:
+- path: templates/disk-image.py
 - path: templates/swarm.py
 - path: templates/manager.py
+- path: templates/managers.py
 - path: templates/worker.py
 - path: templates/workers.py
 - path: templates/network.py
@@ -16,10 +18,3 @@ resources:
   type: runtimeconfig.v1beta1.config
   properties:
     config: swarm-config
-
-- name: swarm-token
-  type: runtimeconfig.v1beta1.variable
-  properties:
-    parent: $(ref.swarm-config.name)
-    variable: token
-    value: ""
diff --git a/gcp/configuration/templates/disk-image.py b/gcp/configuration/templates/disk-image.py
@@ -0,0 +1,17 @@
+# Copyright 2016 Docker Inc. All rights reserved.
+
+"""Moby disk image used for all the instances."""
+
+def GenerateConfig(context):
+  resources = [{
+      'name': context.env['name'],
+      'type': 'compute.v1.image',
+      'properties': {
+          'family': 'docker',
+          'rawDisk': {
+              'source': 'https://storage.cloud.google.com/docker-image/docker.image.tar.gz'
+          }
+      }
+  }]
+
+  return {'resources': resources}
diff --git a/gcp/configuration/templates/firewall.py b/gcp/configuration/templates/firewall.py
@@ -6,38 +6,39 @@ def GenerateConfig(context):
   network = '$(ref.' + context.properties['network'] + '.selfLink)'
 
   resources = [{
-      'name': 'ssh',
+      'name': 'allow-ssh',
       'type': 'compute.v1.firewall',
       'properties': {
           'network': network,
           'sourceRanges': ['0.0.0.0/0'],
           'allowed': [{
-              'IPProtocol': 'TCP',
-              'ports': [22]
+              'IPProtocol': 'tcp',
+              'ports': ['22']
           }]
       }
   },{
-      'name': 'http',
+      'name': 'allow-http',
       'type': 'compute.v1.firewall',
       'properties': {
           'network': network,
           'sourceRanges': ['0.0.0.0/0'],
           'allowed': [{
-              'IPProtocol': 'TCP',
-              'ports': [80]
-          },{
-              'IPProtocol': 'TCP',
-              'ports': [443]
+              'IPProtocol': 'tcp',
+              'ports': ['80', '443']
           }]
       }
   },{
-      'name': 'internal',
+      'name': 'allow-internal',
       'type': 'compute.v1.firewall',
       'properties': {
           'network': network,
-          'sourceTags': ['swarm'],
+          'sourceRanges': ['10.128.0.0/9'],
           'allowed': [{
-              'IPProtocol': 'TCP'
+              'IPProtocol': 'tcp',
+              "ports": ['0-65535']
+          },{
+              'IPProtocol': 'udp',
+              "ports": ['0-65535']
           }]
       }
   }]
diff --git a/gcp/configuration/templates/manager.py b/gcp/configuration/templates/manager.py
@@ -6,68 +6,91 @@ def GenerateConfig(context):
   project = context.env['project']
   zone = context.properties['zone']
   machineType = context.properties['machineType']
+  image = context.properties['image']
   network = '$(ref.' + context.properties['network'] + '.selfLink)'
 
   script = r"""
 #!/bin/bash
 set -x
 
 service docker start
-docker swarm init --advertise-addr ens4:2377 --listen-addr ens4:2377
+docker swarm init --advertise-addr eth0:2377 --listen-addr eth0:2377
 
-TOKEN=$(docker swarm join-token worker -q)
 PROJECT=$(curl -s http://metadata.google.internal/computeMetadata/v1/project/project-id -H "Metadata-Flavor: Google")
 ACCESS_TOKEN=$(curl -s http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token -H "Metadata-Flavor: Google" | jq -r ".access_token")
 
-curl -s -X PUT -H "Content-Type: application/json" -d "{\"text\":\"${TOKEN}\"}" https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables/token -H "Authorization":"Bearer ${ACCESS_TOKEN}"
-"""
+LEADER_IP=$(curl -s http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip -H "Metadata-Flavor: Google")
+curl -f -s -X POST -H "Content-Type: application/json" -d "{'name':'projects/${PROJECT}/configs/swarm-config/variables/leader-ip','text':'${LEADER_IP}'}" https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables -H "Authorization":"Bearer ${ACCESS_TOKEN}"
 
-  outputs = [{
-      'name': 'internalIP',
-      'value': '$(ref.' + context.env['name'] + '.networkInterfaces[0].networkIP)'
-  }]
+if [ $? -eq 0 ]; then
+    echo "I'm a leader"
+
+    WORKER_TOKEN=$(docker swarm join-token worker -q)
+    curl -s -X POST -H "Content-Type: application/json" -d "{'name':'projects/${PROJECT}/configs/swarm-config/variables/worker-token','text':'${WORKER_TOKEN}'}" https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables -H "Authorization":"Bearer ${ACCESS_TOKEN}"
+
+    MANAGER_TOKEN=$(docker swarm join-token manager -q)
+    curl -s -X POST -H "Content-Type: application/json" -d "{'name':'projects/${PROJECT}/configs/swarm-config/variables/manager-token','text':'${MANAGER_TOKEN}'}" https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables -H "Authorization":"Bearer ${ACCESS_TOKEN}"
+else
+    echo "I'm not a leader"
+
+    docker swarm leave --force
+
+    for i in $(seq 1 300); do
+        LEADER_IP=$(curl -sSL "https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables/leader-ip" -H "Authorization":"Bearer ${ACCESS_TOKEN}" | jq -r ".text // empty")
+        if [ ! -z "${LEADER_IP}" ]; then
+            TOKEN=$(curl -sSL "https://runtimeconfig.googleapis.com/v1beta1/projects/${PROJECT}/configs/swarm-config/variables/manager-token" -H "Authorization":"Bearer ${ACCESS_TOKEN}" | jq -r ".text // empty")
+            docker swarm join --token "${TOKEN}" "${LEADER_IP}" --advertise-addr eth0:2377 --listen-addr eth0:2377
+            break
+        fi
+
+        sleep 1
+    done
+fi
+"""
 
   resources = [{
       'name': context.env['name'],
-      'type': 'compute.v1.instance',
+      'type': 'compute.v1.instanceTemplate',
       'properties': {
-          'zone': zone,
-          'tags': {
-              'items': ['swarm', 'swarm-manager']
-          },
-          'machineType': '/'.join(['projects', project,
-                                  'zones', zone,
-                                  'machineTypes', machineType]),
-          'disks': [{
-              'deviceName': 'boot',
-              'type': 'PERSISTENT',
-              'boot': True,
-              'autoDelete': True,
-              'initializeParams': {
-                  'sourceImage': '/'.join(['projects', project,
-                                          'global',
-                                          'images', 'docker2'])
-              }
-          }],
-          'networkInterfaces': [{
-              'network': network,
-              'accessConfigs': [{
-                  'name': 'External NAT',
-                  'type': 'ONE_TO_ONE_NAT'
+          'properties': {
+              'zone': zone,
+              'machineType': machineType,
+              'tags': {
+                  'items': ['swarm', 'swarm-manager']
+              },
+              'disks': [{
+                  'deviceName': 'boot',
+                  'type': 'PERSISTENT',
+                  'boot': True,
+                  'autoDelete': True,
+                  'initializeParams': {
+                      'sourceImage': image
+                  }
+              }],
+              'networkInterfaces': [{
+                  'network': network,
+                  'accessConfigs': [{
+                      'name': 'External NAT',
+                      'type': 'ONE_TO_ONE_NAT'
+                  }]
+              }],
+              'metadata': {
+                  'items': [{
+                      'key': 'startup-script',
+                      'value': script
+                  }]
+              },
+              'scheduling': {
+                  'preemptible': False,
+                  'onHostMaintenance': 'TERMINATE',
+                  'automaticRestart': False
+              },
+              'serviceAccounts': [{
+                  'scopes': [
+                      'https://www.googleapis.com/auth/cloudruntimeconfig'
+                  ]
               }]
-          }],
-          'metadata': {
-              'items': [{
-                  'key': 'startup-script',
-                  'value': script
-              }]
-          },
-          'serviceAccounts': [{
-              'scopes': [
-                  'https://www.googleapis.com/auth/cloudruntimeconfig'
-              ]
-          }]
+          }
       }
   }]
-
-  return {'resources': resources, 'outputs': outputs}
+  return {'resources': resources}
diff --git a/gcp/configuration/templates/managers.py b/gcp/configuration/templates/managers.py
@@ -0,0 +1,26 @@
+# Copyright 2016 Docker Inc. All rights reserved.
+
+"""Manager Instance Group."""
+
+def GenerateConfig(context):
+  project = context.env['project']
+  zone = context.properties['zone']
+  size = context.properties['size']
+  template = context.properties['template']
+
+  resources = [{
+      'name': context.env['name'],
+      'type': 'compute.v1.instanceGroupManager',
+      'properties': {
+          'zone': zone,
+          'instanceTemplate': '/'.join(['projects', project,
+                                       'global',
+                                       'instanceTemplates', template]),
+          'baseInstanceName': context.env['name'],
+          'targetSize': size,
+          'autoHealingPolicies': [{
+              'initialDelaySec': 300
+          }]
+      }
+  }]
+  return {'resources': resources}
diff --git a/gcp/configuration/templates/network.py b/gcp/configuration/templates/network.py
@@ -7,7 +7,7 @@ def GenerateConfig(context):
       'name': context.env['name'],
       'type': 'compute.v1.network',
       'properties': {
-          'IPv4Range': '10.0.0.1/16'
+          'autoCreateSubnetworks': True
       }
   }]
   return {'resources': resources}
diff --git a/gcp/configuration/templates/swarm.py b/gcp/configuration/templates/swarm.py
@@ -4,33 +4,49 @@
 
 def GenerateConfig(context):
   zone = context.properties['zone']
-  machineType = context.properties['machineType']
-  size = context.properties['size']
+  managerCount = context.properties['managerCount']
+  managerMachineType = context.properties['managerMachineType']
+  workerCount = context.properties['workerCount']
+  workerMachineType = context.properties['workerMachineType']
+  preemptible = context.properties['preemptible']
 
   resources = [{
+      'name': 'docker',
+      'type': 'templates/disk-image.py'
+  }, {
       'name': 'manager',
       'type': 'templates/manager.py',
       'properties': {
           'zone': zone,
-          'machineType': machineType,
+          'machineType': managerMachineType,
+          'image': '$(ref.docker.selfLink)',
           'network': 'swarm-network'
       }
+  }, {
+      'name': 'managers',
+      'type': 'templates/managers.py',
+      'properties': {
+          'zone': zone,
+          'template': '$(ref.manager.name)',
+          'size': managerCount
+      }
   }, {
       'name': 'worker',
       'type': 'templates/worker.py',
       'properties': {
           'zone': zone,
-          'machineType': machineType,
+          'machineType': workerMachineType,
+          'preemptible': preemptible,
+          'image': '$(ref.docker.selfLink)',
           'network': 'swarm-network',
-          'managerIP': '$(ref.manager.internalIP)'
       }
   }, {
       'name': 'workers',
       'type': 'templates/workers.py',
       'properties': {
           'zone': zone,
           'template': '$(ref.worker.name)',
-          'size': size
+          'size': workerCount
       }
   }, {
       'name': 'swarm-network',
diff --git a/gcp/configuration/templates/swarm.py.schema b/gcp/configuration/templates/swarm.py.schema
diff --git a/gcp/configuration/templates/worker.py b/gcp/configuration/templates/worker.py
diff --git a/gcp/configuration/templates/workers.py b/gcp/configuration/templates/workers.py
diff --git a/gcp/scripts/export-image.sh b/gcp/scripts/export-image.sh

Original file line number	Diff line number	Diff line change
`@@ -7,7 +7,7 @@ def GenerateConfig(context):`
`7`	`7`	`'name': context.env['name'],`
`8`	`8`	`'type': 'compute.v1.network',`
`9`	`9`	`'properties': {`
`10`		`- 'IPv4Range': '10.0.0.1/16'`
	`10`	`+ 'autoCreateSubnetworks': True`
`11`	`11`	`}`
`12`	`12`	`}]`
`13`	`13`	`return {'resources': resources}`