Merge pull request docker-archive#411 from docker/ddc-dynamic

nathanleclaire · web-flow · commit a475bcc60607 · 2016-11-10T11:09:36.000-08:00
Added support for dynamic DDC generation
diff --git a/Makefile b/Makefile
@@ -8,6 +8,12 @@ CHANNEL_CLOUD := alpha
 DOCKER_EXPERIMENTAL := 1
 VHD_SKU := docker4azure
 VHD_VERSION := 1.12.18
+# stage offer will have the -preview 
+OFFER_ID := docker4azure
+CS_VHD_SKU := docker4azure-cs-1_12
+CS_VHD_VERSION := 1.0.1
+# stage offer will have the -preview 
+CS_OFFER_ID := docker4azure-cs-preview
 export
 
 release: moby/alpine/cloud/aws/ami_id.out moby/alpine/cloud/azure/vhd_blob_url.out dockerimages
diff --git a/azure/dockerfiles/files/ddc-init/entry.sh b/azure/dockerfiles/files/ddc-init/entry.sh
@@ -14,8 +14,6 @@ echo "REGION=$REGION"
 echo "RGROUP_NAME=$RGROUP_NAME"
 echo "APP_ID=$APP_ID"
 echo "TENANT_ID=$TENANT_ID"
-echo "LB_NAME=$LB_NAME"
-echo "LB_IP=$LB_IP"
 echo "UCP_ADMIN_USER=$UCP_ADMIN_USER"
 echo "UCP_IMAGE=${HUB_NAMESPACE}/ucp:${HUB_TAG}"
 echo "#================"
diff --git a/azure/release/Makefile b/azure/release/Makefile
@@ -9,10 +9,10 @@ build:
 	docker build -t azure-release .
 
 template: clean build
-	docker run -ti --rm -v $(PWD)/dist:/home/docker/dist/ azure-release --docker_version="$(EDITIONS_DOCKER_VERSION)" --edition_version="$(EDITIONS_VERSION)" --vhd_sku="$(VHD_SKU)" --vhd_version="$(VHD_VERSION)" --channel="$(CHANNEL)" --channel_cloud="$(CHANNEL_CLOUD)"
+	docker run -ti --rm -v $(PWD)/dist:/home/docker/dist/ azure-release --docker_version="$(EDITIONS_DOCKER_VERSION)" --edition_version="$(EDITIONS_VERSION)" --vhd_sku="$(VHD_SKU)" --vhd_version="$(VHD_VERSION)" --channel="$(CHANNEL)" --channel_cloud="$(CHANNEL_CLOUD)" --channel_ddc="$(CHANNEL_DDC)" --cs_vhd_sku="$(CS_VHD_SKU)" --cs_vhd_version="$(CS_VHD_VERSION)" --cs_offer_id="$(CS_OFFER_ID)" 
 
 release: clean build
-	docker run -ti --rm -v $(PWD)/dist:/home/docker/dist/ azure-release --upload --docker_version="$(EDITIONS_DOCKER_VERSION)" --edition_version="$(EDITIONS_VERSION)" --vhd_sku="$(VHD_SKU)" --vhd_version="$(VHD_VERSION)" --channel="$(CHANNEL)" --channel_cloud="$(CHANNEL_CLOUD)"
+	docker run -ti --rm -v $(PWD)/dist:/home/docker/dist/ azure-release --upload --docker_version="$(EDITIONS_DOCKER_VERSION)" --edition_version="$(EDITIONS_VERSION)" --vhd_sku="$(VHD_SKU)" --vhd_version="$(VHD_VERSION)" --channel="$(CHANNEL)" --channel_cloud="$(CHANNEL_CLOUD)" --channel_ddc="$(CHANNEL_DDC)" --cs_vhd_sku="$(CS_VHD_SKU)" --cs_vhd_version="$(CS_VHD_VERSION)"  --cs_offer_id="$(CS_OFFER_ID)"
 
 clean:
 	# rm -f tmp/*
diff --git a/azure/release/files/custom-data_manager.sh b/azure/release/files/custom-data_manager.sh
@@ -25,4 +25,4 @@ docker run --log-driver=json-file  --restart=no -d -e ROLE="$ROLE" -e REGION="$R
 docker run --log-driver=json-file  --restart=always -d -e ROLE="$ROLE" -e REGION="$REGION" -e TENANT_ID="$TENANT_ID" -e APP_ID="$APP_ID" -e APP_SECRET="$APP_SECRET" -e ACCOUNT_ID="$ACCOUNT_ID" -e GROUP_NAME="$GROUP_NAME" -e PRIVATE_IP="$MANAGER_IP" -e DOCKER_FOR_IAAS_VERSION="$DOCKER_FOR_IAAS_VERSION" -e SWARM_INFO_TABLE="$SWARM_INFO_TABLE" -e SWARM_INFO_STORAGE_ACCOUNT="$SWARM_INFO_STORAGE_ACCOUNT" -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker -v /var/log:/var/log docker4x/guide-azure:"$DOCKER_FOR_IAAS_VERSION"
 echo default: "$LB_NAME" >> /var/lib/docker/swarm/elb.config
 echo "$LB_NAME" > /var/lib/docker/swarm/lb_name
-docker run --log-driver=json-file  -v /var/run/docker.sock:/var/run/docker.sock  -v /var/lib/docker/swarm:/var/lib/docker/swarm --name=editions_controller docker4x/l4controller-azure:"$DOCKER_FOR_IAAS_VERSION" run --ad_app_id="$APP_ID" --ad_app_secret="$APP_SECRET" --subscription_id="$SUB_ID" --resource_group="$GROUP_NAME" --log=4 --default_lb_name="$LB_NAME" --environment=AzurePublicCloud
+docker run -d --log-driver=json-file  -v /var/run/docker.sock:/var/run/docker.sock  -v /var/lib/docker/swarm:/var/lib/docker/swarm --name=editions_controller docker4x/l4controller-azure:"$DOCKER_FOR_IAAS_VERSION" run --ad_app_id="$APP_ID" --ad_app_secret="$APP_SECRET" --subscription_id="$SUB_ID" --resource_group="$GROUP_NAME" --log=4 --default_lb_name="$LB_NAME" --environment=AzurePublicCloud
diff --git a/azure/release/files/custom-data_manager_cloud.sh b/azure/release/files/custom-data_manager_cloud.sh
@@ -19,7 +19,7 @@ docker run --restart=no -d -e ROLE="$ROLE" -e REGION="$REGION" -e TENANT_ID="$TE
 docker run --restart=always -d -e ROLE="$ROLE -e REGION="$REGION -e ACCOUNT_ID="$ACCOUNT_ID" -e PRIVATE_IP="$MANAGER_IP" -e DOCKER_FOR_IAAS_VERSION="$DOCKER_FOR_IAAS_VERSION" -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker docker4x/guide-azure:"$DOCKER_FOR_IAAS_VERSION"
 echo default: "$LB_NAME" >> /var/lib/docker/swarm/elb.config
 echo "$LB_NAME" > /var/lib/docker/swarm/lb_name
-docker run -v /var/run/docker.sock:/var/run/docker.sock  -v /var/lib/docker/swarm:/var/lib/docker/swarm --name=editions_controller docker4x/l4controller-azure:"$DOCKER_FOR_IAAS_VERSION" run --ad_app_id="$APP_ID" --ad_app_secret="$APP_SECRET" --subscription_id="$SUB_ID" --resource_group="$GROUP_NAME" --log=4 --default_lb_name="$LB_NAME" --environment=AzurePublicCloud
+docker run -d -v /var/run/docker.sock:/var/run/docker.sock  -v /var/lib/docker/swarm:/var/lib/docker/swarm --name=editions_controller docker4x/l4controller-azure:"$DOCKER_FOR_IAAS_VERSION" run --ad_app_id="$APP_ID" --ad_app_secret="$APP_SECRET" --subscription_id="$SUB_ID" --resource_group="$GROUP_NAME" --log=4 --default_lb_name="$LB_NAME" --environment=AzurePublicCloud
 
 
 
diff --git a/azure/release/files/custom-data_manager_ddc.sh b/azure/release/files/custom-data_manager_ddc.sh
@@ -8,29 +8,28 @@ export LB_NAME="variables('lbName')"
 export APP_ID="variables('adServicePrincipalAppID')"
 export APP_SECRET="variables('adServicePrincipalAppSecret')"
 export TENANT_ID="variables('adServicePrincipalTenantID')"
-
 export DDC_USER="variables('ddcUser')"
 export DDC_PASS="variables('ddcPass')"
-export RGROUP_NAME="variables('groupName')"; 
-export LB_NAME="variables('lbPublicIPAddressName')"; 
-
-
- 
-
-docker run --restart=no -d -e ROLE="$ROLE" -e REGION="$REGION" -e ACCOUNT_ID="$ACCOUNT_ID" -e PRIVATE_IP="$MANAGER_IP" -e DOCKER_FOR_IAAS_VERSION="$DOCKER_FOR_IAAS_VERSION" -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker -v /var/log:/var/log docker4x/init-azure:"$DOCKER_FOR_IAAS_VERSION"
-docker run --restart=always -d -e ROLE="$ROLE -e REGION="$REGION -e ACCOUNT_ID="$ACCOUNT_ID" -e PRIVATE_IP="$MANAGER_IP" -e DOCKER_FOR_IAAS_VERSION="$DOCKER_FOR_IAAS_VERSION" -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker docker4x/guide-azure:"$DOCKER_FOR_IAAS_VERSION"
+export RGROUP_NAME="variables('groupName')";
+export SWARM_INFO_TABLE="variables('swarmInfoTable')"
+export SWARM_INFO_STORAGE_ACCOUNT="variables('swarmInfoStorageAccount')"
+export SWARM_LOGS_STORAGE_ACCOUNT="variables('swarmLogsStorageAccount')"
+export MANAGER_IP=$(ifconfig eth0 | grep "inet addr:" | cut -d: -f2 | cut -d" " -f1)
+# create daemon config with custom tag
+echo "{\"log-driver\": \"syslog\",\"log-opts\": {\"syslog-address\": \"udp://localhost:514\", \"tag\": \"{{.Name}}/{{.ID}}\" }}" > /etc/docker/daemon.json
+service docker restart
+sleep 5
+
+# add logging container
+docker volume create --name container-logs
+docker run --log-driver=json-file --name=editions_logger --restart=always -d -e ROLE="$ROLE" -e REGION="$REGION" -e TENANT_ID="$TENANT_ID" -e APP_ID="$APP_ID" -e APP_SECRET="$APP_SECRET" -e ACCOUNT_ID="$ACCOUNT_ID" -e GROUP_NAME="$GROUP_NAME" -e SWARM_LOGS_STORAGE_ACCOUNT="$SWARM_LOGS_STORAGE_ACCOUNT" -e SWARM_FILE_SHARE=`hostname` -p 514:514/udp -v container-logs:/log/ docker4x/logger-azure:"$DOCKER_FOR_IAAS_VERSION"
+
+docker run --log-driver=json-file  --restart=no -d -e ROLE="$ROLE" -e REGION="$REGION" -e TENANT_ID="$TENANT_ID" -e APP_ID="$APP_ID" -e APP_SECRET="$APP_SECRET" -e ACCOUNT_ID="$ACCOUNT_ID" -e GROUP_NAME="$GROUP_NAME" -e PRIVATE_IP="$MANAGER_IP" -e DOCKER_FOR_IAAS_VERSION="$DOCKER_FOR_IAAS_VERSION" -e SWARM_INFO_TABLE="$SWARM_INFO_TABLE" -e SWARM_INFO_STORAGE_ACCOUNT="$SWARM_INFO_STORAGE_ACCOUNT" -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker -v /var/log:/var/log docker4x/init-azure:"$DOCKER_FOR_IAAS_VERSION"
+docker run --log-driver=json-file  --restart=always -d -e ROLE="$ROLE" -e REGION="$REGION" -e TENANT_ID="$TENANT_ID" -e APP_ID="$APP_ID" -e APP_SECRET="$APP_SECRET" -e ACCOUNT_ID="$ACCOUNT_ID" -e GROUP_NAME="$GROUP_NAME" -e PRIVATE_IP="$MANAGER_IP" -e DOCKER_FOR_IAAS_VERSION="$DOCKER_FOR_IAAS_VERSION" -e SWARM_INFO_TABLE="$SWARM_INFO_TABLE" -e SWARM_INFO_STORAGE_ACCOUNT="$SWARM_INFO_STORAGE_ACCOUNT" -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker -v /var/log:/var/log docker4x/guide-azure:"$DOCKER_FOR_IAAS_VERSION"
 echo default: "$LB_NAME" >> /var/lib/docker/swarm/elb.config
 echo "$LB_NAME" > /var/lib/docker/swarm/lb_name
-docker run -v /var/run/docker.sock:/var/run/docker.sock  -v /var/lib/docker/swarm:/var/lib/docker/swarm --name=editions_controller docker4x/l4controller-azure:"$DOCKER_FOR_IAAS_VERSION" run --ad_app_id="$APP_ID" --ad_app_secret="$APP_SECRET" --subscription_id="$SUB_ID" --resource_group="$GROUP_NAME" --log=4 --default_lb_name="$LB_NAME" --environment=AzurePublicCloud
-
-
-
-export IS_LEADER=$(docker node inspect self -f "{{ .ManagerStatus.Leader }}") 
-if [ "$IS_LEADER" == "true" ]; then 
-docker run --rm --name=cloud_registration -v /var/run/docker.sock:/var/run/docker.sock -e DOCKER_USER="$CLOUD_USER" -e DOCKER_PASS="$CLOUD_KEY" -e SWARM_NAME="$SWARM_NAME" -e INTERNAL_ENDPOINT="$LB_SSH_IP:2376" docker4x/cloud-azure:$DOCKER_FOR_IAAS_VERSION
-fi
-
 
+docker run --log-driver=json-file --restart=no --rm -e ROLE=$ROLE -e REGION=$REGION -e ACCOUNT_ID=$ACCOUNT_ID -e APP_ID=$APP_ID -e APP_SECRET=$APP_SECRET -e TENANT_ID=$TENANT_ID -e RGROUP_NAME=$RGROUP_NAME -e UCP_ADMIN_USER=$DDC_USER -e UCP_ADMIN_PASSWORD=$DDC_PASS -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker docker4x/ddc-init-azure:$DOCKER_FOR_IAAS_VERSION
 
-docker run --restart=no --rm -e ROLE=$ROLE -e REGION=$REGION -e ACCOUNT_ID=$ACCOUNT_ID -e APP_ID=$APP_ID -e APP_SECRET=$APP_SECRET -e TENANT_ID=$TENANT_ID -e RGROUP_NAME=$RGROUP_NAME -e UCP_ADMIN_USER=$DDC_USER -e UCP_ADMIN_PASSWORD=$DDC_PASS -e LB_NAME=$LB_NAME -e LB_IP=$LB_SSH_IP -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker docker4x/ddc-init-azure:$DOCKER_FOR_IAAS_VERSION
+docker run -d --log-driver=json-file -v /var/run/docker.sock:/var/run/docker.sock -v /var/lib/docker/swarm:/var/lib/docker/swarm --name=editions_controller docker4x/l4controller-azure:"$DOCKER_FOR_IAAS_VERSION" run --ad_app_id="$APP_ID" --ad_app_secret="$APP_SECRET" --subscription_id="$SUB_ID" --resource_group="$GROUP_NAME" --log=4 --default_lb_name="$LB_NAME" --environment=AzurePublicCloud
 
diff --git a/azure/release/files/custom-data_worker_ddc.sh b/azure/release/files/custom-data_worker_ddc.sh
@@ -0,0 +1,25 @@
+export ROLE="WORKER"
+export REGION="variables('storageLocation')"
+export ACCOUNT_ID="variables('accountID')"
+export DOCKER_FOR_IAAS_VERSION="variables('dockerForIAASVersion')"
+export GROUP_NAME="variables('groupName')"
+export APP_ID="variables('adServicePrincipalAppID')"
+export APP_SECRET="variables('adServicePrincipalAppSecret')"
+export TENANT_ID="variables('adServicePrincipalTenantID')"
+export SWARM_INFO_TABLE="variables('swarmInfoTable')"
+export SWARM_INFO_STORAGE_ACCOUNT="variables('swarmInfoStorageAccount')"
+export SWARM_LOGS_STORAGE_ACCOUNT="variables('swarmLogsStorageAccount')"
+
+# create daemon config with custom tag
+echo "{\"log-driver\": \"syslog\",\"log-opts\": {\"syslog-address\": \"udp://localhost:514\", \"tag\": \"{{.Name}}/{{.ID}}\" }}" > /etc/docker/daemon.json
+service docker restart
+sleep 5
+
+# add logging container
+docker volume create --name container-logs
+docker run --log-driver=json-file --name=editions_logger --restart=always -d -e ROLE="$ROLE" -e REGION="$REGION" -e TENANT_ID="$TENANT_ID" -e APP_ID="$APP_ID" -e APP_SECRET="$APP_SECRET" -e ACCOUNT_ID="$ACCOUNT_ID" -e GROUP_NAME="$GROUP_NAME" -e SWARM_LOGS_STORAGE_ACCOUNT="$SWARM_LOGS_STORAGE_ACCOUNT" -e SWARM_FILE_SHARE=`hostname` -p 514:514/udp -v container-logs:/log/ docker4x/logger-azure:"$DOCKER_FOR_IAAS_VERSION"
+
+docker run --log-driver=json-file --restart=no -d -e ROLE=$ROLE -e REGION=$REGION -e TENANT_ID="$TENANT_ID" -e APP_ID="$APP_ID" -e APP_SECRET="$APP_SECRET" -e GROUP_NAME="$GROUP_NAME" -e SWARM_INFO_TABLE="$SWARM_INFO_TABLE" -e SWARM_INFO_STORAGE_ACCOUNT="$SWARM_INFO_STORAGE_ACCOUNT" -e ACCOUNT_ID=$ACCOUNT_ID -e MANAGER_IP=$MANAGER_IP -e DOCKER_FOR_IAAS_VERSION=$DOCKER_FOR_IAAS_VERSION -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker -v /var/log:/var/log docker4x/init-azure:$DOCKER_FOR_IAAS_VERSION
+docker run --log-driver=json-file --restart=no --rm -e ROLE=$ROLE -e REGION=$REGION -e ACCOUNT_ID=$ACCOUNT_ID -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker docker4x/ddc-init-azure:$DOCKER_FOR_IAAS_VERSION
+
+docker run --log-driver=json-file --restart=always -d -e ROLE=$ROLE -e REGION=$REGION -e ACCOUNT_ID=$ACCOUNT_ID -e MANAGER_IP=$MANAGER_IP -e DOCKER_FOR_IAAS_VERSION=$DOCKER_FOR_IAAS_VERSION -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker docker4x/guide-azure:$DOCKER_FOR_IAAS_VERSION
diff --git a/azure/release/files/release.py b/azure/release/files/release.py
@@ -2,7 +2,7 @@
 import json
 import argparse
 
-from utils import (create_rg_template, create_rg_cloud_template, upload_rg_template)
+from utils import (create_rg_template, create_rg_cloud_template, create_rg_ddc_template, upload_rg_template)
 
 
 CFN_TEMPLATE = '/home/docker/editions.template'
@@ -27,38 +27,66 @@ def main():
     parser.add_argument('-u', '--channel_cloud',
                         dest='channel_cloud', default="alpha",
                         help="Cloud release channel (beta, alpha, rc, nightly)")
+    parser.add_argument('--channel_ddc',
+                        dest='channel_ddc', default="alpha",
+                        help="DDC release channel (beta, alpha, rc, nightly)")
+    parser.add_argument('--offer_id',
+                        dest='offer_id', default="docker4azure",
+                        help="The Azure VHD Offer ID")
+    parser.add_argument('--cs_vhd_sku',
+                        dest='cs_vhd_sku',
+                        help="The Azure CS VHD SKU (i.e. docker4azure)")
+    parser.add_argument('--cs_vhd_version',
+                        dest='cs_vhd_version',
+                        help="The Azure CS VHD version (i.e. 1.0.0)")
+    parser.add_argument('--cs_offer_id',
+                        dest='cs_offer_id',
+                        help="The Azure CS VHD Offer ID")
     parser.add_argument("--upload", action="store_true",
                         help="Upload the Azure template once generated")
 
     args = parser.parse_args()
 
     release_channel = args.channel
     release_cloud_channel = args.channel_cloud
+    release_ddc_channel = args.channel_cloud
     docker_version = args.docker_version
     # TODO change to something else? where to get moby version?
     moby_version = docker_version
     edition_version = args.edition_version
     flat_edition_version = edition_version.replace(" ", "")
     vhd_sku = args.vhd_sku
     vhd_version = args.vhd_version
+    offer_id = args.offer_id
+    cs_vhd_sku = args.cs_vhd_sku
+    cs_vhd_version = args.cs_vhd_version
+    cs_offer_id = args.cs_offer_id
+
     docker_for_azure_version = u"azure-v{}".format(flat_edition_version)
     image_name = u"Moby Linux {}".format(docker_for_azure_version)
     image_description = u"The best OS for running Docker, version {}".format(moby_version)
     print("\n Variables")
     print(u"release_channel={}".format(release_channel))
     print(u"release_cloud_channel={}".format(release_cloud_channel))
+    print(u"release_ddc_channel={}".format(release_ddc_channel))
     print(u"docker_version={}".format(docker_version))
     print(u"edition_version={}".format(edition_version))
     print(u"vhd_sku={}".format(vhd_sku))
     print(u"vhd_version={}".format(vhd_version))
+    print(u"cs_vhd_sku={}".format(cs_vhd_sku))
+    print(u"cs_vhd_version={}".format(cs_vhd_version))
 
     print("Create CloudFormation template..")
     template_name = u"{}.json".format(docker_for_azure_version)
-    base_url = create_rg_template(vhd_sku, vhd_version, release_channel, docker_version,
+    base_url = create_rg_template(vhd_sku, vhd_version, offer_id, release_channel, docker_version,
                                  docker_for_azure_version, edition_version, CFN_TEMPLATE, template_name)
     cloud_template_name = u"{}-cloud.json".format(docker_for_azure_version)
     cloud_url = create_rg_cloud_template(release_cloud_channel, docker_version,
                                  docker_for_azure_version, edition_version, base_url, cloud_template_name)
+    
+    ddc_template_name = u"{}-ddc.json".format(docker_for_azure_version)
+    ddc_url = create_rg_ddc_template(cs_vhd_sku, cs_vhd_version, cs_offer_id, release_ddc_channel, docker_version,
+                                 docker_for_azure_version, edition_version, base_url, ddc_template_name)
 
     print("------------------")
     print(u"Finshed.. \n")
@@ -67,7 +95,8 @@ def main():
         print(u"Uploading templates.. \n")
         s3_url = upload_rg_template(release_channel, template_name, base_url)
         s3_cloud_url = upload_rg_template(release_channel, cloud_template_name, cloud_url)
-        print(u"Uploaded CloudFormation \n\t URL={0} \n\t CLOUD_URL={1} \n".format(s3_url, s3_cloud_url))
+        s3_ddc_url = upload_rg_template(release_channel, ddc_template_name, ddc_url)
+        print(u"Uploaded CloudFormation \n\t URL={0} \n\t CLOUD_URL={1} \n\t DDC_URL={2} \n".format(s3_url, s3_cloud_url, s3_ddc_url))
 
     # TODO: git commit, tag release. requires github keys, etc.
     print("Don't forget to tag the code (git tag -a v{0}-{1} -m {0}; git push --tags)".format(
diff --git a/azure/release/files/utils.py b/azure/release/files/utils.py
