Merge pull request kubernetes-sigs#673 from jroden/master

✨ Spin flavor clusters as tilt resources

Author: k8s-ci-robot

Tiltfile

@@ -194,6 +194,86 @@ def capz():
 
     k8s_yaml(blob(yaml))
 
+
+# run worker clusters specified from 'tilt up' or in 'tilt_config.json'
+def flavors():
+    config.define_string_list("templates-to-run", args=True)
+    config.define_string_list("worker-flavors")
+    cfg = config.parse()
+    worker_templates = cfg.get('templates-to-run', [])
+    for flavor in cfg.get("worker-flavors", []):
+        if flavor not in worker_templates:
+            worker_templates.append(flavor)
+    for flavor in worker_templates:
+        deploy_worker_templates(flavor)
+
+
+def deploy_worker_templates(flavor):
+    # validate required metadata
+    required_metadata = ["AZURE_SUBSCRIPTION_ID","AZURE_TENANT_ID","AZURE_CLIENT_ID","AZURE_CLIENT_SECRET", "AZURE_SSH_PUBLIC_KEY"]
+    metadata = settings.get("kustomize_substitutions", {})
+    missing = [k for k in required_metadata if k not in metadata]
+    if missing:
+        fail("missing kustomize_substitutions keys {} in tilt-setting.json".format(missing))
+
+    # validate flavor exists
+    if flavor == "default":
+        yaml_file = "./templates/cluster-template.yaml"
+    else:
+        yaml_file = "./templates/cluster-template-" + flavor + ".yaml"
+        if not os.path.exists(yaml_file):
+            fail(yaml_file + " not found")
+
+    yaml = str(read_file(yaml_file))
+
+    # azure account information replacements
+    substitutions = settings.get("kustomize_substitutions", {})
+    for substitution in substitutions:
+        value = substitutions[substitution]
+        yaml = yaml.replace("${" + substitution + "}", value)
+
+    # if metadata defined for worker-templates in tilt_settings
+    if "worker-templates" in settings:
+        # first priority replacements defined per template
+        if "flavors" in settings.get("worker-templates", {}):
+            substitutions = settings.get("worker-templates").get("flavors").get(flavor, {})
+            for substitution in substitutions:
+                value = substitutions[substitution]
+                yaml = yaml.replace("${" + substitution + "}", value)
+
+        # second priority replacements defined common to templates
+        if "metadata" in settings.get("worker-templates", {}):
+            substitutions = settings.get("worker-templates").get("metadata", {})
+            for substitution in substitutions:
+                value = substitutions[substitution]
+                yaml = yaml.replace("${" + substitution + "}", value)
+
+    # programmatically define any remaining vars
+    substitutions = {
+        "CLUSTER_NAME": flavor + "-template",
+        "AZURE_LOCATION": "eastus",
+        "AZURE_VNET_NAME": flavor + "-template-vnet",
+        "AZURE_RESOURCE_GROUP": flavor + "-template-rg",
+        "CONTROL_PLANE_MACHINE_COUNT": "1",
+        "KUBERNETES_VERSION": "v1.18.3",
+        "AZURE_CONTROL_PLANE_MACHINE_TYPE": "Standard_D2s_v3",
+        "WORKER_MACHINE_COUNT": "2",
+        "AZURE_NODE_MACHINE_TYPE": "Standard_D2s_v3",
+    }
+    for substitution in substitutions:
+        value = substitutions[substitution]
+        yaml = yaml.replace("${" + substitution + "}", value)
+
+    yaml = yaml.replace('"', '\\"')     # add escape character to double quotes in yaml
+    
+    local_resource(
+        "worker-" + flavor,
+        cmd = "make generate-flavors; echo \"" + yaml + "\" > ./.tiltbuild/worker-" + flavor + ".yaml; kubectl apply -f ./.tiltbuild/worker-" + flavor + ".yaml",
+        auto_init = False,
+        trigger_mode = TRIGGER_MODE_MANUAL
+    )
+
+
 ##############################
 # Actual work happens here
 ##############################
@@ -208,3 +288,5 @@ if settings.get("deploy_cert_manager"):
 deploy_capi()
 
 capz()
+
+flavors()

docs/development.md

@@ -20,6 +20,7 @@
     - [Tilt for dev in CAPZ](#tilt-for-dev-in-capz)
     - [Tilt for dev in both CAPZ and CAPI](#tilt-for-dev-in-both-capz-and-capi)
     - [Deploying a workload cluster](#deploying-a-workload-cluster)
+    - [Deploying a flavor cluster as a local tilt resource](../templates/flavors/README.md#Running-flavor-clusters-as-a-tilt-resource)
   - [Manual Testing](#manual-testing)
     - [Creating a dev cluster](#creating-a-dev-cluster)
       - [Building and pushing dev images](#building-and-pushing-dev-images)
@@ -145,6 +146,8 @@ If you would like to enable these features, add `extra_args` as specified in [Th
 
 Once your kind management cluster is up and running, you can [deploy a workload cluster](#deploying-a-workload-cluster).
 
+You can also [deploy a flavor cluster as a local tilt resource](../templates/flavors/README.md#Running-flavor-clusters-as-a-tilt-resource).
+
 To tear down the kind cluster built by the command above, just run:
 
 ```shell

templates/flavors/README.md

@@ -14,3 +14,76 @@ appended to the end of the cluster-template.yaml, e.g cluster-template-{director
 flavor can be used by specifying `--flavor {directory-name}`.
 
 To generate all CAPZ flavors, run `make generate-flavors`.
+
+
+## Running flavor clusters as a tilt resource
+
+#### From CLI
+run ```tilt up ${flavors}``` to spin up worker clusters in Azure represented by a Tilt local resource.  You can also modify flavors while Tilt is up by running ```tilt args ${flavors}```
+
+#### From Tilt Config
+Add your desired flavors to tilt_config.json:
+```json
+{
+    "worker-flavors": ["default", "aks", "external-cloud-provider", "machinepool", "system-assigned-identity", "user-assigned-identity"]
+}
+```
+
+#### Requirements
+Please note your tilt-settings.json must contain at minimum the following fields when using tilt resources to deploy cluster flavors:
+```json
+{
+    "kustomize_substitutions": {
+        "AZURE_SUBSCRIPTION_ID_B64": "******",
+        "AZURE_TENANT_ID_B64": "******",
+        "AZURE_CLIENT_SECRET_B64": "******",
+        "AZURE_CLIENT_ID_B64": "******",
+        "AZURE_SUBSCRIPTION_ID": "******",
+        "AZURE_TENANT_ID": "******",
+        "AZURE_CLIENT_ID": "******",
+        "AZURE_CLIENT_SECRET": "******",
+        "AZURE_SSH_PUBLIC_KEY": "********"
+    }
+}
+```
+
+#### Defining Variable Overrides
+If you wish to override the default variables for flavor workers, you can specify them as part of your tilt-settings.json as seen in the example below.  Please note, the precedence of variables is as follows:
+
+1. explicitly defined vars for each flavor i.e. worker-templates.flavors[0].AZURE_VNET_NAME
+2. vars defined at 'metadata' level-- spans workers i.e. metadata.AZURE_VNET_NAME
+3. programmatically defined default vars i.e. everything except azure tenant, client, subscription
+
+
+```json
+{
+    "kustomize_substitutions": {
+        "AZURE_SUBSCRIPTION_ID_B64": "****",
+        "AZURE_TENANT_ID_B64": "****",
+        "AZURE_CLIENT_SECRET_B64": "****",
+        "AZURE_CLIENT_ID_B64": "****"
+    },
+    "worker-templates": {
+        "flavors": [{
+            "default": {
+                "CLUSTER_NAME": "example-default-cluster-name",
+                "AZURE_VNET_NAME": "example-vnet-one"
+            },
+            "system-assigned-identity": {
+                "CLUSTER_NAME": "example-SAI-cluster-name",
+                "AZURE_LOCATION": "westus",
+                "AZURE_VNET_NAME": "example-vnet-two"
+            }
+        }],
+        "metadata": {
+            "AZURE_LOCATION": "eastus",
+            "AZURE_RESOURCE_GROUP": "test-resource-group-name",
+            "CONTROL_PLANE_MACHINE_COUNT": "1",
+            "KUBERNETES_VERSION": "v1.18.3",
+            "AZURE_CONTROL_PLANE_MACHINE_TYPE": "Standard_D2s_v3",
+            "WORKER_MACHINE_COUNT": "2",
+            "AZURE_NODE_MACHINE_TYPE": "Standard_D2s_v3"
+        }
+    }
+}
+```