feat: add tilt support for capz focused

Author: devigned

.gitignore

@@ -49,6 +49,8 @@ bazel-*
 
 # Tilt files.
 .tiltbuild
+/tilt.d
+tilt-settings.json
 
 # e2e output
 test/e2e/junit.e2e_suite.*.xml

Makefile

@@ -293,7 +293,7 @@ release-notes: $(RELEASE_NOTES)
 .PHONY: create-management-cluster
 create-management-cluster: $(ENVSUBST)
 	## Create kind management cluster.
-	kind create cluster --name=clusterapi
+	$(MAKE) kind-create
 
 	# Install cert manager and wait for availability
 	kubectl create -f https://github.com/jetstack/cert-manager/releases/download/v0.11.1/cert-manager.yaml
@@ -303,7 +303,7 @@ create-management-cluster: $(ENVSUBST)
 	kubectl apply -f https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.3/cluster-api-components.yaml
 
 	# Deploy CAPZ
-	kind load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=clusterapi
+	kind load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=capz
 	kustomize build config | $(ENVSUBST) | kubectl apply -f -
 
 	# Wait for CAPI pods
@@ -316,7 +316,7 @@ create-management-cluster: $(ENVSUBST)
 
 	# required sleep for when creating management and workload cluster simultaneously
 	sleep 10
-	@echo 'Set kubectl context to the kind management cluster by running "kubectl config set-context kind-clusterapi"'
+	@echo 'Set kubectl context to the kind management cluster by running "kubectl config set-context kind-capz"'
 
 .PHONY: create-workload-cluster
 create-workload-cluster: $(ENVSUBST)
@@ -342,13 +342,26 @@ delete-workload-cluster: ## Deletes the example workload Kubernetes cluster
 	@echo 'Your Azure resources will now be deleted, this can take up to 20 minutes'
 	kubectl delete cluster $(CLUSTER_NAME)
 
+## --------------------------------------
+## Tilt / Kind
+## --------------------------------------
+
+.PHONY: kind-create
+kind-create: ## create capz kind cluster if needed
+	./scripts/kind-with-registry.sh
+
+.PHONY: tilt-up
+tilt-up: kind-create ## start tilt and build kind cluster if needed
+	tilt up
+
 .PHONY: delete-cluster
-delete-cluster: delete-workload-cluster  ## Deletes the example kind cluster "clusterapi"
-	kind delete cluster --name=clusterapi
+delete-cluster: delete-workload-cluster  ## Deletes the example kind cluster "capz"
+	kind delete cluster --name=capz
 
 .PHONY: kind-reset
-kind-reset: ## Destroys the "clusterapi" kind cluster.
-	kind delete cluster --name=clusterapi || true
+kind-reset: ## Destroys the "capz" kind cluster.
+	kind delete cluster --name=capz || true
+
 
 ## --------------------------------------
 ## Cleanup / Verification

Tiltfile

@@ -0,0 +1,132 @@
+# -*- mode: Python -*-
+
+# set defaults
+
+settings = {
+    "allowed_contexts": [
+        "kind-capz"
+    ],
+    "deploy_cert_manager": True,
+    "preload_images_for_kind": True,
+    "kind_cluster_name": "capz",
+    "capi_version": "v0.3.3",
+    "cert_manager_version": "v0.11.0",
+    "feature_gates": '--feature-gates MachinePool=true'
+}
+
+keys = ["AZURE_SUBSCRIPTION_ID_B64", "AZURE_TENANT_ID_B64", "AZURE_CLIENT_SECRET_B64", "AZURE_CLIENT_ID_B64"]
+
+# global settings
+settings.update(read_json(
+    "tilt-settings.json",
+    default = {},
+))
+
+if settings.get("trigger_mode") == "manual":
+    trigger_mode(TRIGGER_MODE_MANUAL)
+
+if "allowed_contexts" in settings:
+    allow_k8s_contexts(settings.get("allowed_contexts"))
+
+if "default_registry" in settings:
+    default_registry(settings.get("default_registry"))
+
+
+# Prepull all the cert-manager images to your local environment and then load them directly into kind. This speeds up
+# setup if you're repeatedly destroying and recreating your kind cluster, as it doesn't have to pull the images over
+# the network each time.
+def deploy_cert_manager():
+    registry = "quay.io/jetstack"
+    version = settings.get("cert_manager_version")
+    images = ["cert-manager-controller", "cert-manager-cainjector", "cert-manager-webhook"]
+
+    if settings.get("preload_images_for_kind"):
+        for image in images:
+            local("docker pull {}/{}:{}".format(registry, image, version))
+            local("kind load docker-image --name {} {}/{}:{}".format(settings.get("kind_cluster_name"), registry, image, version))
+
+    local("kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/{}/cert-manager.yaml".format(version))
+
+    # wait for the service to become available
+    local("kubectl wait --for=condition=Available --timeout=300s apiservice v1beta1.webhook.cert-manager.io")
+
+
+# deploy CAPI
+def deploy_capi():
+    version = settings.get("capi_version")
+    local("kubectl apply -f https://github.com/kubernetes-sigs/cluster-api/releases/download/{}/cluster-api-components.yaml".format(version))
+    if settings.get("feature_gates"):
+        args_str = str(local('kubectl get deployments capi-controller-manager -n capi-system -o jsonpath={.spec.template.spec.containers[1].args}'))
+        if settings.get("feature_gates") not in args_str:
+            args = args_str[1:-1].split() # "[arg1 arg2 ...]" trim off the first and last, then split
+            args.append("\"{}\"".format(settings.get("feature_gates")))
+            patch = [{
+                "op": "replace",
+                "path": "/spec/template/spec/containers/1/args",
+                "value": args,
+            }]
+            local("kubectl patch deployment capi-controller-manager -n capi-system  --type json -p='{}'".format(str(encode_json(patch)).replace("\n", "")))
+
+
+# Users may define their own Tilt customizations in tilt.d. This directory is excluded from git and these files will
+# not be checked in to version control.
+def include_user_tilt_files():
+    user_tiltfiles = listdir("tilt.d")
+    for f in user_tiltfiles:
+        include(f)
+
+
+def append_arg_for_container_in_deployment(yaml_stream, name, namespace, contains_image_name, arg):
+    for item in yaml_stream:
+        if item["kind"] == "Deployment" and item.get("metadata").get("name") == name and item.get("metadata").get("namespace") == namespace:
+            containers = item.get("spec").get("template").get("spec").get("containers")
+            for container in containers:
+                if contains_image_name in container.get("image"):
+                    container.get("args").append("\"{}\"".format(arg))
+
+
+def fixup_yaml_empty_arrays(yaml_str):
+    yaml_str = yaml_str.replace("conditions: null", "conditions: []")
+    return yaml_str.replace("storedVersions: null", "storedVersions: []")
+
+
+def validate_auth():
+    substitutions = settings.get("kustomize_substitutions", {})
+    missing = [k for k in keys if k not in substitutions]
+    if missing:
+        fail("missing kustomize_substitutions keys {} in tilt-setting.json".format(missing))
+
+
+# Build CAPZ and add feature gates
+def capz():
+    # Apply the kustomized yaml for this provider
+    yaml = str(kustomize("./config"))
+    substitutions = settings.get("kustomize_substitutions", {})
+    for substitution in substitutions:
+        value = substitutions[substitution]
+        yaml = yaml.replace("${" + substitution + "}", value)
+
+    # add feature gates if they are defined
+    if settings.get("feature_gates"):
+        yaml_dict = decode_yaml_stream(yaml)
+        append_arg_for_container_in_deployment(yaml_dict, "capz-controller-manager", "capz-system", "cluster-api-azure-controller", settings.get("feature_gates"))
+        yaml = str(encode_yaml_stream(yaml_dict))
+        yaml = fixup_yaml_empty_arrays(yaml)
+
+    k8s_yaml(blob(yaml))
+    docker_build( "gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller", ".")
+
+##############################
+# Actual work happens here
+##############################
+
+validate_auth()
+
+include_user_tilt_files()
+
+if settings.get("deploy_cert_manager"):
+    deploy_cert_manager()
+
+deploy_capi()
+
+capz()

docs/development.md

@@ -97,6 +97,45 @@ A few Makefile and scripts are offered to work with go modules:
 
 ### Using Tilt
 
+Both of the [Tilt](https://tilt.dev) setups below will get you started developing CAPZ in a local kind cluster.
+The main difference is the number of components you will build from source and the scope of the changes you'd like to make.
+If you only want to make changes in CAPZ, then follow [CAPZ instructions](#tilt-for-dev-in-capz). This will
+save you from having to build all of the images for CAPI, which can take a while. If the scope of your
+development will span both CAPZ and CAPI, then follow the [CAPI and CAPZ instructions](#tilt-for-dev-in-both-capz-and-capi).
+
+#### Tilt for dev in CAPZ
+
+If you want to develop in CAPZ and get a local development cluster working quickly, this is the path for you.
+
+From the root of the CAPZ repository and after configuring the environment variables, you can run the following to generate your `tilt-settings.json` file:
+
+```shell
+cat <<EOF > tilt-settings.json
+{
+  "kustomize_substitutions": {
+      "AZURE_SUBSCRIPTION_ID_B64": "$(echo "${AZURE_SUBSCRIPTION_ID}" | tr -d '\n' | base64 | tr -d '\n')",
+      "AZURE_TENANT_ID_B64": "$(echo "${AZURE_TENANT_ID}" | tr -d '\n' | base64 | tr -d '\n')",
+      "AZURE_CLIENT_SECRET_B64": "$(echo "${AZURE_CLIENT_SECRET}" | tr -d '\n' | base64 | tr -d '\n')",
+      "AZURE_CLIENT_ID_B64": "$(echo "${AZURE_CLIENT_ID}" | tr -d '\n' | base64 | tr -d '\n')"
+  }
+}
+EOF
+```
+
+To build a kind cluster and start tilt, just run:
+```shell
+make tilt-up
+```
+
+To tear down the kind cluster built by the command above, just run:
+```shell
+make kind-reset
+```
+
+#### Tilt for dev in both CAPZ and CAPI
+
+If you want to develop in both CAPI and CAPZ at the same time, then this is the path for you.
+
 To use [Tilt](https://tilt.dev/) for a simplified development workflow, follow the [instructions](https://cluster-api.sigs.k8s.io/developer/tilt.html) in the cluster-api repo.  The instructions will walk you through cloning the Cluster API (capi) repository and configuring Tilt to use `kind` to delpoy the cluster api managment components.
 
 > you may wish to checkout out the correct version of capi to match the [version used in capz](go.mod)

scripts/kind-with-registry.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -o errexit
+
+# desired cluster name; default is "kind"
+KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-capz}"
+
+if [[ "$(kind get clusters)" =~ .*"${KIND_CLUSTER_NAME}".* ]]; then
+  echo "cluster already exists, moving on"
+  exit 0
+fi
+
+# create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5000'
+running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
+if [ "${running}" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \
+    registry:2
+fi
+
+# create a cluster with the local registry enabled in containerd
+cat <<EOF | kind create cluster --name "${KIND_CLUSTER_NAME}" --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry:${reg_port}"]
+    endpoint = ["http://registry:${reg_port}"]
+EOF
+
+# add the registry to /etc/hosts on each node
+ip_fmt='{{.NetworkSettings.IPAddress}}'
+cmd="echo $(docker inspect -f "${ip_fmt}" "${reg_name}") registry >> /etc/hosts"
+for node in $(kind get nodes --name "${KIND_CLUSTER_NAME}"); do
+  docker exec "${node}" sh -c "${cmd}"
+  kubectl annotate node "${node}" \
+          tilt.dev/registry=localhost:${reg_port} \
+          tilt.dev/registry-from-cluster=registry:${reg_port}
+done