feat!: Added support for multi-project GKE Hub registration (#840)

* Added support for multi-project GKE Hub registration

Added an optional variable HUB_PROJECT_ID which will allow you to specific a seperate
GCP project for the GKE HUB than the project the cluster is deployed to.

This included updating the 3 examples the leveraged the hub module.

Issue: #837

* Adding service idenity resource for multi project deployments

Ensuring that the Hub default Service Account exists
when adding a cluster from outside the hub project

Issue: #837

* Fixing bharathkkb comments

Fixes #837

* Fix linting issue

* Removed Google project data souce

Issue: #837

* Adding upgrade documentation for this change.

* Lint updates on readme.

Co-authored-by: James Duncan <[email protected]>
Co-authored-by: Bharath KKB <[email protected]>

Author: JamesDuncanNz

docs/upgrading_to_v14.0.md

@@ -41,3 +41,13 @@ The module now uses the new ASM [installation script](https://cloud.google.com/s
 - Supports migrations from open source Istio 1.7 or 1.8 to ASM
 
 Please see the script page for up to date details.
+
+### GKE Hub Register & Unregister behaviour has changed
+
+The [Hub submodule](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/modules/hub) now supports registering a cluster to a Hub that is in a separate project. This is via the introduction of the `hub_project_id`.
+variable. If you specify this variable, the cluster will be registered to this project and the GKE cluster will be deployed in the project specified in the `project_id` variable.
+
+To upgrade to the latest version, you will need to remove the state for the `run_destroy_command[0]` resource because, as of this release we register / unregister clusters using the `--gke-uri` option.
+
+If you run into errors during upgrade, you can remove the state for the run_destroy_command resource by running:
+`terraform state rm module.hub.module.gke_hub_registration.null_resource.run_destroy_command[0]`

examples/simple_zonal_with_hub_kubeconfig/README.md

@@ -9,7 +9,7 @@ It creates a [kind](https://kind.sigs.k8s.io/) cluster, sets current kubecontext
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| project\_id | The project ID (environ) to register the cluster in | `any` | n/a | yes |
+| project\_id | The project ID to host the cluster in | `any` | n/a | yes |
 
 ## Outputs
 

examples/simple_zonal_with_hub_kubeconfig/variables.tf

@@ -15,5 +15,5 @@
  */
 
 variable "project_id" {
-  description = "The project ID (environ) to register the cluster in"
+  description = "The project ID to host the cluster in"
 }

modules/hub/README.md

@@ -39,6 +39,7 @@ To deploy this config:
 | gcloud\_sdk\_version | The gcloud sdk version to use. Minimum required version is 293.0.0 | `string` | `"296.0.1"` | no |
 | gke\_hub\_membership\_name | Membership name that uniquely represents the cluster being registered on the Hub | `string` | `"gke-hub-membership"` | no |
 | gke\_hub\_sa\_name | Name for the GKE Hub SA stored as a secret `creds-gcp` in the `gke-connect` namespace. | `string` | `"gke-hub-sa"` | no |
+| hub\_project\_id | The project in which the GKE Hub belongs. | `string` | `""` | no |
 | labels | Comma separated labels in the format name=value to apply to cluster in the GCP Console. | `string` | `""` | no |
 | location | The location (zone or region) this cluster has been created in. | `string` | n/a | yes |
 | module\_depends\_on | List of modules or resources this module depends on. | `list` | `[]` | no |

modules/hub/main.tf

@@ -17,11 +17,14 @@
 locals {
   gke_hub_sa_key = var.use_existing_sa ? var.sa_private_key : google_service_account_key.gke_hub_key[0].private_key
 
-  is_gke_flag               = var.use_kubeconfig ? 0 : 1
+  is_gke_flag = var.use_kubeconfig ? 0 : 1
+  hub_project = var.hub_project_id == "" ? var.project_id : var.hub_project_id
+
+  cluster_uri               = "https://container.googleapis.com/projects/${var.project_id}/locations/${var.location}/clusters/${var.cluster_name}"
   create_cmd_gke_entrypoint = "${path.module}/scripts/gke_hub_registration.sh"
-  create_cmd_gke_body       = "${local.is_gke_flag} ${var.gke_hub_membership_name} ${var.location} ${var.cluster_name} ${local.gke_hub_sa_key} ${var.project_id} ${var.labels}"
+  create_cmd_gke_body       = "${local.is_gke_flag} ${var.gke_hub_membership_name} ${local.gke_hub_sa_key} ${local.cluster_uri} ${local.hub_project} ${var.labels}"
   destroy_gke_entrypoint    = "${path.module}/scripts/gke_hub_unregister.sh"
-  destroy_gke_body          = "${local.is_gke_flag} ${var.gke_hub_membership_name} ${var.location} ${var.cluster_name} ${var.project_id}"
+  destroy_gke_body          = "${local.is_gke_flag} ${var.gke_hub_membership_name} ${local.cluster_uri} ${local.hub_project}"
 }
 
 data "google_client_config" "default" {
@@ -30,17 +33,38 @@ data "google_client_config" "default" {
 resource "google_service_account" "gke_hub_sa" {
   count        = var.use_existing_sa ? 0 : 1
   account_id   = var.gke_hub_sa_name
-  project      = var.project_id
+  project      = local.hub_project
   display_name = "Service Account for GKE Hub Registration"
 }
 
 resource "google_project_iam_member" "gke_hub_member" {
   count   = var.use_existing_sa ? 0 : 1
-  project = var.project_id
+  project = local.hub_project
   role    = "roles/gkehub.connect"
   member  = "serviceAccount:${google_service_account.gke_hub_sa[0].email}"
 }
 
+resource "google_project_iam_member" "hub_service_agent_gke" {
+  count   = var.hub_project_id == "" ? 0 : 1
+  project = var.project_id
+  role    = "roles/gkehub.serviceAgent"
+  member  = "serviceAccount:${google_project_service_identity.sa_gkehub[0].email}"
+}
+
+resource "google_project_iam_member" "hub_service_agent_hub" {
+  count   = var.hub_project_id == "" ? 0 : 1
+  project = local.hub_project
+  role    = "roles/gkehub.serviceAgent"
+  member  = "serviceAccount:${google_project_service_identity.sa_gkehub[0].email}"
+}
+
+resource "google_project_service_identity" "sa_gkehub" {
+  count    = var.hub_project_id == "" ? 0 : 1
+  provider = google-beta
+  project  = local.hub_project
+  service  = "gkehub.googleapis.com"
+}
+
 resource "google_service_account_key" "gke_hub_key" {
   count              = var.use_existing_sa ? 0 : 1
   service_account_id = google_service_account.gke_hub_sa[0].name

modules/hub/scripts/gke_hub_registration.sh

@@ -22,11 +22,10 @@ fi
 
 GKE_CLUSTER_FLAG=$1
 MEMBERSHIP_NAME=$2
-CLUSTER_LOCATION=$3
-CLUSTER_NAME=$4
-SERVICE_ACCOUNT_KEY=$5
-PROJECT_ID=$6
-LABELS=$7
+SERVICE_ACCOUNT_KEY=$3
+CLUSTER_URI=$4
+HUB_PROJECT_ID=$5
+LABELS=$6
 
 #write temp key, cleanup at exit
 tmp_file=$(mktemp)
@@ -37,18 +36,18 @@ echo "${SERVICE_ACCOUNT_KEY}" | base64 ${B64_ARG} > "$tmp_file"
 
 if [[ ${GKE_CLUSTER_FLAG} == 1 ]]; then
     echo "Registering GKE Cluster."
-    gcloud container hub memberships register "${MEMBERSHIP_NAME}" --gke-cluster="${CLUSTER_LOCATION}"/"${CLUSTER_NAME}" --service-account-key-file="${tmp_file}" --project="${PROJECT_ID}" --quiet
+    gcloud container hub memberships register "${MEMBERSHIP_NAME}" --gke-uri="${CLUSTER_URI}" --service-account-key-file="${tmp_file}" --project="${HUB_PROJECT_ID}" --quiet
 else
     echo "Registering a non-GKE Cluster. Using current-context to register Hub membership."
     #Get the kubeconfig
     CONTEXT=$(kubectl config current-context)
-    gcloud container hub memberships register "${MEMBERSHIP_NAME}" --context="${CONTEXT}" --service-account-key-file="${tmp_file}" --project="${PROJECT_ID}" --quiet
+    gcloud container hub memberships register "${MEMBERSHIP_NAME}" --context="${CONTEXT}" --service-account-key-file="${tmp_file}" --project="${HUB_PROJECT_ID}" --quiet
 fi
 
 
 # Add labels to the registered cluster
 if [ -z ${LABELS+x} ]; then
     echo "No hub labels to apply."
 else
-    gcloud container hub memberships update "${MEMBERSHIP_NAME}" --update-labels "$LABELS" --project="${PROJECT_ID}"
+    gcloud container hub memberships update "${MEMBERSHIP_NAME}" --update-labels "$LABELS" --project="${HUB_PROJECT_ID}"
 fi

modules/hub/scripts/gke_hub_unregister.sh

@@ -15,25 +15,22 @@
 
 set -e
 
-if [ "$#" -lt 5 ]; then
+if [ "$#" -lt 4 ]; then
     >&2 echo "Not all expected arguments set."
     exit 1
 fi
 
 GKE_CLUSTER_FLAG=$1
 MEMBERSHIP_NAME=$2
-CLUSTER_LOCATION=$3
-CLUSTER_NAME=$4
-PROJECT_ID=$5
-
-
+CLUSTER_URI=$3
+HUB_PROJECT_ID=$4
 
 if [[ ${GKE_CLUSTER_FLAG} == 1 ]]; then
     echo "Un-Registering GKE Cluster."
-    gcloud container hub memberships unregister "${MEMBERSHIP_NAME}" --gke-cluster="${CLUSTER_LOCATION}"/"${CLUSTER_NAME}" --project "${PROJECT_ID}"
+    gcloud container hub memberships unregister "${MEMBERSHIP_NAME}" --gke-uri="${CLUSTER_URI}" --project "${HUB_PROJECT_ID}"
 else
     echo "Un-Registering a non-GKE Cluster. Using current-context to unregister Hub membership."
     #Get Current context
     CONTEXT=$(kubectl config current-context)
-    gcloud container hub memberships unregister "${MEMBERSHIP_NAME}" --context="${CONTEXT}" --project="${PROJECT_ID}"
+    gcloud container hub memberships unregister "${MEMBERSHIP_NAME}" --context="${CONTEXT}" --project="${HUB_PROJECT_ID}"
 fi

modules/hub/variables.tf

@@ -29,6 +29,12 @@ variable "project_id" {
   type        = string
 }
 
+variable "hub_project_id" {
+  description = "The project in which the GKE Hub belongs."
+  type        = string
+  default     = ""
+}
+
 variable "location" {
   description = "The location (zone or region) this cluster has been created in."
   type        = string