added outputs for instance_principal dynamic group, enabled update_dynamic_group.sh, bug fix in update_dynamic_group.sh, updated documentation

Signed-off-by: Ali Mukadam <[email protected]>

Author: hyder

docs/configuration.adoc

@@ -29,6 +29,7 @@
 :uri-oci: https://cloud.oracle.com/cloud-infrastructure
 :uri-oci-documentation: https://docs.cloud.oracle.com/iaas/Content/home.htm
 :uri-oci-instance-principal: https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm
+:uri-oci-kms: https://docs.cloud.oracle.com/iaas/Content/KeyManagement/Concepts/keyoverview.htm
 :uri-oci-loadbalancer-annotations: https://github.com/oracle/oci-cloud-controller-manager/blob/master/docs/load-balancer-annotations.md
 :uri-oci-region: https://docs.cloud.oracle.com/iaas/Content/General/Concepts/regions.htm
 :uri-oci-ocir: https://docs.cloud.oracle.com/iaas/Content/Registry/Concepts/registryoverview.htm
@@ -143,7 +144,7 @@ If you need to change the default VCN's CIDR, note the following:
 
 The bastion host parameters concern whether you want to enable the bastion. 1 parameter to keep in mind here is the enable_instance_principal. Be aware that if this is enabled, it gives API access to the bastion host without authentication.
 
-Read more about {uri-oci-instance-principal}[instance_principal].
+Read {uri-instructions}#enabling-instance_principal-on-the-bastion-host[more] about {uri-oci-instance-principal}[instance_principal].
 
 {uri-terraform-options}#bastion-host[Reference]
 
@@ -213,3 +214,16 @@ The calico parameters control the installation of {uri-calico}[Calico] for {uri-
 The Kubernetes Metrics Server parameter controls the installation of {uri-metrics-server}[Kubernetes Metrics Server]. *Required* for {uri-kubernetes-hpa}[Horizontal Pod Autoscaling].
 
 {uri-terraform-options}#kubernetes-metrics-server[Reference]
+
+=== Configure KMS Integration parameters
+
+The KMS integration parameters control whether {uri-oci-kms}[OCI Key Management Service] will be used for encrypting Kubernetes secrets. Additionally, the bastion host must be enabled as well as instance_principal on the bastion.
+
+----
+create_bastion = true
+enable_instance_principal = true
+use_encryption = true
+existing_key_id = <existing_key_ocid>
+----
+
+{uri-terraform-options}#kms-integration[Reference]

docs/instructions.adoc

@@ -23,6 +23,8 @@
 :uri-networks-subnets-cidr: https://erikberg.com/notes/networks.html
 :uri-oci: https://cloud.oracle.com/cloud-infrastructure
 :uri-oci-documentation: https://docs.cloud.oracle.com/iaas/Content/home.htm
+:uri-oci-instance-principal: https://docs.cloud.oracle.com/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm
+:uri-oci-kms: https://docs.cloud.oracle.com/iaas/Content/KeyManagement/Concepts/keyoverview.htm
 :uri-oci-ocir: https://docs.cloud.oracle.com/iaas/Content/Registry/Concepts/registryoverview.htm
 :uri-oke: https://docs.cloud.oracle.com/iaas/Content/ContEng/Concepts/contengoverview.htm
 :uri-oracle: https://www.oracle.com
@@ -40,9 +42,13 @@
 :uri-k8s-dashboard: http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/
 
 . link:#assumptions[Assumptions]
+. link:#kms-integration[KMS Integration]
 . link:#creating-the-oke-cluster[Creating the OKE Cluster]
 . link:#adding-the-bastion-host[Adding the bastion host]
 . link:#using-the-bastion-host[Using the bastion host]
+.. link:#enabling-instance_principal-on-the-bastion-host[Enabling instance_principal on the bastion host]
+.. link:#disabling-instance_principal-on-the-bastion-host[Disabling instance_principal on the bastion host]
+.. link:#recommendations-for-using-instance_principal[Recommendations for using instance_principal]
 . link:#interacting-with-the-oke-cluster-locally[Interacting with the OKE Cluster locally]
 . link:#creating-an-auth-token-for-ocir[Creating an auth token for OCIR]
 . link:#installing-helm[Installing helm]
@@ -60,6 +66,18 @@ This section assumes you have completed the following:
 . all the {uri-prereqs}[pre-requisites]
 . all the required {uri-configuration}[configuration]
 
+=== KMS Integration
+
+If you wish to use {uri-oci-kms}[OCI KMS] to encrypt Kubernetes secrets, the following is required:
+
+* the Terraform user must have the following rights
+** manage dynamic groups
+** manage policies in root tenancy
+* link:#adding-the-bastion-host[bastion must be enabled]
+* link:#enabling-instance_principal-on-the-bastion-host[bastion instance_principal must be enabled]
+* use_encryption must be set to _true_
+* existing_key_id must be provided
+
 === Creating the OKE Cluster
 
 Initialize a working directory containing Terraform configuration files:
@@ -116,6 +134,51 @@ A utility script is also generated that contains the command to ssh to the basti
 scripts/tesseract.sh
 ----
 
+==== Enabling instance_principal on the bastion host
+{uri-oci-instance-principal}[instance_principal] is an IAM service feature that enables instances to be authorized actors (or principals) to perform actions on service resources. Each compute instance has its own identity, and it authenticates using the certificates that are added to it. These certificates are automatically created, assigned to instances and rotated, preventing the need for you to distribute credentials to your hosts and rotate them.
+
+Any user who has access to the instance (who can SSH to the instance), automatically inherits the privileges granted to the instance. Before you enable this feature, ensure that you know who can access it, and that they should be authorized with the permissions you are granting to the instance.
+
+By default, this feature is *_disabled_*. However, it is *_required_* at the time of cluster creation *_if_* you wish to enable link:#kms-integration[KMS Integration].
+
+When you enable this feature, by default, the bastion has privileges to all resources in the compartment. If you are enabling it for link:#kms-integration[KMS Integration], the bastion host will also have rights to create policies in the root tenancy. 
+
+You can also turn on and off the feature at any time without impact on the bastion or the cluster.
+
+To enable, set enable_instance_principal to true:
+
+----
+enable_instance_principal = "true"
+----
+
+and verify:
+
+----
+oci network vcn list --compartment-id <compartment-ocid>
+----
+
+==== Disabling instance_principal on the bastion host
+
+. Set enable_instance_principal to false in terraform.tfvars
+
++
+----
+enable_instance_principal = false
+----
+
+. Run terraform apply again:
+
++
+----
+terraform apply
+----
+
+==== Recommendations for using instance_principal
+
+. Do not enable instance_principal if you are not using link:#kms-integration[KMS Integration]
+. Enable instance_principal *_if and only if_* you are using link:#kms-integration[KMS Integration]
+. Disable instance_principal once the cluster is created
+
 === Interacting with the OKE Cluster locally
 
 kubectl installed in bastion host by default and the kubeconfig file is set in the default location (~/.kube/config) so you don't need to set the KUBECONFIG environment variable every time you log in to the bastion. An alias "*k*" will be created for kubectl on the bastion host. 

docs/prerequisites.adoc

@@ -19,6 +19,9 @@
 :uri-oci-documentation: https://docs.cloud.oracle.com/iaas/Content/home.htm
 :uri-oci-keys: https://docs.cloud.oracle.com/iaas/Content/API/Concepts/apisigningkey.htm#two
 :uri-oci-keys-upload: https://docs.cloud.oracle.com/iaas/Content/API/Concepts/apisigningkey.htm#two
+:uri-oci-kms: https://docs.cloud.oracle.com/iaas/Content/KeyManagement/Concepts/keyoverview.htm
+:uri-oci-managing-keys: https://docs.cloud.oracle.com/iaas/Content/KeyManagement/Tasks/managingkeys.htm
+:uri-oci-managing-vaults: https://docs.cloud.oracle.com/iaas/Content/KeyManagement/Tasks/managingvaults.htm
 :uri-oci-oke-policy: https://docs.cloud.oracle.com/iaas/Content/ContEng/Concepts/contengpolicyconfig.htm#PolicyPrerequisitesService
 
 :uri-terraform: https://www.terraform.io
@@ -28,13 +31,21 @@ This section will guide you through the pre-requisites before you can use this p
 
 You can proceed to {uri-instructions}[creating the cluster] if you have already done these.
 
+. link:#identity-and-access-management-rights[Identity and Access Management Rights]
 . link:#install-terraform[Install Terraform]
 . link:#generate-api-keys[Generate API Keys]
 . link:#upload-your-api-keys[Upload API Keys]
 . link:#create-an-oci-compartment[Create an OCI Compartment]
 . link:#obtain-the-necessary-ocids[Obtain the necessary OCIDs]
 . link:#configure-oci-policy-for-oke[Configure OCI Policy for OKE]
 
+== Identity and Access Management Rights
+
+The Terraform user must have the rights to:
+
+. manage dynamic groups
+. manage policies in root tenancy
+
 == Install Terraform
 
 Start by installing Terraform and configuring your path.
@@ -108,6 +119,8 @@ To obtain the compartment OCID:
 2. Click on your Compartment
 3. Locate OCID on the page and click on 'Copy'
 
+If you wish to encrypt Kubernetes secrets with a key from {uri-oci-kms}[OCI KMS], you also need to create {uri-oci-managing-vaults}[a vault] and {uri-oci-managing-keys}[a key] and obtain the key id.
+
 == Configure OCI Policy for OKE
 
 Follow the documentation for {uri-oci-oke-policy}[to create the necessary OKE policy].

docs/terraformoptions.adoc

@@ -17,6 +17,7 @@
 :uri-kubernetes-hpa: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/
 :uri-metrics-server: https://github.com/kubernetes-incubator/metrics-server
 :uri-oci-images: https://docs.cloud.oracle.com/iaas/images/
+:uri-oci-kms: https://docs.cloud.oracle.com/iaas/Content/KeyManagement/Concepts/keyoverview.htm
 :uri-oci-loadbalancer-annotations: https://github.com/oracle/oci-cloud-controller-manager/blob/master/docs/load-balancer-annotations.md
 :uri-oci-region: https://docs.cloud.oracle.com/iaas/Content/General/Concepts/regions.htm
 :uri-terraform-cidrsubnet: https://www.terraform.io/docs/configuration/functions/cidrsubnet.html
@@ -495,3 +496,23 @@ Refer to {uri-topology}[topology] for more thorough examples.
 |Whether to install {uri-metrics-server}[Kubernetes Metrics Server]. *Required* for {uri-kubernetes-hpa}[Horizontal Pod Autoscaling].
 |true/false
 |false
+|===
+
+== KMS integration
+
+[stripes=odd,cols="1d,4d,3a,3a", options=header,width="100%"] 
+|===
+|Parameter
+|Description
+|Values
+|Default
+
+|use_encryption
+|Whether to use {uri-oci-kms}[OCI KMS] to encrypt secrets.
+|true/false
+|false
+
+|existing_key_id
+|ocid of existing KMS key
+|
+|

main.tf

@@ -36,6 +36,8 @@ module "policies" {
 
   bastion = local.oke_bastion
 
+  dynamic_group = module.base.group_name
+
   oke_kms = local.oke_kms
 
   cluster_id = module.oke.cluster_id

modules/base/bastion/iam.tf

@@ -22,20 +22,20 @@ data "oci_identity_compartments" "compartments_name" {
   }
 }
 
-resource "oci_identity_dynamic_group" "instance_principal" {
+resource "oci_identity_dynamic_group" "bastion_instance_principal" {
   provider       = "oci.home"
   compartment_id = var.oci_base_identity.tenancy_ocid
   description    = "dynamic group to allow instances to call services for 1 bastion"
   matching_rule  = "ALL {instance.id = '${join(",", data.oci_core_instance.bastion.*.id)}'}"
-  name           = "${var.oci_bastion_general.label_prefix}-instance_principal"
+  name           = "${var.oci_bastion_general.label_prefix}-bastion_instance_principal"
   count          = var.oci_bastion.enable_instance_principal == true ? 1 : 0
 }
 
-resource "oci_identity_policy" "instance_principal" {
+resource "oci_identity_policy" "bastion_instance_principal" {
   provider       = "oci.home"
   compartment_id = var.oci_base_identity.compartment_ocid
-  description    = "dynamic group to allow instances to call services"
-  name           = "${var.oci_bastion_general.label_prefix}-instance_principal"
-  statements     = ["Allow dynamic-group ${oci_identity_dynamic_group.instance_principal[0].name} to manage all-resources in compartment ${data.oci_identity_compartments.compartments_name.compartments.0.name}"]
+  description    = "policy to allow bastion host to call services"
+  name           = "${var.oci_bastion_general.label_prefix}-bastion_instance_principal"
+  statements     = ["Allow dynamic-group ${oci_identity_dynamic_group.bastion_instance_principal[0].name} to manage all-resources in compartment ${data.oci_identity_compartments.compartments_name.compartments.0.name}"]
   count          = var.oci_bastion.enable_instance_principal == true ? 1 : 0
-}
+}

modules/base/bastion/outputs.tf

@@ -4,3 +4,7 @@
 output "bastion_public_ip" {
   value = join(",", data.oci_core_vnic.bastion_vnic.*.public_ip_address)
 }
+
+output "bastion_instance_principal_group_name" {
+  value = oci_identity_dynamic_group.bastion_instance_principal[0].name
+}

modules/base/outputs.tf

@@ -11,6 +11,10 @@ output "bastion_public_ip" {
   value = module.bastion.bastion_public_ip
 }
 
+output "group_name" {
+  value = module.bastion.bastion_instance_principal_group_name
+}
+
 output "ig_route_id" {
   value = module.vcn.ig_route_id
 }

modules/policies/groups.tf

@@ -6,7 +6,6 @@ resource "oci_identity_dynamic_group" "oke-kms-cluster" {
   compartment_id = var.oci_identity.tenancy_ocid
   description    = "dynamic group to allow cluster to use kms"
   matching_rule  = local.dynamic_group_rule_all_clusters
-  # matching_rule  = local.dynamic_group_rule_this_cluster
   name  = "${var.label_prefix}-oke-kms-cluster"
   count = var.oke_kms.use_encryption == true ? 1 : 0
 
@@ -25,7 +24,7 @@ data "template_file" "update_dynamic_group_script" {
 
   depends_on = ["oci_identity_dynamic_group.oke-kms-cluster"]
 
-  count = var.oke_kms.use_encryption == true && var.bastion.create_bastion ==true && var.bastion.enable_instance_principal == true ? 1 : 0
+  count = var.oke_kms.use_encryption == true && var.bastion.create_bastion == true && var.bastion.enable_instance_principal == true ? 1 : 0
 }
 
 resource null_resource "update_dynamic_group" {
@@ -41,20 +40,20 @@ resource null_resource "update_dynamic_group" {
     user        = var.bastion.image_operating_system == "Canonical Ubuntu" ? "ubuntu" : "opc"
   }
 
-  depends_on = ["oci_identity_dynamic_group.oke-kms-cluster"]
+  depends_on = ["oci_identity_dynamic_group.oke-kms-cluster", "oci_identity_policy.bastion_instance_principal_dynamic_group"]
 
   provisioner "file" {
     content     = data.template_file.update_dynamic_group_script[0].rendered
     destination = "~/update_dynamic_group.sh"
   }
 
-  # provisioner "remote-exec" {
-  #   inline = [
-  #     "chmod +x $HOME/update_dynamic_group.sh",
-  #     "$HOME/update_dynamic_group.sh",
-  #     # "rm -f $HOME/update_dynamic_group.sh"
-  #   ]
-  # }
+  provisioner "remote-exec" {
+    inline = [
+      "chmod +x $HOME/update_dynamic_group.sh",
+      "$HOME/update_dynamic_group.sh",
+      "rm -f $HOME/update_dynamic_group.sh"
+    ]
+  }
 
-    count = var.oke_kms.use_encryption == true && var.bastion.create_bastion ==true && var.bastion.enable_instance_principal == true ? 1 : 0
+  count = var.oke_kms.use_encryption == true && var.bastion.create_bastion == true && var.bastion.enable_instance_principal == true ? 1 : 0
 }

modules/policies/policies.tf

@@ -1,6 +1,15 @@
 # Copyright 2017, 2019, Oracle Corporation and/or affiliates.  All rights reserved.
 # Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl
 
+resource "oci_identity_policy" "bastion_instance_principal_dynamic_group" {
+  provider       = "oci.home"
+  compartment_id = var.oci_identity.tenancy_ocid
+  description    = "policy to allow bastion host to manage dynamic group"
+  name           = "${var.label_prefix}-bastion-instance-principal-dynamic-group"
+  statements     = ["Allow dynamic-group ${var.dynamic_group} to use dynamic-groups in tenancy"]
+  count          = var.oke_kms.use_encryption == true ? 1 : 0
+}
+
 resource "oci_identity_policy" "oke-kms" {
   provider       = "oci.home"
   compartment_id = var.oci_identity.compartment_ocid