feat: add sub-module to enable kube-audit in base-ocp clusters (#619)

Author: Aashiq-J

README.md

@@ -26,6 +26,7 @@ Optionally, the module supports advanced security group management for the worke
 * [terraform-ibm-base-ocp-vpc](#terraform-ibm-base-ocp-vpc)
 * [Submodules](./modules)
     * [fscloud](./modules/fscloud)
+    * [kube-audit](./modules/kube-audit)
 * [Examples](./examples)
     * [2 MZR clusters in same VPC example](./examples/multiple_mzr_clusters)
     * [Advanced example (mzr, auto-scale, kms, taints)](./examples/advanced)

examples/advanced/README.md

@@ -10,3 +10,6 @@ The following resources are provisioned by this example:
 - A multi-zone (3 zone) KMS encrypted OCP VPC cluster, with worker pools in each zone.
 - Auto scaling enabled for the default worker pool.
 - Taints against the workers in zone-2 and zone-3.
+- Enable Kubernetes API server audit logs.
+- A Cloud logs instance
+- Logs agent to send logs to the cloud logs.

examples/advanced/main.tf

@@ -184,3 +184,85 @@ data "ibm_container_cluster_config" "cluster_config" {
   resource_group_id = module.ocp_base.resource_group_id
   config_dir        = "${path.module}/../../kubeconfig"
 }
+
+########################################################################################################################
+# Kube Audit
+########################################################################################################################
+
+module "kube_audit" {
+  depends_on                = [module.ocp_base] # Wait for the cluster to completely deploy.
+  source                    = "../../modules/kube-audit"
+  cluster_id                = module.ocp_base.cluster_id
+  cluster_resource_group_id = module.resource_group.resource_group_id
+  audit_log_policy          = "WriteRequestBodies"
+  region                    = var.region
+  ibmcloud_api_key          = var.ibmcloud_api_key
+}
+
+
+########################################################################################################################
+# Observability (Instance + Agents)
+########################################################################################################################
+
+locals {
+  logs_agent_namespace = "ibm-observe"
+  logs_agent_name      = "logs-agent"
+}
+
+module "observability_instances" {
+  source                     = "terraform-ibm-modules/observability-instances/ibm"
+  version                    = "3.4.3"
+  resource_group_id          = module.resource_group.resource_group_id
+  region                     = var.region
+  cloud_logs_plan            = "standard"
+  cloud_monitoring_plan      = "graduated-tier"
+  enable_platform_metrics    = false
+  cloud_logs_instance_name   = "${var.prefix}-cloud-logs"
+  cloud_monitoring_provision = false
+}
+
+module "trusted_profile" {
+  source                      = "terraform-ibm-modules/trusted-profile/ibm"
+  version                     = "2.0.1"
+  trusted_profile_name        = "${var.prefix}-profile"
+  trusted_profile_description = "Logs agent Trusted Profile"
+  # As a `Sender`, you can send logs to your IBM Cloud Logs service instance - but not query or tail logs. This role is meant to be used by agents and routers sending logs.
+  trusted_profile_policies = [{
+    roles = ["Sender"]
+    resources = [{
+      service = "logs"
+    }]
+  }]
+  # Set up fine-grained authorization for `logs-agent` running in ROKS cluster in `ibm-observe` namespace.
+  trusted_profile_links = [{
+    cr_type = "ROKS_SA"
+    links = [{
+      crn       = module.ocp_base.cluster_crn
+      namespace = local.logs_agent_namespace
+      name      = local.logs_agent_name
+    }]
+    }
+  ]
+}
+
+module "observability_agents" {
+  depends_on                = [module.kube_audit]
+  source                    = "terraform-ibm-modules/observability-agents/ibm"
+  version                   = "2.6.0"
+  cluster_id                = module.ocp_base.cluster_id
+  cluster_resource_group_id = module.resource_group.resource_group_id
+  # Cloud Logs agent
+  logs_agent_trusted_profile  = module.trusted_profile.trusted_profile.id
+  logs_agent_namespace        = local.logs_agent_namespace
+  logs_agent_name             = local.logs_agent_name
+  cloud_logs_ingress_endpoint = module.observability_instances.cloud_logs_ingress_private_endpoint
+  cloud_logs_ingress_port     = 3443
+  # example of how to add additional metadata to the logs agents
+  logs_agent_additional_metadata = [{
+    key   = "cluster_id"
+    value = module.ocp_base.cluster_id
+  }]
+  # example of how to add only kube-audit log source path
+  logs_agent_selected_log_source_paths = ["/var/log/audit/*.log"]
+  cloud_monitoring_enabled             = false
+}

examples/advanced/provider.tf

@@ -12,3 +12,17 @@ provider "kubernetes" {
   token                  = data.ibm_container_cluster_config.cluster_config.token
   cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate
 }
+
+provider "helm" {
+  kubernetes {
+    host                   = data.ibm_container_cluster_config.cluster_config.host
+    token                  = data.ibm_container_cluster_config.cluster_config.token
+    cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate
+  }
+  # IBM Cloud credentials are required to authenticate to the helm repo
+  registry {
+    url      = "oci://icr.io/ibm/observe/logs-agent-helm"
+    username = "iamapikey"
+    password = var.ibmcloud_api_key
+  }
+}

examples/advanced/version.tf

@@ -12,5 +12,9 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = ">= 2.16.1"
     }
+    helm = {
+      source  = "hashicorp/helm"
+      version = ">= 2.15.0"
+    }
   }
 }

modules/kube-audit/README.md

@@ -0,0 +1,100 @@
+# Kubernetes API server audit logs
+
+To monitor user-initiated, Kubernetes administrative activity made within your cluster, you can collect and forward audit events that are passed through your Kubernetes API server to IBM Cloud Logs or an external server.
+
+This sub-module helps you to create a Kubernetes audit system by using the provided image and deployment in your existing cluster. [Learn more](https://cloud.ibm.com/docs/openshift?topic=openshift-health-audit)
+
+**Important**: The sub-module uses the `icr.io/ibm/ibmcloud-kube-audit-to-ibm-cloud-logs` image to forward logs to IBM Cloud Logs. This image is for demonstration purposes only. For a production solution, configure and maintain your own log forwarding image.
+
+### Usage
+
+```hcl
+# ############################################################################
+# Init cluster config for helm
+# ############################################################################
+
+data "ibm_container_cluster_config" "cluster_config" {
+  # update this value with the Id of the cluster where these agents will be provisioned
+  cluster_name_id = "cluster_id"
+}
+
+# ############################################################################
+# Config providers
+# ############################################################################
+
+provider "ibm" {
+  # update this value with your IBM Cloud API key value
+  ibmcloud_api_key = "XXXXXXXXXXXXXXXXX" #pragma: allowlist secret
+}
+
+provider "helm" {
+  kubernetes {
+    host                   = data.ibm_container_cluster_config.cluster_config.host
+    token                  = data.ibm_container_cluster_config.cluster_config.token
+    cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate
+  }
+}
+
+provider "kubernetes" {
+  host                   = data.ibm_container_cluster_config.cluster_config.host
+  token                  = data.ibm_container_cluster_config.cluster_config.token
+  cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate
+}
+
+module "kube_audit" {
+  source                    = "terraform-ibm-modules/terraform-ibm-base-ocp-vpc/ibm//modules/kube-audit"
+  version                   = "X.X.X" # Replace "X.X.X" with a release version to lock into a specific release
+  cluster_id                = "cluster_id"
+  cluster_resource_group_id = "resource group id"
+  region                    = "us-south"
+}
+```
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+### Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.9.0 |
+| <a name="requirement_helm"></a> [helm](#requirement\_helm) | >= 2.15.0, <3.0.0 |
+| <a name="requirement_ibm"></a> [ibm](#requirement\_ibm) | >= 1.70.0, <2.0.0 |
+| <a name="requirement_null"></a> [null](#requirement\_null) | >= 3.2.1, < 4.0.0 |
+| <a name="requirement_time"></a> [time](#requirement\_time) | >= 0.9.1, < 1.0.0 |
+
+### Modules
+
+No modules.
+
+### Resources
+
+| Name | Type |
+|------|------|
+| [helm_release.kube_audit](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [null_resource.set_audit_log_policy](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
+| [null_resource.set_audit_webhook](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
+| [time_sleep.wait_for_kube_audit](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
+| [ibm_container_cluster_config.cluster_config](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/container_cluster_config) | data source |
+| [ibm_container_vpc_cluster.cluster](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/container_vpc_cluster) | data source |
+
+### Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_audit_deployment_name"></a> [audit\_deployment\_name](#input\_audit\_deployment\_name) | The name of log collection deployement and service. | `string` | `"ibmcloud-kube-audit"` | no |
+| <a name="input_audit_log_policy"></a> [audit\_log\_policy](#input\_audit\_log\_policy) | Specify the amount of information that is logged to the API server audit logs by choosing the audit log policy profile to use. Supported values are `default` and `WriteRequestBodies`. | `string` | `"default"` | no |
+| <a name="input_audit_namespace"></a> [audit\_namespace](#input\_audit\_namespace) | The name of the namespace where log collection service and a deployment will be created. | `string` | `"ibm-kube-audit"` | no |
+| <a name="input_audit_webhook_listener_image"></a> [audit\_webhook\_listener\_image](#input\_audit\_webhook\_listener\_image) | The audit webhook listener image reference in the format of `[registry-url]/[namespace]/[image]`.The sub-module uses the `icr.io/ibm/ibmcloud-kube-audit-to-ibm-cloud-logs` image to forward logs to IBM Cloud Logs. This image is for demonstration purposes only. For a production solution, configure and maintain your own log forwarding image. | `string` | `"icr.io/ibm/ibmcloud-kube-audit-to-ibm-cloud-logs"` | no |
+| <a name="input_audit_webhook_listener_image_version"></a> [audit\_webhook\_listener\_image\_version](#input\_audit\_webhook\_listener\_image\_version) | The tag or digest for the audit webhook listener image to deploy. If changing the value, ensure it is compatible with `audit_webhook_listener_image`. | `string` | `"deaabcb8225e800385413ba420cf3f819d3b0671@sha256:acf123f4dba63534cbc104c6886abedff9d25a22a34ab7b549ede988ed6e7144"` | no |
+| <a name="input_cluster_config_endpoint_type"></a> [cluster\_config\_endpoint\_type](#input\_cluster\_config\_endpoint\_type) | Specify which type of endpoint to use for for cluster config access: 'default', 'private', 'vpe', 'link'. 'default' value will use the default endpoint of the cluster. | `string` | `"default"` | no |
+| <a name="input_cluster_id"></a> [cluster\_id](#input\_cluster\_id) | The ID of the cluster to deploy the log collection service in. | `string` | n/a | yes |
+| <a name="input_cluster_resource_group_id"></a> [cluster\_resource\_group\_id](#input\_cluster\_resource\_group\_id) | The resource group ID of the cluster. | `string` | n/a | yes |
+| <a name="input_ibmcloud_api_key"></a> [ibmcloud\_api\_key](#input\_ibmcloud\_api\_key) | The IBM Cloud api key to generate an IAM token. | `string` | n/a | yes |
+| <a name="input_region"></a> [region](#input\_region) | The IBM Cloud region where the cluster is provisioned. | `string` | n/a | yes |
+| <a name="input_use_private_endpoint"></a> [use\_private\_endpoint](#input\_use\_private\_endpoint) | Set this to true to force all api calls to use the IBM Cloud private endpoints. | `bool` | `false` | no |
+| <a name="input_wait_till"></a> [wait\_till](#input\_wait\_till) | To avoid long wait times when you run your Terraform code, you can specify the stage when you want Terraform to mark the cluster resource creation as completed. Depending on what stage you choose, the cluster creation might not be fully completed and continues to run in the background. However, your Terraform code can continue to run without waiting for the cluster to be fully created. Supported args are `MasterNodeReady`, `OneWorkerNodeReady`, `IngressReady` and `Normal` | `string` | `"IngressReady"` | no |
+| <a name="input_wait_till_timeout"></a> [wait\_till\_timeout](#input\_wait\_till\_timeout) | Timeout for wait\_till in minutes. | `number` | `90` | no |
+
+### Outputs
+
+No outputs.
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/kube-audit/helm-charts/kube-audit/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

modules/kube-audit/helm-charts/kube-audit/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: kube-audit
+description: A Helm chart for kube-audit
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.0.0"

modules/kube-audit/helm-charts/kube-audit/templates/deployment.yaml

@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ .Values.metadata.name }}"
+  namespace: "{{ .Values.metadata.namespace }}"
+  labels:
+    app: "{{ .Values.metadata.name }}"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: "{{ .Values.metadata.name }}"
+  template:
+    metadata:
+      labels:
+        app: "{{ .Values.metadata.name }}"
+    spec:
+      containers:
+        - name: "{{ .Values.metadata.name }}"
+          image: "{{ .Values.image.name }}:{{ .Values.image.tag }}"
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 3000
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsNonRoot: true
+            capabilities:
+              drop:
+                - ALL
+            seccompProfile:
+              type: RuntimeDefault

modules/kube-audit/helm-charts/kube-audit/templates/namespace.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: "{{ .Values.metadata.namespace }}"
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/enforce-version: latest
+    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/audit-version: latest
+    pod-security.kubernetes.io/warn: restricted
+    pod-security.kubernetes.io/warn-version: latest
+    security.openshift.io/scc.podSecurityLabelSync: "false"