Skip to content

Commit b139b0a

Browse files
robo-caphyder
robo-cap
authored and
hyder
committed
add cluster addon support
1 parent 560b741 commit b139b0a

File tree

11 files changed

+255
-0
lines changed

11 files changed

+255
-0
lines changed

‎docs/src/SUMMARY.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
- [Subnets](./guide/network_subnets.md)
1313
- [Network Security Groups](./guide/network_nsgs.md)
1414
- [Cluster](./guide/cluster.md)
15+
- [Cluster Add-ons](./guide/cluster_addons.md)
1516
- [Workers](./guide/workers.md)
1617
- [Mode](./guide/workers_mode.md)
1718
- [Node Pool](./guide/workers_mode_nodepool.md)

‎docs/src/guide/cluster_addons.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Cluster Add-ons
2+
3+
With this module to manage both essential and optional add-ons on enhanced OKE clusters.
4+
5+
This module provides the option to remove [Essential addons](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengintroducingclusteraddons.htm#contengintroducingclusteraddons__section-essential-addons) and to manage, both essential & [optional addons](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengintroducingclusteraddons.htm#contengintroducingclusteraddons__section-optional-addons).
6+
7+
Cluster add-on removal (using the `cluster_addons_to_remove` variable) requires the creation of the operator host.
8+
9+
**Note**: For the cluster autoscaler you should choose **only one** of the options:
10+
- the stand-alone cluster-autoscaler deployment, using the [extension module](./extensions_cluster_autoscaler.md)
11+
- the cluster-autoscaler add-on
12+
13+
## Example usage
14+
```javascript
15+
{{#include ../../../examples/cluster-addons/vars-cluster-addons.auto.tfvars:4:}}
16+
```
17+
18+
## Reference
19+
* [OKE Cluster Add-ons Documentation](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringclusteraddons.htm)

‎docs/src/guide/extensions_cluster_autoscaler.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
# Extensions: Standalone Cluster Autoscaler
22

3+
**Note**: For the cluster autoscaler you should choose **only one** of the options:
4+
- the stand-alone cluster-autoscaler deployment, using this extension
5+
- the [cluster-autoscaler add-on](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringclusteraddons-configurationarguments.htm#contengconfiguringclusteraddons-configurationarguments_ClusterAutoscaler), using the [addons](./cluster_addons.md).
6+
37
Deployed using the [cluster-autoscaler Helm chart](https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler) with configuration from the `worker_pools` variable.
48

59
The module is using the `oke.oraclecloud.com/cluster_autoscaler` nodepool label to facilitate the understanding of how the Kubernetes cluster auto-scaler will interact with the node:

‎examples/cluster-addons/vars-cluster-addons.auto.tfvars‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
3+
4+
cluster_addons = {
5+
"CertManager" = {
6+
remove_addon_resources_on_delete = true
7+
# The list of supported configurations for the cluster addons is here: https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringclusteraddons-configurationarguments.htm#contengconfiguringclusteraddons-configurationarguments_CertificateManager
8+
configurations = [
9+
{
10+
key = "numOfReplicas"
11+
value = "1"
12+
}
13+
]
14+
}
15+
}
16+
17+
cluster_addons_to_remove = {
18+
Flannel = {
19+
remove_k8s_resources = true
20+
}
21+
}

‎module-cluster-addons.tf‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
3+
4+
module "cluster-addons" {
5+
count = local.cluster_enabled && lower(var.cluster_type) == "enhanced" ? 1 : 0
6+
source = "./modules/cluster-addons"
7+
8+
operator_enabled = local.operator_enabled
9+
10+
cluster_addons = var.cluster_addons
11+
cluster_addons_to_remove = var.cluster_addons_to_remove
12+
13+
cluster_id = coalesce(var.cluster_id, one(module.cluster[*].cluster_id))
14+
kubernetes_version = var.kubernetes_version
15+
16+
# Bastion/operator connection
17+
ssh_private_key = sensitive(local.ssh_private_key)
18+
bastion_host = local.bastion_public_ip
19+
bastion_user = var.bastion_user
20+
operator_host = local.operator_private_ip
21+
operator_user = var.operator_user
22+
}
23+
24+
25+
# output "supported_addons" {
26+
# description = "Supported cluster addons"
27+
# value = var.output_detail ? try(one(module.cluster-addons[*].supported_addons), null) : null
28+
# }

‎modules/cluster-addons/addons.tf‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
3+
4+
data "oci_containerengine_addon_options" "k8s_addon_options" {
5+
kubernetes_version = var.kubernetes_version
6+
}
7+
8+
locals {
9+
supported_addons = [for entry in data.oci_containerengine_addon_options.k8s_addon_options.addon_options : entry.name]
10+
primary_addons = ["CertManager"]
11+
addons_defaults = {
12+
remove_addon_resources_on_delete = true
13+
configurations = []
14+
version = null
15+
}
16+
addons_with_defaults = { for addon_name, addon_value in var.cluster_addons :
17+
addon_name => merge(local.addons_defaults, addon_value)
18+
}
19+
}
20+
21+
resource "oci_containerengine_addon" "primary_addon" {
22+
for_each = { for k, v in local.addons_with_defaults : k => v if contains(local.primary_addons, k) }
23+
24+
addon_name = each.key
25+
cluster_id = var.cluster_id
26+
27+
remove_addon_resources_on_delete = lookup(each.value, "remove_addon_resources_on_delete", true)
28+
29+
dynamic "configurations" {
30+
for_each = lookup(each.value, "configurations", [])
31+
iterator = config
32+
33+
content {
34+
key = tostring(lookup(config.value, "key"))
35+
value = tostring(lookup(config.value, "value"))
36+
}
37+
}
38+
39+
version = lookup(each.value, "version", null)
40+
41+
lifecycle {
42+
43+
precondition {
44+
condition = contains(local.supported_addons, each.key)
45+
error_message = <<-EOT
46+
The addon ${each.key} is not supported.
47+
The list of supported addons is: ${join(", ", local.supported_addons)}.
48+
EOT
49+
}
50+
}
51+
}
52+
53+
resource "oci_containerengine_addon" "secondary_addon" {
54+
for_each = { for k, v in local.addons_with_defaults : k => v if !contains(local.primary_addons, k) }
55+
depends_on = [oci_containerengine_addon.primary_addon]
56+
addon_name = each.key
57+
cluster_id = var.cluster_id
58+
59+
remove_addon_resources_on_delete = lookup(each.value, "remove_addon_resources_on_delete", true)
60+
61+
dynamic "configurations" {
62+
for_each = lookup(each.value, "configurations", [])
63+
iterator = config
64+
65+
content {
66+
key = tostring(lookup(config.value, "key"))
67+
value = tostring(lookup(config.value, "value"))
68+
}
69+
}
70+
71+
version = lookup(each.value, "version", null)
72+
73+
lifecycle {
74+
75+
precondition {
76+
condition = contains(local.supported_addons, each.key)
77+
error_message = <<-EOT
78+
The addon ${each.key} is not supported.
79+
The list of supported addons is: ${join(", ", local.supported_addons)}.
80+
EOT
81+
}
82+
}
83+
}

‎modules/cluster-addons/delete_addons.tf‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
3+
4+
locals {
5+
remove_addon_command = "oci ce cluster disable-addon --addon-name %s --cluster-id %s --is-remove-existing-add-on %t --force"
6+
remove_addons_defaults = {
7+
custom_commands = []
8+
remove_k8s_resources = true
9+
}
10+
remove_addons_with_defaults = { for addon_name, addon_value in var.cluster_addons_to_remove :
11+
addon_name => merge(local.remove_addons_defaults, addon_value)
12+
}
13+
}
14+
15+
resource "null_resource" "remove_addons" {
16+
for_each = var.operator_enabled ? local.remove_addons_with_defaults : {}
17+
depends_on = [oci_containerengine_addon.primary_addon, oci_containerengine_addon.secondary_addon]
18+
19+
connection {
20+
bastion_host = var.bastion_host
21+
bastion_user = var.bastion_user
22+
bastion_private_key = var.ssh_private_key
23+
host = var.operator_host
24+
user = var.operator_user
25+
private_key = var.ssh_private_key
26+
timeout = "40m"
27+
type = "ssh"
28+
}
29+
30+
provisioner "remote-exec" {
31+
inline = concat(
32+
[
33+
"echo 'Removing ${each.key} addon'",
34+
format(local.remove_addon_command, each.key, var.cluster_id, lookup(each.value, "remove_k8s_resources"))
35+
],
36+
lookup(each.value, "custom_commands")
37+
)
38+
}
39+
40+
lifecycle {
41+
precondition {
42+
condition = contains(local.supported_addons, each.key)
43+
error_message = <<-EOT
44+
The addon ${each.key} is not supported.
45+
The list of supported addons is: ${join(", ", local.supported_addons)}.
46+
EOT
47+
}
48+
}
49+
}

‎modules/cluster-addons/outputs.tf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
3+
4+
output "supported_addons" {
5+
value = data.oci_containerengine_addon_options.k8s_addon_options.addon_options
6+
}

‎modules/cluster-addons/variables.tf‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
3+
4+
# General variables
5+
variable "cluster_id" { type = string }
6+
variable "cluster_addons" { type = any }
7+
variable "cluster_addons_to_remove" { type = any }
8+
variable "kubernetes_version" { type = string }
9+
10+
# Variables required to access the operator host
11+
variable "bastion_host" { type = string }
12+
variable "bastion_user" { type = string }
13+
variable "operator_enabled" { type = bool }
14+
variable "operator_host" { type = string }
15+
variable "operator_user" { type = string }
16+
variable "ssh_private_key" { type = string }
17+

‎modules/cluster-addons/versions.tf‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates.
2+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl
3+
4+
terraform {
5+
required_version = ">= 1.2.0"
6+
7+
required_providers {
8+
oci = {
9+
source = "oracle/oci"
10+
version = ">= 4.119.0"
11+
}
12+
}
13+
}

0 commit comments

Comments
 (0)