Skip to content

Commit a066228

Browse files
devoncrousedevoncrouse
authored
Secondary VNIC for IP/CN, compatible defaults for block volume/AD (#635)
* Secondary VNIC for IP/CN, compatible defaults for block volume/AD Signed-off-by: Devon Crouse <[email protected]> * Consistent worker image lookup and variables Signed-off-by: Devon Crouse <[email protected]> --------- Signed-off-by: Devon Crouse <[email protected]>
1 parent d7ef0b1 commit a066228

15 files changed

+175
-154
lines changed

‎docs/terraformoptions.adoc‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -799,25 +799,25 @@ node_pools = {
799799
}
800800
|{}
801801

802-
|node_pool_image_id
803-
|The OCID of custom image to use when provisioning worker nodes. When no OCID is specified, the worker nodes will use the node_pool_os and node_pool_os_version to identify an image to provision the worker nodes.
802+
|worker_image_id
803+
|The OCID of custom image to use when provisioning worker nodes. When no OCID is specified, the worker nodes will use the worker_image_os and worker_image_os_version to identify an image to provision the worker nodes.
804804
|
805805
|"none"
806806

807-
|node_pool_image_type
808-
|Whether to use a Platform, OKE or custom image. When custom is set, the node_pool_image_id *must* be specified. Using an OKE image minimizes the time it takes to provision worker nodes at runtime when compared to platform images and custom images. OKE images are optimized for use as worker node base images, with all the necessary configurations and required software. The use of OKE images reduces worker node provisioning time by more than half when compared to platform images. OKE images are provided by Oracle and built on top of platform images.
807+
|worker_image_type
808+
|Whether to use a Platform, OKE or custom image. When custom is set, the worker_image_id *must* be specified. Using an OKE image minimizes the time it takes to provision worker nodes at runtime when compared to platform images and custom images. OKE images are optimized for use as worker node base images, with all the necessary configurations and required software. The use of OKE images reduces worker node provisioning time by more than half when compared to platform images. OKE images are provided by Oracle and built on top of platform images.
809809
| "custom","oke","platform"
810810
|"oke"
811811

812-
|node_pool_os
812+
|worker_image_os
813813
|The name of the Operating System image to use to provision the worker nodes.
814814
|
815815
|Oracle Linux
816816

817-
|node_pool_os_version
817+
|worker_image_os_version
818818
|The corresponding version of the Operating System image to use to provision the worker nodes.
819819
|
820-
|7.9
820+
|8
821821

822822
|cloudinit_nodepool_common
823823
|cloud-init common for all nodepools when no specific script mentioned for nodepool in cloudinit_nodepool.

‎locals.tf‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,4 @@ locals {
1212
validate_drg_input = var.create_drg && (var.drg_id != null) ? tobool("[ERROR]: create_drg variable can not be true if drg_id is provided.]") : true
1313

1414
worker_pool_subnet_id = coalesce(var.worker_pool_subnet_id, lookup(module.network.subnet_ids, "workers", ""))
15-
worker_image_id = length(var.worker_pool_image_id) > 0 ? var.worker_pool_image_id : var.node_pool_image_id != "none" ? var.node_pool_image_id : ""
16-
worker_image_type = length(var.worker_pool_image_type) > 0 ? var.worker_pool_image_type : var.node_pool_image_type != "none" ? var.node_pool_image_type : ""
1715
}

‎main.tf‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -268,10 +268,10 @@ module "oke" {
268268
max_pods_per_node = var.max_pods_per_node
269269
node_pools = var.node_pools
270270
node_pool_name_prefix = var.node_pool_name_prefix
271-
node_pool_image_id = var.node_pool_image_id
272-
node_pool_image_type = var.node_pool_image_type
273-
node_pool_os = var.node_pool_os
274-
node_pool_os_version = var.node_pool_os_version
271+
node_pool_image_id = var.worker_image_id
272+
node_pool_image_type = var.worker_image_type
273+
node_pool_os = var.worker_image_os
274+
node_pool_os_version = var.worker_image_os_version
275275
node_pool_timezone = var.node_pool_timezone
276276
enable_pv_encryption_in_transit = var.enable_pv_encryption_in_transit
277277
use_node_pool_volume_encryption = var.use_node_pool_volume_encryption

‎modules/workerpools/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ Many parameters to a worker pool can be defined at multiple levels, taken in pri
4141
label_prefix = ""
4242
worker_pool_enabled = true
4343
worker_pool_size = 0
44-
worker_pool_image_id = "ocid1.image..." # Required here and/or on group
44+
worker_image_id = "ocid1.image..." # Required here and/or on group
4545
worker_pool_mode = "node-pool"
4646
worker_pool_shape = "VM.Standard.E4.Flex"
4747
worker_pool_ocpus = 2

‎modules/workerpools/clusternetworks.tf‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,30 @@ resource "oci_core_cluster_network" "workers" {
66
# Create an OCI Cluster Network resource for each enabled entry of the worker_pools map with that mode.
77
for_each = local.enabled_cluster_networks
88
compartment_id = each.value.compartment_id
9-
display_name = "${each.value.label_prefix}-${each.key}"
9+
display_name = each.key
1010
defined_tags = merge(local.defined_tags, contains(keys(each.value), "defined_tags") ? each.value.defined_tags : {})
1111
freeform_tags = merge(local.freeform_tags, contains(keys(each.value), "freeform_tags") ? each.value.freeform_tags : { worker_pool = each.key })
1212

1313
instance_pools {
1414
instance_configuration_id = oci_core_instance_configuration.workers[each.key].id
15-
display_name = join("-", compact([lookup(each.value, "label_prefix", var.label_prefix), each.key]))
15+
display_name = each.key
1616
size = each.value.size
1717
defined_tags = merge(coalesce(local.defined_tags, {}), contains(keys(each.value), "defined_tags") ? each.value.defined_tags : {})
1818
freeform_tags = merge(coalesce(local.freeform_tags, {}), contains(keys(each.value), "freeform_tags") ? each.value.freeform_tags : { worker_pool = each.key })
1919
}
2020

2121
placement_configuration {
22-
# Define the configured availability domain for placement, bounded to a single value
23-
# The configured AD number e.g. 2 is converted into a tenancy/compartment-specific name
24-
availability_domain = lookup(local.ad_number_to_name, (
25-
contains(keys(each.value), "placement_ads")
26-
? element(tolist(setintersection(each.value.placement_ads, local.ad_numbers)), 1)
27-
: element(local.ad_numbers, 1)
28-
), local.first_ad_name)
29-
primary_subnet_id = each.value.subnet_id
22+
availability_domain = element(each.value.availability_domains, 1)
23+
primary_subnet_id = each.value.subnet_id
24+
25+
dynamic "secondary_vnic_subnets" {
26+
for_each = lookup(each.value, "secondary_vnics", {})
27+
iterator = vnic
28+
content {
29+
display_name = vnic.key
30+
subnet_id = lookup(vnic.value, "subnet_id", each.value.subnet_id)
31+
}
32+
}
3033
}
3134

3235
lifecycle {
@@ -35,10 +38,16 @@ resource "oci_core_cluster_network" "workers" {
3538
instance_pools["display_name"], instance_pools["defined_tags"], instance_pools["freeform_tags"],
3639
placement_configuration["availability_domain"],
3740
]
41+
3842
precondition {
3943
condition = var.cni_type == "flannel"
4044
error_message = "Cluster Networks require a cluster with `cni_type = flannel`."
4145
}
46+
47+
precondition {
48+
condition = coalesce(each.value.image_id, "none") != "none"
49+
error_message = "Missing image_id for pool ${each.key}. Check provided value for image_id if image_type is 'custom', or image_os/image_os_version if image_type is 'oke' or 'platform'."
50+
}
4251
}
4352

4453
depends_on = [

‎modules/workerpools/datasources.tf‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,3 @@ data "oci_containerengine_node_pool_option" "np_options" {
1515
data "oci_containerengine_cluster_kube_config" "kube_config" {
1616
cluster_id = var.cluster_id
1717
}
18-
19-
data "oci_core_image" "worker_images" {
20-
count = length(local.enabled_worker_pool_image_ids)
21-
image_id = local.enabled_worker_pool_image_ids[count.index]
22-
}

‎modules/workerpools/instanceconfig.tf‎

Lines changed: 43 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,17 @@ resource "oci_core_instance_configuration" "workers" {
55
# Create an OCI Instance Configuration resource for each enabled entry of the worker_pools map with a mode that uses one.
66
for_each = local.enabled_instance_configs
77
compartment_id = each.value.compartment_id
8-
display_name = "${each.value.label_prefix}-${each.key}"
8+
display_name = each.key
99

1010
instance_details {
1111
instance_type = "compute"
1212

13+
14+
1315
launch_details {
14-
compartment_id = each.value.compartment_id
16+
availability_domain = element(each.value.availability_domains, 1)
17+
compartment_id = each.value.compartment_id
18+
1519
defined_tags = merge(
1620
local.defined_tags,
1721
lookup(each.value, "defined_tags", {}),
@@ -61,19 +65,47 @@ resource "oci_core_instance_configuration" "workers" {
6165
source_type = "image"
6266
}
6367

64-
is_pv_encryption_in_transit_enabled = var.enable_pv_encryption_in_transit
68+
is_pv_encryption_in_transit_enabled = each.value.pv_encryption
6569
}
6670

67-
block_volumes {
68-
attach_details {
69-
type = var.block_volume_type
70-
is_pv_encryption_in_transit_enabled = var.block_volume_type == "paravirtualized" && var.enable_pv_encryption_in_transit
71+
dynamic "block_volumes" {
72+
for_each = each.value.availability_domains
73+
iterator = ad
74+
content {
75+
attach_details {
76+
type = each.value.block_volume_type
77+
is_pv_encryption_in_transit_enabled = each.value.pv_encryption
78+
}
79+
80+
create_details {
81+
availability_domain = ad.value
82+
compartment_id = each.value.compartment_id
83+
display_name = each.key
84+
kms_key_id = var.volume_kms_key_id
85+
}
7186
}
87+
}
88+
89+
dynamic "secondary_vnics" {
90+
for_each = lookup(each.value, "secondary_vnics", {})
91+
iterator = vnic
7292

73-
create_details {
74-
display_name = "${each.value.label_prefix}-${each.key}"
75-
kms_key_id = var.volume_kms_key_id
76-
compartment_id = each.value.compartment_id
93+
content {
94+
display_name = vnic.key
95+
nic_index = lookup(vnic.value, "nic_index", null)
96+
97+
create_vnic_details {
98+
assign_private_dns_record = lookup(vnic.value, "assign_private_dns_record", null)
99+
assign_public_ip = lookup(vnic.value, "assign_public_ip", null)
100+
display_name = vnic.key
101+
defined_tags = lookup(vnic.value, "defined_tags", null)
102+
freeform_tags = lookup(vnic.value, "freeform_tags", null)
103+
hostname_label = lookup(vnic.value, "hostname_label", null)
104+
nsg_ids = lookup(vnic.value, "nsg_ids", null)
105+
private_ip = lookup(vnic.value, "private_ip", null)
106+
skip_source_dest_check = lookup(vnic.value, "skip_source_dest_check", null)
107+
subnet_id = lookup(vnic.value, "subnet_id", each.value.subnet_id)
108+
}
77109
}
78110
}
79111
}

‎modules/workerpools/instancepools.tf‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,28 @@ resource "oci_core_instance_pool" "workers" {
66
# Create an OCI Instance Pool resource for each enabled entry of the worker_pools map with that mode.
77
for_each = local.enabled_instance_pools
88
compartment_id = each.value.compartment_id
9-
display_name = "${each.value.label_prefix}-${each.key}"
9+
display_name = each.key
1010
size = each.value.size
1111
instance_configuration_id = oci_core_instance_configuration.workers[each.key].id
1212
defined_tags = merge(local.defined_tags, contains(keys(each.value), "defined_tags") ? each.value.defined_tags : {})
1313
freeform_tags = merge(local.freeform_tags, contains(keys(each.value), "freeform_tags") ? each.value.freeform_tags : { worker_pool = each.key })
1414

1515
dynamic "placement_configurations" {
16-
# Define each configured availability domain for placement, with bounds on # available
17-
# Configured AD numbers e.g. [1,2,3] are converted into tenancy/compartment-specific names
18-
iterator = ad_number
19-
for_each = (contains(keys(each.value), "placement_ads")
20-
? tolist(setintersection(each.value.placement_ads, local.ad_numbers))
21-
: local.ad_numbers
22-
)
16+
for_each = each.value.availability_domains
17+
iterator = ad
2318

2419
content {
25-
availability_domain = lookup(local.ad_number_to_name, ad_number.value, local.first_ad_name)
20+
availability_domain = ad.value
2621
primary_subnet_id = each.value.subnet_id
22+
23+
dynamic "secondary_vnic_subnets" {
24+
for_each = lookup(each.value, "secondary_vnics", {})
25+
iterator = vnic
26+
content {
27+
display_name = vnic.key
28+
subnet_id = lookup(vnic.value, "subnet_id", each.value.subnet_id)
29+
}
30+
}
2731
}
2832
}
2933

@@ -32,10 +36,16 @@ resource "oci_core_instance_pool" "workers" {
3236
display_name, defined_tags, freeform_tags,
3337
placement_configurations,
3438
]
39+
3540
precondition {
3641
condition = var.cni_type == "flannel"
3742
error_message = "Instance Pools require a cluster with `cni_type = flannel`."
3843
}
44+
45+
precondition {
46+
condition = coalesce(each.value.image_id, "none") != "none"
47+
error_message = "Missing image_id for pool ${each.key}. Check provided value for image_id if image_type is 'custom', or image_os/image_os_version if image_type is 'oke' or 'platform'."
48+
}
3949
}
4050

4151
dynamic "load_balancers" {

‎modules/workerpools/locals.tf‎

Lines changed: 39 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ locals {
88
ad_number_to_name = local.ads != null ? {
99
for ad in local.ads : parseint(substr(ad.name, -1, -1), 10) => ad.name
1010
} : { -1 : "" } # Fallback handles failure when unavailable but not required
11-
first_ad_name = local.ad_number_to_name[1]
1211

1312
k8s_version_length = length(var.kubernetes_version)
1413
k8s_version_only = substr(var.kubernetes_version, 1, local.k8s_version_length)
@@ -23,9 +22,7 @@ locals {
2322
freeform_tags = merge(coalesce(var.freeform_tags, {}), { "role" = "worker" })
2423

2524
# OKE managed node pool images
26-
node_pool_images = try(data.oci_containerengine_node_pool_option.np_options.sources, [{
27-
source_type = "IMAGE"
28-
}])
25+
node_pool_images = try(data.oci_containerengine_node_pool_option.np_options.sources, [])
2926

3027
# Parse platform/operating system information from node pool image names
3128
parsed_images = {
@@ -51,44 +48,55 @@ locals {
5148
}
5249

5350
worker_pools_default = {
54-
mode = var.mode
55-
size = var.size
56-
shape = var.shape
57-
image_id = var.image_id
58-
image_type = var.image_type
59-
os = var.os
60-
os_version = var.os_version
61-
boot_volume_size = var.boot_volume_size
62-
memory = var.memory
63-
ocpus = var.ocpus
64-
compartment_id = local.worker_compartment_id
65-
subnet_id = var.subnet_id
66-
pod_subnet_id = var.pod_subnet_id
67-
pod_nsgs = var.pod_nsg_ids
68-
worker_nsgs = var.worker_nsg_ids
69-
assign_public_ip = var.assign_public_ip
70-
label_prefix = var.label_prefix # TODO Deprecate
71-
node_labels = {}
51+
mode = var.mode
52+
size = var.size
53+
shape = var.shape
54+
image_id = var.image_id
55+
image_type = var.image_type
56+
os = var.os
57+
os_version = var.os_version
58+
boot_volume_size = var.boot_volume_size
59+
memory = var.memory
60+
ocpus = var.ocpus
61+
compartment_id = local.worker_compartment_id
62+
placement_ads = local.ad_numbers
63+
block_volume_type = var.block_volume_type
64+
pv_encryption = var.enable_pv_encryption_in_transit
65+
subnet_id = var.subnet_id
66+
pod_subnet_id = var.pod_subnet_id
67+
pod_nsgs = var.pod_nsg_ids
68+
worker_nsgs = var.worker_nsg_ids
69+
assign_public_ip = var.assign_public_ip
70+
node_labels = {}
7271
}
7372

7473
# Filter worker_pools map variable for enabled entries
75-
worker_pools_enabled = {
74+
worker_pools_enabled = { for x, y in { # Final dynamic configuration for pool requirements
75+
# Merge desired pool configuration onto defaults
7676
for k, v in var.worker_pools : k => merge(local.worker_pools_default, v) if lookup(v, "enabled", var.enabled)
77+
} : x => merge(y, {
78+
# Translate configured + available AD numbers e.g. 2 into a tenancy/compartment-specific name
79+
availability_domains = compact([for ad_number in tolist(setintersection(y.placement_ads, local.ad_numbers)) :
80+
lookup(local.ad_number_to_name, ad_number, null)
81+
])
82+
block_volume_type = y.mode == "cluster-network" ? "iscsi" : var.block_volume_type
83+
pv_encryption = var.enable_pv_encryption_in_transit && y.block_volume_type == "paravirtualized" && y.mode != "cluster-network"
84+
image_id = (y.image_type == "custom" ? y.image_id : element(tolist(setintersection([
85+
lookup(local.image_ids, y.image_type, null),
86+
length(regexall("GPU", y.shape)) > 0 ? local.image_ids.gpu : local.image_ids.nongpu,
87+
length(regexall("A1", y.shape)) > 0 ? local.image_ids.aarch64 : local.image_ids.x86_64,
88+
[for parsed_image_id, iv in local.parsed_images : parsed_image_id
89+
if length(regexall(iv.os, y.os)) > 0 && trimprefix(iv.os_version, y.os_version) != iv.os_version
90+
],
91+
]...)), 0))
92+
})
7793
}
7894

7995
worker_compartments = distinct(compact([for k, v in local.worker_pools_enabled : lookup(v, "compartment_id", "")]))
8096

8197
# Number of nodes expected from enabled worker pools
8298
expected_node_count = length(local.worker_pools_enabled) == 0 ? 0 : sum([for k, v in local.worker_pools_enabled : lookup(v, "size", 0)])
8399

84-
# Filter worker_pools map variable for entries with image_id defined, returning a distinct list
85-
enabled_worker_pool_image_ids = distinct([
86-
for v in local.worker_pools_enabled : v.image_id if contains(keys(v), "image_id")
87-
])
88-
89-
# Intermediate worker image result from data source
90-
enabled_worker_pool_images = data.oci_core_image.worker_images
91-
92100
# Filter enabled worker_pool map entries for node pools
93101
enabled_node_pools = {
94102
for k, v in local.worker_pools_enabled : k => v if lookup(v, "mode", "") == "node-pool"

0 commit comments

Comments
 (0)