diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 00000000..804bce73 --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,31 @@ +# Migration advice when updating the module + +# 2.18.3 -> 2.19.0 + +## User Kustomization +The extra_kustomization-feature has been moved to a module so that multiple extra_kustomizations can be run in sequential steps. +A new variable `user_kustomizations` is now in use, which contains the previous extra_kustomize_* vars. + +### Affects +If you are using Helm charts from the `extra-manifests` folder or if you are using any of the following variables: `extra_kustomize_deployment_commands`, `extra_kustomize_parameters` or `extra_kustomize_folder`. + +### Steps + +1. Create a new variable `user_kustomizations`, see below and the kube.tf.example. + +``` +user_kustomizations = { + "1" = { + source_folder = "extra-manifests" # Place here the source-folder defined previously in `var.extra_kustomize_folder`. If `var.extra_kustomize_folder` was previously undefined, leave as "extra-manifests". + + kustomize_parameters = {} # Replace with contents of `var.extra_kustomize_parameters`. If `var.extra_kustomize_parameters` was previously undefined, remove the line or keep the default {}. + + pre_commands = "" + + post_commands = "" # Replace with contents of `var.extra_kustomize_deployment_commands`. If `var.extra_kustomize_deployment_commands` was previously undefined, remove the line or keep the default "". + + } +} +``` + +2. After placing the variables, remove the variables `extra_kustomize_deployment_commands`, `extra_kustomize_parameters` and `extra_kustomize_folder` from kube.tf. diff --git a/README.md b/README.md index ee2d3bcc..8e5c3e26 100644 --- a/README.md +++ b/README.md @@ -318,12 +318,26 @@ See the [guide on adding robot servers](docs/add-robot-server.md) If you need to install additional Helm charts or Kubernetes manifests that are not provided by default, you can easily do so by using [Kustomize](https://kustomize.io). This is done by creating one or more `extra-manifests/kustomization.yaml.tpl` files beside your `kube.tf`. -If you'd like to use a different folder name, you can configure it using the `extra_kustomize_folder` variable. By default, it is set to `extra-manifests`. This can be useful when working with multiple environments, allowing you to deploy different manifests for each one. - -These files need to be valid `Kustomization` manifests, additionally supporting terraform templating! (The templating parameters can be passed via the `extra_kustomize_parameters` variable (via a map) to the module). +These files need to be valid `Kustomization` manifests, additionally supporting terraform templating! (The templating parameters can be passed via the `user_kustomizations` variable (via a map) to the module). All files in the `extra-manifests` directory and its subdirectories including the rendered versions of the `*.yaml.tpl` will be applied to k3s with `kubectl apply -k` (which will be executed after and independently of the basic cluster configuration). +If you'd like to use a different folder name, specify the template parameters or apply the kustomizations in multiple steps due to CRD installation etc, you can override the values using `user_kustomizations`-variable, see below: + +Example of using user_kustomizations in `kube.tf`: +``` + user_kustomizations = { + "1" = { + source_folder = "extra-manifests-pre" + }, + "2" = { + source_folder = "extra-manifests" + kustomize_parameters = {key: "LowSecretKey"} + post_commands = "kubectl wait --for=condition=Available deployment --all -A --timeout=120s || true" + } + } +``` + See a working example in [examples/kustomization_user_deploy](https://github.com/kube-hetzner/terraform-hcloud-kube-hetzner/tree/master/examples/kustomization_user_deploy). _You can use the above to pass all kinds of Kubernetes YAML configs, including HelmChart and/or HelmChartConfig definitions (see the previous section if you do not know what those are in the context of k3s)._ @@ -334,20 +348,126 @@ _That said, you can also use pure Terraform and import the kube-hetzner module a
-Custom post-install actions +Adding applications with Helm, custom pre- and post-install actions After the initial bootstrapping of your Kubernetes cluster, you might want to deploy applications using the same terraform mechanism. For many scenarios it is sufficient to create a `kustomization.yaml.tpl` file (see [Adding Extras](#adding-extras)). All applied kustomizations will be applied at once by executing a single `kubectl apply -k` command. -However, some applications that e.g. provide custom CRDs (e.g. [ArgoCD](https://argoproj.github.io/cd/)) need a different deployment strategy: one has to deploy CRDs first, then wait for the deployment, before being able to install the actual application. In the ArgoCD case, not waiting for the CRD setup to finish will cause failures. Therefore, an additional mechanism is available to support these kind of deployments. Specify `extra_kustomize_deployment_commands` in your `kube.tf` file containing a series of commands to be executed, after the `Kustomization` step finished: +However, some applications that e.g. provide custom CRDs (e.g. [ArgoCD](https://argoproj.github.io/cd/)) need a different deployment strategy: one has to deploy CRDs first, then wait for the deployment, before being able to install the actual application. In the ArgoCD case, not waiting for the CRD setup to finish will cause failures. -```tf - extra_kustomize_deployment_commands = <<-EOT - kubectl -n argocd wait --for condition=established --timeout=120s crd/appprojects.argoproj.io - kubectl -n argocd wait --for condition=established --timeout=120s crd/applications.argoproj.io - kubectl apply -f /var/user_kustomize/argocd-projects.yaml - kubectl apply -f /var/user_kustomize/argocd-application-argocd.yaml +To support these scenarios, you can use the `user_kustomizations` variable in your `kube.tf`. This allows you to define multiple "sets" of kustomizations that are applied sequentially. Each set can have its own `source_folder`, `kustomize_parameters`, and `pre_commands`/`post_commands` for running scripts before or after the `kubectl apply -k`. + +### Example: external-secrets repo and Helm +You can install Helm repos and CRDs before the main Kustomization scripts by adding the Helm charts a different folder, e.g. `extra-manifests-preinstall` and specifying the folder in `user_kustomizations`-settings. + +For example, to add `external-secrets` you can first create the CRDs: +```yaml +# extra-manifests-1/eso-crd.yaml.tpl +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: external-secrets + namespace: kube-system +spec: + chart: external-secrets + repo: https://charts.external-secrets.io + targetNamespace: external-secrets + createNamespace: true +``` +```yaml +# extra-manifests-1/kustomization.yaml.tpl +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - eso-crd.yaml +``` + +Then create the objects that use the previously created CRDs: + +```yaml +# extra-manifests-2/eso-secrets.yaml.tpl +apiVersion: v1 +kind: Secret +metadata: + name: vault-creds + namespace: external-secrets +type: Opaque +data: + username: ${base64encode(eso_access_username)} + password: ${base64encode(eso_access_password)} + +--- +apiVersion: external-secrets.io/v1 +kind: SecretStore +metadata: + name: vault-sm-store + namespace: external-secrets +spec: + provider: + aws: + service: SecretsManager + region: us-east-1 + + auth: + secretRef: + accessKeyIDSecretRef: + name: vault-creds + key: username + namespace: external-secrets + secretAccessKeySecretRef: + name: vault-creds + key: password + namespace: external-secrets + +``` +```yaml +# extra-manifests-2/kustomization.yaml.tpl +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - eso-secrets.yaml +``` + +In `kube.tf`, specify the folders in `user_kustomizations`. + +``` + # kube.tf + ... + user_kustomizations = { + "1" = { + source_folder = "extra-manifests-1" + post_commands = "kubectl -n external-secrets wait --for=condition=Available deployment --all --timeout=120s || true" + }, + "2" = { + source_folder = "extra-manifests-2" + kustomize_parameters = { + eso_access_username = "..." + eso_access_password = "..." + } + }, ... - EOT + } +``` + +### Example: ArgoCD with Post-install actions + +See examples from `examples/kustomization_user_deploy/helm-chart`, place the chart-files along with Kustomization.yaml.tpl into folder `argocd`. +Then specify additional project-helms in `argocd-projects`. + +``` + user_kustomizations = { + "1" = { + source_folder = "argocd" + post_commands = <<-EOT + kubectl -n argocd wait --for condition=established --timeout=120s crd/appprojects.argoproj.io + kubectl -n argocd wait --for condition=established --timeout=120s crd/applications.argoproj.io + EOT + }, + "2" = { + source_folder = "argocd-projects" + } + } ```
diff --git a/docs/llms.md b/docs/llms.md index 963d8ff1..80e9e7bc 100644 --- a/docs/llms.md +++ b/docs/llms.md @@ -2201,33 +2201,34 @@ Locked and loaded! Let's continue the detailed exploration. **Section 2.21: Kustomize and Post-Deployment Operations** ```terraform - # Extra commands to be executed after the `kubectl apply -k` (useful for post-install actions, e.g. wait for CRD, apply additional manifests, etc.). - # extra_kustomize_deployment_commands="" + # You can add user kustomizations to be deployed in sequence by setting the `user_kustomizations` variable. + # The Kustomization "sets" are run in sequential order (by numeric key) so that you can for example install a CRD and wait for it to be deployed. + # + # Properties of each value: + # - source_folder: Sets the source folder for *.yaml.tpl and Kustomization.yaml.tpl + # - kustomize_parameters: Key-value map for passing variables into Kustomization. Applies only to the Kustomization-set in the object, but to all files defined in the source_folder of the "set". Defaults to {}. + # - pre_commands: Commands to be executed before applying the Kustomization ("kubectl apply -k"). Defaults to "". + # - post_commands: Commands to be executed after applying the Kustomization ("kubectl apply -k"). You can use it to wait for CRD deployment etc. Defaults to "". + # -- An example to wait for deployments in all namespaces: `kubectl wait --for=condition=Available deployment --all -A --timeout=120s || true` (The `|| true` is necessary to prevent the script from exiting on a timeout if you want the sequence to continue.) + # -- It is recommended to use more specific `kubectl wait` commands depending on the case, for example filtering for a certain deployment or pod. + # -- You can pass full bash-compatible scripts into the `post_commands`-variable with EOT + # + # An example: + # user_kustomizations = { + # "1" = { + # source_folder = "extra-manifests" + # kustomize_parameters = { myvar = "myvalue" } + # pre_commands = "" + # post_commands = "kubectl wait --for=condition=Available deployment --all -A --timeout=120s || true" + # } + # } ``` -* **`extra_kustomize_deployment_commands` (String or List of Strings, Optional):** - * **Purpose:** Allows you to specify shell commands that will be executed *after* the module has run its main Kustomize deployment (which applies manifests for core components like CCM, CSI, Ingress, etc., based on your selections). - * **Mechanism:** The module likely uses a `local-exec` or `remote-exec` provisioner (if commands need to run on a node) to execute these. If they are `kubectl` commands, they'd run from where Terraform is executed, using the generated kubeconfig. +* **`user_kustomizations` (Map of Objects, Optional):** + * **Purpose:** Allows you to specify Kustomization sets that are run sequentially, with each set containing its own source_folder, pre_commands, post_commands and kustomize_parameters * **Use Cases:** - * **Waiting for CRDs:** Some applications deployed via Helm or Kustomize install CustomResourceDefinitions (CRDs) first, and then CustomResources (CRs) that depend on those CRDs. There can be a race condition if the CRs are applied before the CRDs are fully registered. You could add a command here to wait for CRDs to become available (e.g., `kubectl wait --for condition=established crd/mycrd.example.com --timeout=120s`). - * Applying additional Kubernetes manifests that depend on the core setup. - * Running post-install scripts or triggering initial application setup jobs. - * **Format:** Can be a single string with commands separated by `&&` or `\n`, or a list of individual command strings. - -```terraform - # Extra values that will be passed to the `extra-manifests/kustomization.yaml.tpl` if its present. - # extra_kustomize_parameters={} -``` - -* **`extra_kustomize_parameters` (Map of Strings, Optional):** - * **Purpose:** If you are using the module's "extra manifests" feature (where you can provide your own Kustomize setup in an `extra-manifests` directory), this map allows you to pass key-value parameters into a `kustomization.yaml.tpl` template file within that directory. - * **Mechanism:** The module would process `extra-manifests/kustomization.yaml.tpl` as a template, substituting placeholders with values from this map, and then run `kustomize build` on the result. - * **Use Case:** Parameterizing your custom Kustomize deployments based on Terraform inputs or computed values from the `kube-hetzner` module (e.g., passing in the cluster name, node IPs, etc., to your custom manifests). - * **Reference:** The comment points to examples in the module's repository for how to use this feature. - -```terraform - # See working examples for extra manifests or a HelmChart in examples/kustomization_user_deploy/README.md -``` + * Some applications deployed via Helm or Kustomize install CustomResourceDefinitions (CRDs) first, and then CustomResources (CRs) that depend on those CRDs. There can be a race condition if the CRs are applied before the CRDs are fully registered. You could add a command here to wait for CRDs to become available (e.g., `kubectl wait --for condition=established crd/mycrd.example.com --timeout=120s`). + * The `user_kustomizations`-map allows you to define steps of install where e.g. the first step installs CRDs, checks for their proper existence and then second step that install further CRs. * **Documentation Pointer:** This directs users to example usage of the "extra manifests" feature, which is crucial for extending the module's capabilities with custom deployments. diff --git a/examples/kustomization_user_deploy/README.md b/examples/kustomization_user_deploy/README.md index c4b34dff..556ef144 100644 --- a/examples/kustomization_user_deploy/README.md +++ b/examples/kustomization_user_deploy/README.md @@ -4,6 +4,8 @@ Kube-Hetzner allows you to provide user-defined resources after the initial setu When you execute terraform apply, the manifests in the extra-manifests directory, including the rendered versions of the `*.yaml.tpl` files, will be automatically deployed to the cluster. +Note: If you would like to use a different folder, define the `user_kustomizations` in kube.tf. + ## Examples Here are some examples of common use cases for deploying additional resources: @@ -12,14 +14,18 @@ Here are some examples of common use cases for deploying additional resources: ### Deploying Simple Resources -The easiest use case is to deploy simple resources to the cluster. Since the Kustomize resources are [Terraform template](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) files, they can make use of parameters provided in the `extra_kustomize_parameters` map of the `kube.tf` file. +The easiest use case is to deploy simple resources to the cluster. Since the Kustomize resources are [Terraform template](https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file) files, they can make use of parameters provided in the `kustomize_parameters` map of the `user_kustomizations`. #### `kube.tf` ``` ... -extra_kustomize_parameters = { - my_config_key = "somestring" +user_kustomizations = { + ... + kustomize_parameters = { + my_config_key = "somestring" + } + ... } ... ``` diff --git a/examples/kustomization_user_deploy/letsencrypt/kustomization.yaml.tpl b/examples/kustomization_user_deploy/letsencrypt/kustomization.yaml.tpl index 90822221..2db76bf3 100644 --- a/examples/kustomization_user_deploy/letsencrypt/kustomization.yaml.tpl +++ b/examples/kustomization_user_deploy/letsencrypt/kustomization.yaml.tpl @@ -3,4 +3,3 @@ kind: Kustomization resources: - letsencrypt.yaml - diff --git a/kube.tf.example b/kube.tf.example index b5eeee24..1602f96b 100644 --- a/kube.tf.example +++ b/kube.tf.example @@ -961,12 +961,27 @@ module "kube-hetzner" { # More information about the registration can be found here https://rancher.com/docs/rancher/v2.6/en/cluster-provisioning/registered-clusters/ # rancher_registration_manifest_url = "https://rancher.xyz.dev/v3/import/xxxxxxxxxxxxxxxxxxYYYYYYYYYYYYYYYYYYYzzzzzzzzzzzzzzzzzzzzz.yaml" - # Extra commands to be executed after the `kubectl apply -k` (useful for post-install actions, e.g. wait for CRD, apply additional manifests, etc.). - # extra_kustomize_deployment_commands="" - - # Extra values that will be passed to the `extra-manifests/kustomization.yaml.tpl` if its present. - # extra_kustomize_parameters={} - + # You can add user kustomizations to be deployed in sequence by setting the `user_kustomizations` variable. + # The Kustomization "sets" are run in sequential order (by numeric key) so that you can for example install a CRD and wait for it to be deployed. + # + # Properties of each value: + # - source_folder: Sets the source folder for *.yaml.tpl and Kustomization.yaml.tpl + # - kustomize_parameters: Key-value map for passing variables into Kustomization. Applies only to the Kustomization-set in the object, but to all files defined in the source_folder of the "set". Defaults to {}. + # - pre_commands: Commands to be executed before applying the Kustomization ("kubectl apply -k"). Defaults to "". + # - post_commands: Commands to be executed after applying the Kustomization ("kubectl apply -k"). You can use it to wait for CRD deployment etc. Defaults to "". + # -- An example to wait for deployments in all namespaces: `kubectl wait --for=condition=Available deployment --all -A --timeout=120s || true` (The `|| true` is necessary to prevent the script from exiting on a timeout if you want the sequence to continue.) + # -- It is recommended to use more specific `kubectl wait` commands depending on the case, for example filtering for a certain deployment or pod. + # -- You can pass full bash-compatible scripts into the `post_commands`-variable with EOT + # + # An example: + # user_kustomizations = { + # "1" = { + # source_folder = "extra-manifests" + # kustomize_parameters = { myvar = "myvalue" } + # pre_commands = "" + # post_commands = "kubectl wait --for=condition=Available deployment --all -A --timeout=120s || true" + # } + # } # See working examples for extra manifests or a HelmChart in examples/kustomization_user_deploy/README.md # It is best practice to turn this off, but for backwards compatibility it is set to "true" by default. diff --git a/kustomization_user.tf b/kustomization_user.tf index bb790386..6be82e88 100644 --- a/kustomization_user.tf +++ b/kustomization_user.tf @@ -1,48 +1,19 @@ locals { - user_kustomization_templates = try(fileset(var.extra_kustomize_folder, "**/*.yaml.tpl"), toset([])) -} - -resource "null_resource" "kustomization_user" { - for_each = local.user_kustomization_templates - - connection { - user = "root" - private_key = var.ssh_private_key - agent_identity = local.ssh_agent_identity - host = local.first_control_plane_ip - port = var.ssh_port - - bastion_host = local.ssh_bastion.bastion_host - bastion_port = local.ssh_bastion.bastion_port - bastion_user = local.ssh_bastion.bastion_user - bastion_private_key = local.ssh_bastion.bastion_private_key - + processed_kustomizes = { + for key, config in var.user_kustomizations : key => merge(config, { + # kustomize_parameters, pre_commands, and post_commands may contain secrets + kustomize_parameters = sensitive(config.kustomize_parameters), + pre_commands = sensitive(config.pre_commands), + post_commands = sensitive(config.post_commands) + }) } - - provisioner "remote-exec" { - inline = [ - "mkdir -p $(dirname /var/user_kustomize/${each.key})" - ] - } - - provisioner "file" { - content = templatefile("${var.extra_kustomize_folder}/${each.key}", var.extra_kustomize_parameters) - destination = replace("/var/user_kustomize/${each.key}", ".yaml.tpl", ".yaml") - } - - triggers = { - manifest_sha1 = "${sha1(templatefile("${var.extra_kustomize_folder}/${each.key}", var.extra_kustomize_parameters))}" - } - - depends_on = [ - null_resource.kustomization - ] } -resource "null_resource" "kustomization_user_deploy" { - count = length(local.user_kustomization_templates) > 0 ? 1 : 0 +module "user_kustomizations" { - connection { + source = "./modules/user_kustomizations" + + ssh_connection = { user = "root" private_key = var.ssh_private_key agent_identity = local.ssh_agent_identity @@ -53,27 +24,11 @@ resource "null_resource" "kustomization_user_deploy" { bastion_port = local.ssh_bastion.bastion_port bastion_user = local.ssh_bastion.bastion_user bastion_private_key = local.ssh_bastion.bastion_private_key - } - # Remove templates after rendering, and apply changes. - provisioner "remote-exec" { - # Debugging: "sh -c 'for file in $(find /var/user_kustomize -type f -name \"*.yaml\" | sort -n); do echo \"\n### Template $${file}.tpl after rendering:\" && cat $${file}; done'", - inline = compact([ - "rm -f /var/user_kustomize/**/*.yaml.tpl", - "echo 'Applying user kustomization...'", - "kubectl apply -k /var/user_kustomize/ --wait=true", - var.extra_kustomize_deployment_commands - ]) - } - - lifecycle { - replace_triggered_by = [ - null_resource.kustomization_user - ] - } + kustomizations_map = local.processed_kustomizes depends_on = [ - null_resource.kustomization_user + null_resource.kustomization, ] } diff --git a/modules/user_kustomization_set/locals.tf b/modules/user_kustomization_set/locals.tf new file mode 100644 index 00000000..f53e4dd6 --- /dev/null +++ b/modules/user_kustomization_set/locals.tf @@ -0,0 +1,12 @@ +locals { + source_folder_files = var.source_folder == "" ? toset([]) : try(fileset(var.source_folder, "**/*.tpl"), toset([])) + + source_files_sha = join("", [ + for file_path in local.source_folder_files : + filesha1("${var.source_folder}/${file_path}") + ]) + + parameters_sha = nonsensitive(sha256(jsonencode(var.template_parameters))) + pre_commands_string_sha = nonsensitive(sha256(var.pre_commands_string)) + post_commands_string_sha = nonsensitive(sha256(var.post_commands_string)) +} diff --git a/modules/user_kustomization_set/main.tf b/modules/user_kustomization_set/main.tf new file mode 100644 index 00000000..92fbb43c --- /dev/null +++ b/modules/user_kustomization_set/main.tf @@ -0,0 +1,78 @@ + +# Purpose of this module is to copy a single user kustomization "set" to control plane. +# The set contains the yaml-files for Kustomization and the postinstall.sh script. + +resource "null_resource" "install_scripts" { + + triggers = { + source_files_sha = local.source_files_sha + parameters_sha = local.parameters_sha + pre_commands_string_sha = local.pre_commands_string_sha + post_commands_string_sha = local.post_commands_string_sha + } + + connection { + user = var.ssh_connection.user + private_key = var.ssh_connection.private_key + agent_identity = var.ssh_connection.agent_identity + host = var.ssh_connection.host + port = var.ssh_connection.port + + bastion_host = var.ssh_connection.bastion_host + bastion_port = var.ssh_connection.bastion_port + bastion_user = var.ssh_connection.bastion_user + bastion_private_key = var.ssh_connection.bastion_private_key + } + + provisioner "remote-exec" { + inline = [ + "mkdir -p ${var.destination_folder}" + ] + } + + provisioner "file" { + content = templatefile("${path.module}/templates/bash.sh.tpl", { commands = var.pre_commands_string }) + destination = "${var.destination_folder}/preinstall.sh" + } + + provisioner "file" { + content = templatefile("${path.module}/templates/bash.sh.tpl", { commands = var.post_commands_string }) + destination = "${var.destination_folder}/postinstall.sh" + } +} + +resource "null_resource" "user_kustomization_template_files" { + for_each = nonsensitive(local.source_folder_files) + + lifecycle { + replace_triggered_by = [ + null_resource.install_scripts + ] + } + + connection { + user = var.ssh_connection.user + private_key = var.ssh_connection.private_key + agent_identity = var.ssh_connection.agent_identity + host = var.ssh_connection.host + port = var.ssh_connection.port + + bastion_host = var.ssh_connection.bastion_host + bastion_port = var.ssh_connection.bastion_port + bastion_user = var.ssh_connection.bastion_user + bastion_private_key = var.ssh_connection.bastion_private_key + } + + provisioner "remote-exec" { + inline = [ + "mkdir -p $(dirname \"${var.destination_folder}/${each.key}\")" + ] + } + + provisioner "file" { + content = templatefile("${var.source_folder}/${each.key}", var.template_parameters) + destination = replace("${var.destination_folder}/${each.key}", ".tpl", "") + } + + depends_on = [null_resource.install_scripts] +} diff --git a/modules/user_kustomization_set/out.tf b/modules/user_kustomization_set/out.tf new file mode 100644 index 00000000..e86c4381 --- /dev/null +++ b/modules/user_kustomization_set/out.tf @@ -0,0 +1,26 @@ +output "source_files_sha" { + value = local.source_files_sha +} + +output "parameters_sha" { + value = local.parameters_sha +} + +output "pre_commands_string_sha" { + value = local.pre_commands_string_sha +} + +output "post_commands_string_sha" { + value = local.post_commands_string_sha +} + +output "files_count" { + description = "Number of template files found in the source folder." + value = length(local.source_folder_files) +} + +output "changes_sha" { + value = nonsensitive(sha256(join("", [ + local.source_files_sha, local.parameters_sha, local.pre_commands_string_sha, local.post_commands_string_sha + ]))) +} diff --git a/modules/user_kustomization_set/templates/bash.sh.tpl b/modules/user_kustomization_set/templates/bash.sh.tpl new file mode 100644 index 00000000..9b675bda --- /dev/null +++ b/modules/user_kustomization_set/templates/bash.sh.tpl @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +${commands} diff --git a/modules/user_kustomization_set/variables.tf b/modules/user_kustomization_set/variables.tf new file mode 100644 index 00000000..61f49cfb --- /dev/null +++ b/modules/user_kustomization_set/variables.tf @@ -0,0 +1,48 @@ +variable "ssh_connection" { + type = object({ + user = string + private_key = string + agent_identity = string + host = string + port = string + + bastion_host = string + bastion_port = number + bastion_user = string + bastion_private_key = string + }) + sensitive = true +} + +variable "source_folder" { + type = string + default = "" +} + +variable "destination_folder" { + type = string + default = "/var/user_kustomize" + + validation { + condition = startswith(var.destination_folder, "/") && can(regex("^[^\\s]*$", var.destination_folder)) + error_message = "destination_folder must start with '/' and must not contain spaces." + } +} + +variable "template_parameters" { + type = map(any) + default = {} + sensitive = true +} + +variable "pre_commands_string" { + type = string + default = "" + sensitive = true +} + +variable "post_commands_string" { + type = string + default = "" + sensitive = true +} diff --git a/modules/user_kustomizations/locals.tf b/modules/user_kustomizations/locals.tf new file mode 100644 index 00000000..af091994 --- /dev/null +++ b/modules/user_kustomizations/locals.tf @@ -0,0 +1,7 @@ +locals { + + destination_keys = [ + for key, mod in module.user_kustomization_set : key + ] + base_destination_folder = "/var/user_kustomize" +} diff --git a/modules/user_kustomizations/main.tf b/modules/user_kustomizations/main.tf new file mode 100644 index 00000000..ca1ef97d --- /dev/null +++ b/modules/user_kustomizations/main.tf @@ -0,0 +1,77 @@ +# Purpose of this module is to initiate the copy of each user kustomization "set" and the deploy them one by one in sequential order. + + +module "user_kustomization_set" { + source = "../user_kustomization_set" + + for_each = nonsensitive(toset(keys(var.kustomizations_map))) + + ssh_connection = var.ssh_connection + + source_folder = var.kustomizations_map[each.key].source_folder + destination_folder = "${local.base_destination_folder}/${each.key}" + template_parameters = var.kustomizations_map[each.key].kustomize_parameters + + pre_commands_string = var.kustomizations_map[each.key].pre_commands + post_commands_string = var.kustomizations_map[each.key].post_commands +} + +resource "null_resource" "kustomization_user_deploy" { + + triggers = { + kustomization_shas = sha256(yamlencode(module.user_kustomization_set)) + } + + connection { + user = var.ssh_connection.user + private_key = var.ssh_connection.private_key + agent_identity = var.ssh_connection.agent_identity + host = var.ssh_connection.host + port = var.ssh_connection.port + + bastion_host = var.ssh_connection.bastion_host + bastion_port = var.ssh_connection.bastion_port + bastion_user = var.ssh_connection.bastion_user + bastion_private_key = var.ssh_connection.bastion_private_key + } + + provisioner "remote-exec" { + inline = [ + <<-EOT + #!/bin/bash + set -e + + function cleanup { + echo "Cleaning up ${local.base_destination_folder}..." + rm -rf ${local.base_destination_folder} + } + trap cleanup EXIT + + sorted_dest_keys=$(printf '%s\n' ${join(" ", local.destination_keys)} | sort -n | tr '\n' ' ') + + for dest_key in $sorted_dest_keys; do + dest_folder="${local.base_destination_folder}/$dest_key" + + if [ -d "$dest_folder" ]; then + echo "Running pre-install script from $dest_folder" + /bin/bash "$dest_folder/preinstall.sh" + + if [ -s "$dest_folder/kustomization.yaml" ] || [ -s "$dest_folder/kustomization.yml" ] || [ -s "$dest_folder/Kustomization" ]; then + echo "Applying kustomization from $dest_folder" + kubectl apply -k "$dest_folder" + else + echo "No valid kustomization file found in $dest_folder, skipping apply." + fi + + echo "Running post-install script from $dest_folder" + /bin/bash "$dest_folder/postinstall.sh" + fi + done + EOT + ] + } + + depends_on = [ + module.user_kustomization_set, + ] +} diff --git a/modules/user_kustomizations/variables.tf b/modules/user_kustomizations/variables.tf new file mode 100644 index 00000000..f7015fce --- /dev/null +++ b/modules/user_kustomizations/variables.tf @@ -0,0 +1,35 @@ +variable "ssh_connection" { + type = object({ + user = string + private_key = string + agent_identity = string + host = string + port = string + + bastion_host = string + bastion_port = number + bastion_user = string + bastion_private_key = string + }) + sensitive = true +} + +variable "kustomizations_map" { + type = map(object({ + source_folder = optional(string, "") + kustomize_parameters = optional(map(any), {}) + pre_commands = optional(string, "") + post_commands = optional(string, "") + })) + default = {} + description = "Map of kustomization entries, where key is the order number." + sensitive = true + + validation { + condition = alltrue([ + for key in keys(var.kustomizations_map) : + can(regex("^[0-9]+$", key)) && tonumber(key) > 0 + ]) + error_message = "All keys in kustomizations_map must be numeric strings (e.g., '1', '2')." + } +} diff --git a/variables.tf b/variables.tf index 84e445b3..af337a14 100644 --- a/variables.tf +++ b/variables.tf @@ -1074,23 +1074,30 @@ variable "postinstall_exec" { description = "Additional to execute after the install calls, for example restoring a backup." } +variable "user_kustomizations" { + type = map(object({ + source_folder = optional(string, "") + kustomize_parameters = optional(map(any), {}) + pre_commands = optional(string, "") + post_commands = optional(string, "") + })) + default = { + "1" = { + source_folder = "extra-manifests" + kustomize_parameters = {} + pre_commands = "" + post_commands = "" + } + } + description = "Map of Kustomization-set entries, where key is the order number." -variable "extra_kustomize_deployment_commands" { - type = string - default = "" - description = "Commands to be executed after the `kubectl apply -k ` step." -} - -variable "extra_kustomize_parameters" { - type = any - default = {} - description = "All values will be passed to the `kustomization.tmp.yml` template." -} - -variable "extra_kustomize_folder" { - type = string - default = "extra-manifests" - description = "Folder from where to upload extra manifests" + validation { + condition = alltrue([ + for key in keys(var.user_kustomizations) : + can(regex("^[0-9]+$", key)) && tonumber(key) > 0 + ]) + error_message = "All keys in user_kustomizations must be positive numeric strings (e.g., '1', '2')." + } } variable "create_kubeconfig" {