Add pre-install Kustomization folder

Author: vsalomaki

README.md

@@ -334,11 +334,41 @@ _That said, you can also use pure Terraform and import the kube-hetzner module a
 
 <details>
 
-<summary>Custom post-install actions</summary>
+<summary>Custom pre- and post-install actions</summary>
 
 After the initial bootstrapping of your Kubernetes cluster, you might want to deploy applications using the same terraform mechanism. For many scenarios it is sufficient to create a `kustomization.yaml.tpl` file (see [Adding Extras](#adding-extras)). All applied kustomizations will be applied at once by executing a single `kubectl apply -k` command.
 
-However, some applications that e.g. provide custom CRDs (e.g. [ArgoCD](https://argoproj.github.io/cd/)) need a different deployment strategy: one has to deploy CRDs first, then wait for the deployment, before being able to install the actual application. In the ArgoCD case, not waiting for the CRD setup to finish will cause failures. Therefore, an additional mechanism is available to support these kind of deployments. Specify `extra_kustomize_deployment_commands` in your `kube.tf` file containing a series of commands to be executed, after the `Kustomization` step finished:
+However, some applications that e.g. provide custom CRDs (e.g. [ArgoCD](https://argoproj.github.io/cd/)) need a different deployment strategy: one has to deploy CRDs first, then wait for the deployment, before being able to install the actual application. In the ArgoCD case, not waiting for the CRD setup to finish will cause failures. Therefore, an additional mechanism is available to support these kind of deployments. 
+
+### Pre-install Actions, Example: external-secrets repo and Helm
+You can install Helm repos and CRDs before the Kustomization-scripts by adding the helm charts to `extra-manifests-preinstall`-folder and specifying them there in kustomization.yaml.tpl, just like with `extra-manifests`.
+Also, if you need to wait for the CRDs to be run, you can specify additional wait commands to `extra_kustomize_pre_install_post_deploy_commands`, which defaults to `sleep 10`.
+
+For example, to add `external-secrets` so that it can be referenced later on, create files:
+1. extra-manifests-preinstall/eso.yaml.tpl
+```yaml
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: external-secrets
+  namespace: kube-system
+spec:
+  chart: external-secrets
+  repo: https://charts.external-secrets.io
+  targetNamespace: external-secrets
+  createNamespace: true
+```
+2. extra-manifests-preinstall/kustomization.yaml.tpl
+```yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - eso.yaml
+```
+
+### Post-install actions, ArgoCD-example
+Specify `extra_kustomize_deployment_commands` in your `kube.tf` file containing a series of commands to be executed, after the `Kustomization` step finished:
 
 ```tf
   extra_kustomize_deployment_commands = <<-EOT

kube.tf.example

@@ -952,10 +952,18 @@ module "kube-hetzner" {
   # More information about the registration can be found here https://rancher.com/docs/rancher/v2.6/en/cluster-provisioning/registered-clusters/
   # rancher_registration_manifest_url = "https://rancher.xyz.dev/v3/import/xxxxxxxxxxxxxxxxxxYYYYYYYYYYYYYYYYYYYzzzzzzzzzzzzzzzzzzzzz.yaml"
 
+
+  # Local folder for Kustomization *.yaml.tpl-files that are to be run before the `extra-manifests`. Defaults to `"extra-manifests-preinstall"`.
+  # extra_kustomize_pre_install_folder = "extra-manifests-preinstall"
+  
+  # Command to be executed after the `apply -k` of pre-install Kustomizations defined by `<extra_kustomize_pre_install_folder>/kustomization.yaml.tpl`. Defaults to "sleep 10".
+  # Can be used with installation of CRDs with commands such as `kubectl wait --for=condition=Available deployment/mycrd`
+  # extra_kustomize_pre_install_post_deploy_commands = "sleep 10"
+
   # Extra commands to be executed after the `kubectl apply -k` (useful for post-install actions, e.g. wait for CRD, apply additional manifests, etc.).
   # extra_kustomize_deployment_commands=""
 
-  # Extra values that will be passed to the `extra-manifests/kustomization.yaml.tpl` if its present.
+  # Extra values that will be passed to the `extra-manifests/kustomization.yaml.tpl` and `extra-manifests-preinstall/kustomization.yaml.tpl` when present.
   # extra_kustomize_parameters={}
 
   # See working examples for extra manifests or a HelmChart in examples/kustomization_user_deploy/README.md

kustomization_user.tf

@@ -1,5 +1,82 @@
 locals {
-  user_kustomization_templates = try(fileset(var.extra_kustomize_folder, "**/*.yaml.tpl"), toset([]))
+  user_kustomization_templates             = try(fileset(var.extra_kustomize_folder, "**/*.yaml.tpl"), toset([]))
+  user_kustomization_pre_install_templates = try(fileset(var.extra_kustomize_pre_install_folder, "**/*.yaml.tpl"), toset([]))
+}
+
+resource "null_resource" "kustomization_user_pre_install_templates" {
+  for_each = local.user_kustomization_pre_install_templates
+
+  connection {
+    user           = "root"
+    private_key    = var.ssh_private_key
+    agent_identity = local.ssh_agent_identity
+    host           = local.first_control_plane_ip
+    port           = var.ssh_port
+
+    bastion_host        = local.ssh_bastion.bastion_host
+    bastion_port        = local.ssh_bastion.bastion_port
+    bastion_user        = local.ssh_bastion.bastion_user
+    bastion_private_key = local.ssh_bastion.bastion_private_key
+
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "mkdir -p $(dirname /var/user_kustomize/${each.key})"
+    ]
+  }
+
+  provisioner "file" {
+    content     = templatefile("${var.extra_kustomize_pre_install_folder}/${each.key}", var.extra_kustomize_parameters)
+    destination = replace("/var/user_kustomize/${each.key}", ".yaml.tpl", ".yaml")
+  }
+
+  triggers = {
+    manifest_sha1 = "${sha1(templatefile("${var.extra_kustomize_pre_install_folder}/${each.key}", var.extra_kustomize_parameters))}"
+  }
+
+  depends_on = [
+    null_resource.kustomization
+  ]
+}
+
+resource "null_resource" "kustomization_user_pre_install_deploy" {
+  count = length(local.user_kustomization_templates) > 0 ? 1 : 0
+
+  connection {
+    user           = "root"
+    private_key    = var.ssh_private_key
+    agent_identity = local.ssh_agent_identity
+    host           = local.first_control_plane_ip
+    port           = var.ssh_port
+
+    bastion_host        = local.ssh_bastion.bastion_host
+    bastion_port        = local.ssh_bastion.bastion_port
+    bastion_user        = local.ssh_bastion.bastion_user
+    bastion_private_key = local.ssh_bastion.bastion_private_key
+
+  }
+
+  # Remove templates after rendering, and apply changes.
+  provisioner "remote-exec" {
+    # Debugging: "sh -c 'for file in $(find /var/user_kustomize -type f -name \"*.yaml\" | sort -n); do echo \"\n### Template $${file}.tpl after rendering:\" && cat $${file}; done'",
+    inline = compact([
+      "rm -f /var/user_kustomize/**/*.yaml.tpl",
+      "echo 'Applying user pre-install kustomization...'",
+      "kubectl apply -k /var/user_kustomize/",
+      var.extra_kustomize_pre_install_post_deploy_commands
+    ])
+  }
+
+  lifecycle {
+    replace_triggered_by = [
+      null_resource.kustomization_user_pre_install_templates
+    ]
+  }
+
+  depends_on = [
+    null_resource.kustomization_user_pre_install_templates,
+  ]
 }
 
 resource "null_resource" "kustomization_user" {
@@ -35,7 +112,8 @@ resource "null_resource" "kustomization_user" {
   }
 
   depends_on = [
-    null_resource.kustomization
+    null_resource.kustomization,
+    null_resource.kustomization_user_pre_install_deploy
   ]
 }
 
@@ -74,6 +152,7 @@ resource "null_resource" "kustomization_user_deploy" {
   }
 
   depends_on = [
-    null_resource.kustomization_user
+    null_resource.kustomization_user,
+    null_resource.kustomization_user_pre_install_deploy,
   ]
 }

variables.tf

@@ -1068,7 +1068,6 @@ variable "postinstall_exec" {
   description = "Additional to execute after the install calls, for example restoring a backup."
 }
 
-
 variable "extra_kustomize_deployment_commands" {
   type        = string
   default     = ""
@@ -1087,6 +1086,18 @@ variable "extra_kustomize_folder" {
   description = "Folder from where to upload extra manifests"
 }
 
+variable "extra_kustomize_pre_install_folder" {
+  type        = string
+  default     = "extra-manifests-preinstall"
+  description = "Folder from where to upload extra manifests that are run before Kustomization, such as repo installation"
+}
+
+variable "extra_kustomize_pre_install_post_deploy_commands" {
+  type        = string
+  default     = "sleep 10"
+  description = "Commands to be executed after the Kustomization pre-install stage"
+}
+
 variable "create_kubeconfig" {
   type        = bool
   default     = true