enh: switch from aws-calico helm chart to tigera operator helm chart (#288)

mglotov · web-flow · commit e3d7f53d33b9 · 2022-05-13T15:51:27.000+06:00
diff --git a/docs/FAQ.md b/docs/FAQ.md
@@ -291,3 +291,16 @@ alertmanager:
 ### If you want to receive alerts **via Slack**, then do next:
 * See [this instruction](https://slack.com/help/articles/115005265063-Incoming-webhooks-for-Slack) and generate Slack Incoming Webhook 
 * Set `alertmanager_slack_webhook`, `alertmanager_slack_channel` variables in [AWS Secrets Manager](https://console.aws.amazon.com/secretsmanager/home?region=us-east-1#!/home) secret with the pattern `/${local.name_wo_region}/infra/layer2-k8s`.
+
+## Deleting Tigera-operator
+1. Run
+```bash
+kubectl delete installations.operator.tigera.io default
+```
+2. Set `enabled: false` for `id: tigera-operator` in the file **helm-releases.yaml** 
+3. Run `terraform apply` in the layer2-k8s folder
+4. Run
+```bash
+kubectl delete ns calico-apiserver calico-system
+```
+5. Restart all nodes
diff --git a/terraform/layer2-k8s/.terraform.lock.hcl b/terraform/layer2-k8s/.terraform.lock.hcl
diff --git a/terraform/layer2-k8s/README.md b/terraform/layer2-k8s/README.md
@@ -19,7 +19,7 @@
 | <a name="provider_http"></a> [http](#provider\_http) | 2.1.0 |
 | <a name="provider_kubectl"></a> [kubectl](#provider\_kubectl) | 1.14.0 |
 | <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | 2.10.0 |
-| <a name="provider_random"></a> [random](#provider\_random) | 3.1.2 |
+| <a name="provider_random"></a> [random](#provider\_random) | 3.1.3 |
 | <a name="provider_terraform"></a> [terraform](#provider\_terraform) | n/a |
 
 ## Modules
@@ -51,6 +51,7 @@
 | <a name="module_kube_prometheus_stack_namespace"></a> [kube\_prometheus\_stack\_namespace](#module\_kube\_prometheus\_stack\_namespace) | ../modules/kubernetes-namespace | n/a |
 | <a name="module_loki_namespace"></a> [loki\_namespace](#module\_loki\_namespace) | ../modules/kubernetes-namespace | n/a |
 | <a name="module_reloader_namespace"></a> [reloader\_namespace](#module\_reloader\_namespace) | ../modules/kubernetes-namespace | n/a |
+| <a name="module_tigera_operator_namespace"></a> [tigera\_operator\_namespace](#module\_tigera\_operator\_namespace) | ../modules/kubernetes-namespace | n/a |
 | <a name="module_victoria_metrics_k8s_stack_namespace"></a> [victoria\_metrics\_k8s\_stack\_namespace](#module\_victoria\_metrics\_k8s\_stack\_namespace) | ../modules/kubernetes-namespace | n/a |
 
 ## Resources
@@ -63,7 +64,6 @@
 | [aws_s3_bucket_public_access_block.gitlab_runner_cache_public_access_block](https://registry.terraform.io/providers/aws/4.10.0/docs/resources/s3_bucket_public_access_block) | resource |
 | [helm_release.aws_loadbalancer_controller](https://registry.terraform.io/providers/helm/2.5.1/docs/resources/release) | resource |
 | [helm_release.aws_node_termination_handler](https://registry.terraform.io/providers/helm/2.5.1/docs/resources/release) | resource |
-| [helm_release.calico_daemonset](https://registry.terraform.io/providers/helm/2.5.1/docs/resources/release) | resource |
 | [helm_release.cert_manager](https://registry.terraform.io/providers/helm/2.5.1/docs/resources/release) | resource |
 | [helm_release.certificate](https://registry.terraform.io/providers/helm/2.5.1/docs/resources/release) | resource |
 | [helm_release.cluster_autoscaler](https://registry.terraform.io/providers/helm/2.5.1/docs/resources/release) | resource |
@@ -80,7 +80,9 @@
 | [helm_release.loki_stack](https://registry.terraform.io/providers/helm/2.5.1/docs/resources/release) | resource |
 | [helm_release.prometheus_operator](https://registry.terraform.io/providers/helm/2.5.1/docs/resources/release) | resource |
 | [helm_release.reloader](https://registry.terraform.io/providers/helm/2.5.1/docs/resources/release) | resource |
+| [helm_release.tigera_operator](https://registry.terraform.io/providers/helm/2.5.1/docs/resources/release) | resource |
 | [helm_release.victoria_metrics_k8s_stack](https://registry.terraform.io/providers/helm/2.5.1/docs/resources/release) | resource |
+| [kubectl_manifest.calico_felix](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource |
 | [kubectl_manifest.istio_prometheus_service_monitor_cp](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource |
 | [kubectl_manifest.istio_prometheus_service_monitor_dp](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource |
 | [kubectl_manifest.kube_prometheus_stack_operator_crds](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource |
diff --git a/terraform/layer2-k8s/eks-calico.tf b/terraform/layer2-k8s/eks-calico.tf
diff --git a/terraform/layer2-k8s/eks-tigera-operator.tf b/terraform/layer2-k8s/eks-tigera-operator.tf
@@ -0,0 +1,96 @@
+locals {
+  tigera_operator = {
+    name          = local.helm_releases[index(local.helm_releases.*.id, "tigera-operator")].id
+    enabled       = local.helm_releases[index(local.helm_releases.*.id, "tigera-operator")].enabled
+    chart         = local.helm_releases[index(local.helm_releases.*.id, "tigera-operator")].chart
+    repository    = local.helm_releases[index(local.helm_releases.*.id, "tigera-operator")].repository
+    chart_version = local.helm_releases[index(local.helm_releases.*.id, "tigera-operator")].chart_version
+    namespace     = local.helm_releases[index(local.helm_releases.*.id, "tigera-operator")].namespace
+  }
+
+  tigera_operator_values = <<VALUES
+installation:
+  kubernetesProvider: EKS
+VALUES
+}
+
+#tfsec:ignore:kubernetes-network-no-public-egress tfsec:ignore:kubernetes-network-no-public-ingress
+module "tigera_operator_namespace" {
+  count = local.tigera_operator.enabled ? 1 : 0
+
+  source = "../modules/kubernetes-namespace"
+  name   = local.tigera_operator.name
+  network_policies = [
+    {
+      name         = "default-deny"
+      policy_types = ["Ingress", "Egress"]
+      pod_selector = {}
+    },
+    {
+      name         = "allow-this-namespace"
+      policy_types = ["Ingress"]
+      pod_selector = {}
+      ingress = {
+        from = [
+          {
+            namespace_selector = {
+              match_labels = {
+                name = local.tigera_operator.namespace
+              }
+            }
+          }
+        ]
+      }
+    },
+    {
+      name         = "allow-egress"
+      policy_types = ["Egress"]
+      pod_selector = {}
+      egress = {
+        to = [
+          {
+            ip_block = {
+              cidr = "0.0.0.0/0"
+              except = [
+                "169.254.169.254/32"
+              ]
+            }
+          }
+        ]
+      }
+    }
+  ]
+}
+
+resource "kubectl_manifest" "calico_felix" {
+  count = local.tigera_operator.enabled ? 1 : 0
+
+  yaml_body = <<YAML
+apiVersion: crd.projectcalico.org/v1
+kind: FelixConfiguration
+metadata:
+  name: default
+spec:
+  logSeverityScreen: Warning
+  usageReportingEnabled: false
+YAML
+
+  depends_on = [
+    helm_release.tigera_operator
+  ]
+}
+
+resource "helm_release" "tigera_operator" {
+  count = local.tigera_operator.enabled ? 1 : 0
+
+  name        = local.tigera_operator.name
+  chart       = local.tigera_operator.chart
+  repository  = local.tigera_operator.repository
+  version     = local.tigera_operator.chart_version
+  namespace   = module.tigera_operator_namespace[count.index].name
+  max_history = var.helm_release_history_size
+
+  values = [
+    local.tigera_operator_values
+  ]
+}
diff --git a/terraform/layer2-k8s/eks-victoria-metrics-k8s-stack.tf b/terraform/layer2-k8s/eks-victoria-metrics-k8s-stack.tf
@@ -532,7 +532,7 @@ resource "helm_release" "victoria_metrics_k8s_stack" {
 
   depends_on = [
     kubectl_manifest.kube_prometheus_stack_operator_crds,
-    helm_release.calico_daemonset,
+    helm_release.tigera_operator,
     helm_release.ingress_nginx
   ]
 
diff --git a/terraform/layer2-k8s/helm-releases.yaml b/terraform/layer2-k8s/helm-releases.yaml
@@ -11,12 +11,6 @@ releases:
     repository: https://aws.github.io/eks-charts
     chart_version: 0.18.1
     namespace: aws-node-termination-handler
-  - id: aws-calico
-    enabled: true
-    chart: aws-calico
-    repository: https://aws.github.io/eks-charts
-    chart_version: 0.3.11
-    namespace: kube-system
   - id: cert-manager
     enabled: false
     chart: cert-manager
@@ -90,7 +84,7 @@ releases:
     chart_version: 1.49.0
     namespace: kiali
   - id: kube-prometheus-stack
-    enabled: false
+    enabled: true
     chart: kube-prometheus-stack
     repository: https://prometheus-community.github.io/helm-charts
     chart_version: 34.10.0
@@ -107,8 +101,14 @@ releases:
     repository: https://stakater.github.io/stakater-charts
     chart_version: 0.0.110
     namespace: reloader
-  - id: victoria-metrics-k8s-stack
+  - id: tigera-operator
     enabled: true
+    chart: tigera-operator
+    repository: https://projectcalico.docs.tigera.io/charts
+    chart_version: v3.23.0
+    namespace: tigera-operator
+  - id: victoria-metrics-k8s-stack
+    enabled: false
     chart: victoria-metrics-k8s-stack
     repository: https://victoriametrics.github.io/helm-charts
     chart_version: 0.8.1

Original file line number	Diff line number	Diff line change
`@@ -532,7 +532,7 @@ resource "helm_release" "victoria_metrics_k8s_stack" {`
`532`	`532`
`533`	`533`	`depends_on = [`
`534`	`534`	`kubectl_manifest.kube_prometheus_stack_operator_crds,`
`535`		`- helm_release.calico_daemonset,`
	`535`	`+ helm_release.tigera_operator,`
`536`	`536`	`helm_release.ingress_nginx`
`537`	`537`	`]`
`538`	`538`