Merge branch 'main' into kubernetes-cluster

Author: itzsaga

cspell.config.yaml

@@ -212,7 +212,8 @@ words: [
   "astro",
   "shellsession",
   "downscaling",
-  "mongodbatlas"
+  "mongodbatlas",
+  "PersistentVolumes",
 ]
 languageSettings:
   - languageId: markdown,mdx

flake.lock

@@ -9,7 +9,7 @@
     tfUtilsPkgsSrc.url = "github:NixOS/nixpkgs/93dc9803a1ee435e590b02cde9589038d5cc3a4e";
     buildkitPkgsSrc.url = "github:NixOS/nixpkgs/226216574ada4c3ecefcbbec41f39ce4655f78ef";
     redisPkgsSrc.url = "github:NixOS/nixpkgs/226216574ada4c3ecefcbbec41f39ce4655f78ef";
-    postgresPkgsSrc.url = "github:NixOS/nixpkgs/daf7bb95821b789db24fc1ac21f613db0c1bf2cb";
+    postgresPkgsSrc.url = "github:NixOS/nixpkgs/16e046229f3b4f53257973a5532bcbb72457d2f2";
     vaultPkgsSrc.url = "github:NixOS/nixpkgs/325eb628b89b9a8183256f62d017bfb499b19bd9";
     linkerdPkgsSrc.url = "github:NixOS/nixpkgs/226216574ada4c3ecefcbbec41f39ce4655f78ef";
     kyvernoPkgsSrc.url = "github:NixOS/nixpkgs/226216574ada4c3ecefcbbec41f39ce4655f78ef";

flake.nix

@@ -571,7 +571,7 @@ resource "aws_eks_node_group" "controllers" {
   # During bootstrapping, we should prevent disruptions as much as possible
   # but after Karpenter is running, we should make the EKS nodes spot as
   # they can already be disrupted at inconvenient times due to 'force_update_version = true'
-  capacity_type = var.bootstrap_mode_enabled ? "ON_DEMAND" : "SPOT"
+  capacity_type = var.bootstrap_mode_enabled || !var.spot_nodes_enabled ? "ON_DEMAND" : "SPOT"
 
   tags = merge(local.instance_tags, {
     description = local.controller_nodes_description
@@ -586,7 +586,7 @@ resource "aws_eks_node_group" "controllers" {
     value  = module.constants.cilium_taint.value
   }
   dynamic "taint" {
-    for_each = var.bootstrap_mode_enabled ? toset(["arm64"]) : toset(["burstable", "spot", "arm64"])
+    for_each = var.bootstrap_mode_enabled ? toset(["arm64"]) : (var.spot_nodes_enabled ? toset(["spot", "burstable", "arm64"]) : toset(["burstable", "arm64"]))
     content {
       effect = "NO_SCHEDULE"
       key    = taint.key

packages/infrastructure/aws_eks/main.tf

@@ -118,6 +118,12 @@ variable "node_ami_name" {
   default     = "bottlerocket-aws-k8s-1.30-aarch64-v1.28.0-0ab4fab4"
 }
 
+variable "spot_nodes_enabled" {
+  description = "Whether to create spot instances instead of on-demand instances"
+  type        = bool
+  default     = true
+}
+
 ################################################################################
 ## Access Control
 ################################################################################

packages/infrastructure/aws_eks/vars.tf

@@ -55,8 +55,9 @@ module "util_controller" {
   az_spread_required                   = false // single instance
   panfactum_scheduler_enabled          = var.panfactum_scheduler_enabled
   pull_through_cache_enabled           = var.pull_through_cache_enabled
-  burstable_nodes_enabled              = true
-  controller_nodes_enabled             = true
+  burstable_nodes_enabled              = var.burstable_nodes_enabled
+  controller_nodes_enabled             = var.controller_nodes_enabled
+  spot_nodes_enabled                   = var.spot_nodes_enabled
   extra_labels                         = data.pf_kube_labels.labels.labels
 }
 
@@ -68,8 +69,9 @@ module "util_server" {
   az_spread_preferred                  = var.sla_target >= 2
   panfactum_scheduler_enabled          = var.panfactum_scheduler_enabled
   pull_through_cache_enabled           = var.pull_through_cache_enabled
-  burstable_nodes_enabled              = true
-  controller_nodes_enabled             = true
+  burstable_nodes_enabled              = var.burstable_nodes_enabled
+  controller_nodes_enabled             = var.controller_nodes_enabled
+  spot_nodes_enabled                   = var.spot_nodes_enabled
   extra_labels                         = data.pf_kube_labels.labels.labels
 }
 
@@ -82,8 +84,9 @@ module "util_events_controller" {
   instance_type_anti_affinity_required = false // single instance
   az_spread_preferred                  = false // single instance
   az_spread_required                   = false // single instance
-  burstable_nodes_enabled              = true
-  controller_nodes_enabled             = true
+  burstable_nodes_enabled              = var.burstable_nodes_enabled
+  controller_nodes_enabled             = var.controller_nodes_enabled
+  spot_nodes_enabled                   = var.spot_nodes_enabled
   extra_labels                         = data.pf_kube_labels.labels.labels
 }
 
@@ -95,8 +98,9 @@ module "util_webhook" {
   az_spread_preferred                  = var.sla_target >= 2
   panfactum_scheduler_enabled          = var.panfactum_scheduler_enabled
   pull_through_cache_enabled           = var.pull_through_cache_enabled
-  burstable_nodes_enabled              = true
-  controller_nodes_enabled             = true
+  burstable_nodes_enabled              = var.burstable_nodes_enabled
+  controller_nodes_enabled             = var.controller_nodes_enabled
+  spot_nodes_enabled                   = var.spot_nodes_enabled
   extra_labels                         = data.pf_kube_labels.labels.labels
 }
 
@@ -258,7 +262,9 @@ module "database" {
   pg_smart_shutdown_timeout            = 2
   aws_iam_ip_allow_list                = var.aws_iam_ip_allow_list
   pull_through_cache_enabled           = var.pull_through_cache_enabled
-  burstable_nodes_enabled              = true
+  burstable_nodes_enabled              = var.burstable_nodes_enabled
+  spot_nodes_enabled                   = var.spot_nodes_enabled
+  controller_nodes_enabled             = false // should not run on controller nodes which can cause disruptions
   backups_force_delete                 = true
   monitoring_enabled                   = var.monitoring_enabled
   panfactum_scheduler_enabled          = var.panfactum_scheduler_enabled
@@ -274,6 +280,7 @@ module "database" {
   pgbouncer_minimum_memory_mb      = var.pgbouncer_minimum_memory_mb
   pgbouncer_maximum_memory_mb      = var.pgbouncer_maximum_memory_mb
 
+  pg_backup_directory      = var.db_backup_directory
   pg_recovery_mode_enabled = var.db_recovery_mode_enabled
   pg_recovery_directory    = var.db_recovery_directory
   pg_recovery_target_time  = var.db_recovery_target_time

packages/infrastructure/kube_argo/main.tf

@@ -8,7 +8,12 @@ output "artifact_bucket_name" {
   value       = module.artifact_bucket.bucket_name
 }
 
-output "db_recovery_directory" {
+output "db_backup_bucket" {
+  description = "The name of the S3 bucket that contains the PostgreSQL backups and WAL archives"
+  value       = module.database.backup_bucket_name
+}
+
+output "db_backup_directory" {
   description = "The name of the directory in the backup bucket that contains the PostgreSQL backups and WAL archives"
-  value       = module.database.recovery_directory
+  value       = module.database.backup_directory
 }

packages/infrastructure/kube_argo/outputs.tf

@@ -104,6 +104,12 @@ variable "db_recovery_directory" {
   default     = null
 }
 
+variable "db_backup_directory" {
+  description = "The name of the directory in the backup bucket containing the backups files for the database."
+  type        = string
+  default     = "initial"
+}
+
 variable "db_recovery_target_time" {
   description = "If provided, will recover the PostgreSQL database to the indicated target time in RFC 3339 format rather than to the latest data."
   type        = string
@@ -113,11 +119,11 @@ variable "db_recovery_target_time" {
 variable "pg_minimum_memory_mb" {
   description = "The minimum amount of memory to allocate to the postgres pods (in Mi)"
   type        = number
-  default     = 400
+  default     = 500
 
   validation {
-    condition     = var.pg_minimum_memory_mb >= 400
-    error_message = "Must provide at least 400MB of memory"
+    condition     = var.pg_minimum_memory_mb >= 500
+    error_message = "Must provide at least 500MB of memory"
   }
 }
 
@@ -188,4 +194,22 @@ variable "wait" {
   description = "Wait for resources to be in a ready state before proceeding. Disabling this flag will allow upgrades to proceed faster but will disable automatic rollbacks. As a result, manual intervention may be required for deployment failures."
   type        = bool
   default     = true
+}
+
+variable "spot_nodes_enabled" {
+  description = "Whether to allow pods to schedule on spot nodes"
+  type        = bool
+  default     = true
+}
+
+variable "burstable_nodes_enabled" {
+  description = "Whether to allow pods to schedule on burstable nodes"
+  type        = bool
+  default     = true
+}
+
+variable "controller_nodes_enabled" {
+  description = "Whether to allow pods to schedule on EKS Node Group nodes (controller nodes)"
+  type        = bool
+  default     = true
 }

packages/infrastructure/kube_argo/vars.tf

@@ -45,8 +45,8 @@ module "util" {
   instance_type_anti_affinity_required = var.replicas > 1 && var.instance_type_anti_affinity_required
   az_spread_preferred                  = var.replicas > 1 && var.az_spread_preferred
 
-  burstable_nodes_enabled     = true
-  controller_nodes_enabled    = true
+  burstable_nodes_enabled     = var.burstable_nodes_enabled
+  controller_nodes_enabled    = var.controller_nodes_enabled
   spot_nodes_enabled          = var.spot_nodes_enabled
   panfactum_scheduler_enabled = var.panfactum_scheduler_enabled
   extra_labels                = data.pf_kube_labels.labels.labels

packages/infrastructure/kube_argo_event_source/main.tf

@@ -38,7 +38,19 @@ variable "panfactum_scheduler_enabled" {
 }
 
 variable "spot_nodes_enabled" {
-  description = "Whether EventSource pods can be run on spot nodes"
+  description = "Whether to allow pods to schedule on spot nodes"
+  type        = bool
+  default     = true
+}
+
+variable "burstable_nodes_enabled" {
+  description = "Whether to allow pods to schedule on burstable nodes"
+  type        = bool
+  default     = true
+}
+
+variable "controller_nodes_enabled" {
+  description = "Whether to allow pods to schedule on EKS Node Group nodes (controller nodes)"
   type        = bool
   default     = true
 }