Add karpenter, node pool and node class

Author: Utwo

cluster.yaml

@@ -3,15 +3,16 @@ kind: ClusterConfig
 
 metadata:
   name: code4ro-prod
-  region: eu-west-1
-  version: "1.27"
-  tags: 
+  region: eu-central-1
+  version: "1.29"
+  tags:
     app: code4ro-k8s-prod
     env: production
+    karpenter.sh/discovery: code4ro-k8s-prod
 vpc:
   publicAccessCIDRs: ["82.79.70.195/16"]
   clusterEndpoints:
-    publicAccess:  true
+    publicAccess: true
     privateAccess: true
 
 iamIdentityMappings:
@@ -22,8 +23,13 @@ iamIdentityMappings:
     username: admin
     noDuplicateARNs: true # prevents shadowing of ARNs
 
+karpenter:
+  version: "v0.35.4"
+  createServiceAccount: true
+  withSpotInterruptionQueue: true # adds all required policies and rules for supporting Spot Interruption Queue, default is false
+
 iam:
-  withOIDC: true   
+  withOIDC: true
   serviceAccounts:
     - metadata:
         name: aws-load-balancer-controller
@@ -43,9 +49,9 @@ addons:
     serviceAccountRoleARN: arn:aws:iam::172019762325:role/AmazonEKS_EBS_CSI_Driver
 
 managedNodeGroups:
-  - name: nodegroup1
+  - name: nodegroup
     instanceType: t3a.medium
-    availabilityZones: ["eu-west-1a"]
+    availabilityZones: ["eu-central-1a"]
     iam:
       attachPolicyARNs:
         - arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy
@@ -55,46 +61,14 @@ managedNodeGroups:
         - arn:aws:iam::aws:policy/ElasticLoadBalancingFullAccess
         - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy # we need only this for viewing logs in Cloudwatch, the rest are there because we need the default https://eksctl.io/usage/iam-policies/#attaching-policies-by-arn
       withAddonPolicies:
-        autoScaler: true
         awsLoadBalancerController: true
         cloudWatch: true
         ebs: true
     desiredCapacity: 2
-    minSize: 2
-    maxSize: 8
+    minSize: 1
+    maxSize: 3
     volumeSize: 20
     tags:
       nodegroup-role: worker
       app: code4ro-k8s-prod
       env: prod
-
-  - name: spot-pool1
-    instanceTypes: ["t3a.medium", "t3.medium", "t3a.large", "t3.large", "t2.medium", "t2.large"]
-    availabilityZones: ["eu-west-1a"]
-    spot: true
-    iam:
-      attachPolicyARNs:
-        - arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy
-        - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
-        - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy
-        - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly
-        - arn:aws:iam::aws:policy/ElasticLoadBalancingFullAccess
-        - arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy # we need only this for viewing logs in Cloudwatch, the rest are there because we need the default https://eksctl.io/usage/iam-policies/#attaching-policies-by-arn
-      withAddonPolicies:
-        autoScaler: true
-        awsLoadBalancerController: true
-        cloudWatch: true
-        ebs: true
-    desiredCapacity: 1
-    minSize: 0
-    maxSize: 6
-    volumeSize: 20
-    labels: { nodegroup-role: spot, autoscaling: enabled }
-    taints:
-      - key: nodegroup-role
-        value: spot
-        effect: NoSchedule
-    tags:
-      nodegroup-role: spot
-      app: code4ro-k8s-prod
-      env: prod

infra/argo-apps/karpenter.yaml

@@ -0,0 +1,35 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: karpenter
+  namespace: argocd
+spec:
+  project: infra
+  sources:
+    - path: infra/karpenter
+      repoURL: https://github.com/code4romania/code4ro-k8s.git
+      targetRevision: HEAD
+    - chart: karpenter
+      helm:
+        valuesObject:
+          settings:
+            clusterName: code4ro-prod
+          controller:
+            resources:
+              requests:
+                cpu: 100m
+                memory: 100M
+              limits:
+                cpu: 1
+                memory: 1Gi
+          serviceAccount:
+            annotations:
+              eks.amazonaws.com/role-arn: arn:aws:iam::172019762325:role/KarpenterControllerRole-code4ro-prod
+      repoURL: public.ecr.aws/karpenter
+      targetRevision: 0.35.4
+  destination:
+    namespace: karpenter
+    server: https://kubernetes.default.svc
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true

infra/karpenter/default-node-class.yaml

@@ -0,0 +1,15 @@
+apiVersion: karpenter.k8s.aws/v1beta1
+kind: EC2NodeClass
+metadata:
+  name: default
+  annotations:
+    kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes"
+spec:
+  amiFamily: AL2
+  role: "KarpenterNodeRole-code4ro-prod"
+  subnetSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: "code4ro-prod"
+  securityGroupSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: "code4ro-prod"

infra/karpenter/general-node-pool.yaml

@@ -0,0 +1,40 @@
+apiVersion: karpenter.sh/v1beta1
+kind: NodePool
+metadata:
+  name: general-purpose
+  annotations:
+    kubernetes.io/description: "General purpose NodePool for generic workloads"
+spec:
+  template:
+    metadata:
+      spec:
+        labels:
+          nodegroup-role: karpenter-worker
+          app: code4ro-k8s-prod
+          env: prod
+    spec:
+      requirements:
+        - key: kubernetes.io/arch
+          operator: In
+          values: ["amd64"]
+        - key: kubernetes.io/os
+          operator: In
+          values: ["linux"]
+        - key: karpenter.sh/capacity-type
+          operator: In
+          values: ["on-demand"]
+        - key: karpenter.k8s.aws/instance-category
+          operator: In
+          values: ["c", "m", "r"]
+        - key: karpenter.k8s.aws/instance-generation
+          operator: Gt
+          values: ["2"]
+        - key: "topology.kubernetes.io/zone"
+          operator: In
+          values: ["eu-central-1a"]
+      nodeClassRef:
+        name: default
+  limits:
+    resources:
+      cpu: "80" # 10 instances * 8 vCPU
+      memory: 320Gi # 10 instances * 32Gi

infra/karpenter/spot-node-pool.yaml

@@ -0,0 +1,44 @@
+apiVersion: karpenter.sh/v1beta1
+kind: NodePool
+metadata:
+  name: spot
+  annotations:
+    kubernetes.io/description: "NodePool for provisioning spot capacity"
+spec:
+  template:
+    metadata:
+      spec:
+        labels:
+          nodegroup-role: karpenter-spot
+          app: code4ro-k8s-prod
+          env: prod
+    spec:
+      requirements:
+        - key: karpenter.sh/capacity-type
+          operator: In
+          values: ["spot"]
+        - key: kubernetes.io/arch
+          operator: In
+          values: ["amd64"]
+        - key: kubernetes.io/os
+          operator: In
+          values: ["linux"]
+        - key: karpenter.k8s.aws/instance-category
+          operator: In
+          values: ["c", "m", "r"]
+        - key: karpenter.k8s.aws/instance-generation
+          operator: Gt
+          values: ["2"]
+        - key: "topology.kubernetes.io/zone"
+          operator: In
+          values: ["eu-central-1a"]
+      taints:
+        - key: nodegroup-role
+          effect: NoSchedule
+          value: spot
+      nodeClassRef:
+        name: default
+  limits:
+    resources:
+      cpu: "80" # 10 instances * 8 vCPU
+      memory: 320Gi # 10 instances * 32Gi