Merge pull request #68 from CS3219-AY2425S1/PEER-242-Cloud-Deployment

SeeuSim · web-flow · commit 7799bfbe3d4f · 2024-11-01T14:12:19.000+08:00
PEER-242: Add Gcloud Deployment
diff --git a/.github/workflows/build-deploy-docker.yaml b/.github/workflows/build-deploy-docker.yaml
@@ -105,7 +105,7 @@ jobs:
         echo "Outputs Generated: $formatted_matrix"
         echo "matrix=$formatted_matrix" >> $GITHUB_OUTPUT
 
-  build-and-push-image:
+  build-push-deploy-image:
     needs: changes
     if: ${{ fromJson(needs.changes.outputs.matrix)[0] != null }}
     runs-on: ubuntu-latest
@@ -186,14 +186,38 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+      - name: Setup GCloud
+        uses: google-github-actions/setup-gcloud@v2
+        if: ${{ contains(github.ref, 'main') && github.event.pull_request.title != 'Feedback' && env.IS_GKE_CLUSTER_UP }}
+        with: 
+          service_account_key: ${{ secrets.GKE_SA_KEY }}
+          project_id: ${{ secrets.GKE_PROJECT }}
+      
+      - name: Get GKE creds
+        uses: google-github-actions/get-gke-credentials@v2
+        if: ${{ contains(github.ref, 'main') && github.event.pull_request.title != 'Feedback' && env.IS_GKE_CLUSTER_UP }}
+        with:
+          cluster_name: ${{ secrets.GKE_CLUSTER }}
+          location: ${{ secrets.GKE_ZONE }}
+          credentials: ${{ secrets.GKE_SA_KEY }}
+      
+      - name: Deploy to GKE
+        if: ${{ contains(github.ref, 'main') && github.event.pull_request.title != 'Feedback' && env.IS_GKE_CLUSTER_UP }}
+        run: |-
+          deployments=("collab-service" "matching-service" "question-service" "user-service" "frontend")
+          for dplymnt in "${deployments[@]}"; do
+            kubectl -n peerprep rollout restart deployment "$dplymnt"
+          done
+
+
   results:
     if: ${{ always() && !github.event.pull_request.draft }}
     runs-on: ubuntu-latest
     name: Final Results
-    needs: build-and-push-image
+    needs: build-push-deploy-image
     steps:
       - run: |
-          result="${{ needs.build-and-push-image.result }}"
+          result="${{ needs.build-push-deploy-image.result }}"
           if [[ $result == "success" || $result == "skipped" ]]; then
             exit 0
           else
diff --git a/k8s/03-collab-db-deployment.yaml b/k8s/03-collab-db-deployment.yaml
@@ -56,7 +56,7 @@ spec:
               containerPort: 5432
           volumeMounts:
             - name: collab-db-vol
-              mountPath: /data/collab-db  
+              mountPath: /data 
   volumeClaimTemplates:
     - metadata:
         name: collab-db-vol
diff --git a/k8s/03-question-db-deployment.yaml b/k8s/03-question-db-deployment.yaml
@@ -56,7 +56,7 @@ spec:
               containerPort: 5432
           volumeMounts:
             - name: question-db-vol
-              mountPath: /data/question-db  
+              mountPath: /data
   volumeClaimTemplates:
     - metadata:
         name: question-db-vol
diff --git a/k8s/03-user-db-deployment.yaml b/k8s/03-user-db-deployment.yaml
@@ -56,7 +56,7 @@ spec:
               containerPort: 5432
           volumeMounts:
             - name: user-db-vol
-              mountPath: /data/user-db  
+              mountPath: /data
   volumeClaimTemplates:
     - metadata:
         name: user-db-vol
diff --git a/k8s/04-collab-svc-deployment.yaml b/k8s/04-collab-svc-deployment.yaml
@@ -64,7 +64,7 @@ spec:
       containers:
         - name: collab-express
           image: ay2425s1cs3219g16/collab-express:latest
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: Always
           envFrom:
           - secretRef: 
               name: collaboration-secret
diff --git a/k8s/04-collab-svc-hpa.yaml b/k8s/04-collab-svc-hpa.yaml
@@ -16,4 +16,4 @@ spec:
       name: cpu
       target:
         type: Utilization
-        averageUtilization: 30
+        averageUtilization: 70
diff --git a/k8s/04-match-svc-deployment.yaml b/k8s/04-match-svc-deployment.yaml
@@ -77,7 +77,7 @@ spec:
       containers:
         - name: match-express
           image: ay2425s1cs3219g16/match-express:latest
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: Always
           envFrom:
           - secretRef: 
               name: matching-secret
diff --git a/k8s/04-question-svc-deployment.yaml b/k8s/04-question-svc-deployment.yaml
@@ -64,7 +64,7 @@ spec:
       containers:
         - name: question-express
           image: ay2425s1cs3219g16/question-express:latest
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: Always
           envFrom:
           - secretRef: 
               name: question-secret
diff --git a/k8s/04-question-svc-hpa.yaml b/k8s/04-question-svc-hpa.yaml
@@ -16,4 +16,4 @@ spec:
       name: cpu
       target:
         type: Utilization
-        averageUtilization: 30
+        averageUtilization: 70
diff --git a/k8s/04-user-svc-deployment.yaml b/k8s/04-user-svc-deployment.yaml
@@ -64,7 +64,7 @@ spec:
       containers:
         - name: user-express
           image: ay2425s1cs3219g16/user-express:latest
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: Always
           envFrom:
           - secretRef: 
               name: user-secret
diff --git a/k8s/04-user-svc-hpa.yaml b/k8s/04-user-svc-hpa.yaml
@@ -16,4 +16,4 @@ spec:
       name: cpu
       target:
         type: Utilization
-        averageUtilization: 30
+        averageUtilization: 70
diff --git a/k8s/05-frontend-deployment.yaml b/k8s/05-frontend-deployment.yaml
@@ -66,7 +66,7 @@ spec:
       containers:
         - name: frontend
           image: ay2425s1cs3219g16/frontend:latest
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: Always
           envFrom:
           - secretRef: 
               name: frontend-secret
diff --git a/k8s/05-frontend-hpa.yaml b/k8s/05-frontend-hpa.yaml
@@ -16,4 +16,4 @@ spec:
       name: cpu
       target:
         type: Utilization
-        averageUtilization: 30
+        averageUtilization: 60
diff --git a/k8s/README.md b/k8s/README.md
@@ -90,7 +90,7 @@
 2. Run the command to set up the ingress controller:
 
     ```sh
-    kubectl apply -f ./k8s/ingress/nginx-ingress.yaml
+    kubectl apply -f ./k8s/local
     ```
 
     It should take a couple of minutes. Once done, you should run this command:
@@ -147,7 +147,150 @@
     A browser window should launch, directing you to the application's frontend.
 
 ## GKE Instructions
+<!-- https://cert-manager.io/docs/tutorials/getting-started-with-cert-manager-on-google-kubernetes-engine-using-lets-encrypt-for-ingress-ssl/ -->
 
-To be added.
+### Setup
 
-<!-- https://cert-manager.io/docs/tutorials/getting-started-with-cert-manager-on-google-kubernetes-engine-using-lets-encrypt-for-ingress-ssl/ -->
+1. Authenticate or ensure you are added as a user to the Google Cloud Project:
+
+    - Project ID: `cs3219-g16`
+    - Project Zone: `asia-southeast1-c`
+
+2. Install the `gcloud` C by following the instructions at this link:
+
+    - [Installation Instructions](https://cloud.google.com/sdk/docs/install)
+
+3. Setup the CLI with the following commands:
+
+    ```sh
+    gcloud auth login
+
+    gcloud config set project cs3219-g16
+
+    gcloud config set compute/zone asia-southeast1-c
+
+    gcloud components install gke-gcloud-auth-plugin
+
+    export USE_GKE_GCLOUD_AUTH_PLUGIN=True
+    ```
+
+4. Create the cluster with the following commands:
+
+    ```sh
+    gcloud container clusters create \
+      cs3219-g16 \
+      --preemptible \
+      --machine-type e2-small \
+      --enable-autoscaling \
+      --num-nodes 1 \
+      --min-nodes 1 \
+      --max-nodes 25 \
+      --region=asia-southeast1-c
+    ```
+
+5. Once the cluster has been created, run the commands below to configure `kubectl` and connect to the cluster:
+
+    ```sh
+    gcloud container clusters get-credentials cs3219-g16
+
+    # You should see some output here
+    kubectl get nodes -o wide
+    ```
+
+6. Run the script (ensure you are in a Bash shell like on Mac or Linux):
+
+    ```sh
+    make k8s-up
+    ```
+
+    - Wait until the deployments all reach status running:
+
+        ```sh
+        kubectl -n peerprep rollout status deployment frontend
+        ```
+
+7. If you haven't already, visit the GCloud console -> 'Cloud Domains' and verify that a domain name has been created.
+
+    - We currently have one as `peerprep-g16.net`.
+        - This can be created under 'Cloud Domains' -> 'Register Domain' in the GCloud console.
+    - We also associate a GCloud Global Web IP `web-ip` to this DNS record as an 'A' record.
+        - To set an IP DNS 'A' record, follow these steps:
+            1. Create an IP:
+
+                ```sh
+                gcloud compute addresses create web-ip --global
+                ```
+
+            2. Verify that it exists:
+
+                ```sh
+                gcloud compute addresses list
+                ```
+
+            3. Grab the IP address:
+
+                ```sh
+                gcloud compute addresses describe web-ip --format='value(address)' --global
+                ```
+
+            4. Associate it via the console:
+                - Cloud DNS -> 'Zone Name': peerprep-g16.net -> 'Add standard'
+                - Paste the IP address
+                - 'Create'
+
+8. Install the `cert-manager` plugin:
+
+    ```sh
+    kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml
+    ```
+
+9. Create the ingress and secrets in the prod environment:
+
+    ```sh
+    kubectl apply -f ./k8s/gcloud
+    ```
+
+    - After 15 minutes, you should be able to access the UI over HTTPS at this link:
+        - `https://peerprep-g16.net`
+
+10. Cleanup:
+
+    - Delete the cluster:
+
+        ```sh
+        gcloud container clusters delete cs3219-g16
+        ```
+
+    - When done with the project, delete the web records:
+
+        ```sh
+        gcloud dns record-sets delete peerprep-g16 --type A
+
+        gcloud compute addresses delete web-ip --global
+        ```
+
+### CD (Continuous Delivery via Github Actions)
+
+1. Setup the following in Github Actions by:
+
+    - heading to the 'Settings' -> 'Secrets and variables' -> 'Actions' -> 'New repository secret'
+    - Adding the following keys:
+
+        ```txt
+        GKE_SA_KEY: <redacted (get from the cloud console IAM -> 'Service Accounts' page)>
+        GKE_PROJECT: cs3219-g16
+        GKE_CLUSTER: cs3219-g16 
+        GKE_ZONE: asia-southeast1-c
+        ```
+
+        - If the `GKE_SA_KEY` is needed, contact us.
+
+2. Merge a PR to `main`. The following will happend:
+
+    1. An action will run under the 'actions' tab in Github.
+
+    2. This will build and push the service images and verify that the cluster is redeployed with the latest images:
+
+    ```sh
+    kubectl -n peerprep get deployment
+    ```
diff --git a/k8s/gcloud-staging/01-issuer-le-staging.yaml b/k8s/gcloud-staging/01-issuer-le-staging.yaml
@@ -0,0 +1,18 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: letsencrypt-staging
+  namespace: peerprep
+  labels:
+    project: peerprep
+    peerprep.service: app-cert-issuer-staging
+spec:
+  acme:
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    email: ay2425s1.cs3219.g16@gmail.com # ❗ Replace this with your email address
+    privateKeySecretRef:
+      name: letsencrypt-staging
+    solvers:
+    - http01:
+        ingress:
+          name: peerprep-ingress
diff --git a/k8s/gcloud-staging/02-web-ssl-secret.yaml b/k8s/gcloud-staging/02-web-ssl-secret.yaml
@@ -0,0 +1,13 @@
+# Placeholder Secret to store TLS keys
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: web-ssl
+  namespace: peerprep
+  labels:
+    project: peerprep
+type: kubernetes.io/tls
+stringData:
+  tls.key: ""
+  tls.crt: ""
diff --git a/k8s/gcloud-staging/03-ingress.yaml b/k8s/gcloud-staging/03-ingress.yaml
@@ -0,0 +1,27 @@
+# ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: peerprep-ingress
+  namespace: peerprep
+  labels:
+    project: peerprep
+    peerprep.service: app-ingress
+  annotations:
+    # This tells Google Cloud to create an External Load Balancer to realize this Ingress
+    kubernetes.io/ingress.class: gce
+    # This enables HTTP connections from Internet clients
+    kubernetes.io/ingress.allow-http: "true"
+    # This tells Google Cloud to associate the External Load Balancer with the static IP which we created earlier
+    kubernetes.io/ingress.global-static-ip-name: web-ip
+    cert-manager.io/issuer: letsencrypt-staging
+spec:
+  tls:
+    - secretName: web-ssl
+      hosts:
+        - peerprep-g16.net
+  defaultBackend:
+    service:
+      name: frontend
+      port:
+        number: 3000
diff --git a/k8s/gcloud/01-issuer-le-prod.yaml b/k8s/gcloud/01-issuer-le-prod.yaml
@@ -0,0 +1,19 @@
+# issuer-lets-encrypt-production.yaml
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: letsencrypt-production
+  namespace: peerprep
+  labels:
+    project: peerprep
+    peerprep.service: app-cert-issuer-prod
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: ay2425s1.cs3219.g16@gmail.com # ❗ Replace this with your email address
+    privateKeySecretRef:
+      name: letsencrypt-production
+    solvers:
+    - http01:
+        ingress:
+          name: peerprep-ingress
diff --git a/k8s/gcloud/02-web-ssl-secret.yaml b/k8s/gcloud/02-web-ssl-secret.yaml
diff --git a/k8s/gcloud/03-ingress.yaml b/k8s/gcloud/03-ingress.yaml
diff --git a/k8s/local/nginx-ingress.yaml b/k8s/local/nginx-ingress.yaml
diff --git a/scripts/k8s-test-load.sh b/scripts/k8s-test-load.sh
