diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-deploy-docker.yaml similarity index 85% rename from .github/workflows/build-docker.yaml rename to .github/workflows/build-deploy-docker.yaml index 9b6b5f02a9..c981a91aba 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-deploy-docker.yaml @@ -105,7 +105,7 @@ jobs: echo "Outputs Generated: $formatted_matrix" echo "matrix=$formatted_matrix" >> $GITHUB_OUTPUT - build-and-push-image: + build-push-deploy-image: needs: changes if: ${{ fromJson(needs.changes.outputs.matrix)[0] != null }} runs-on: ubuntu-latest @@ -186,14 +186,38 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Setup GCloud + uses: google-github-actions/setup-gcloud@v2 + if: ${{ contains(github.ref, 'main') && github.event.pull_request.title != 'Feedback' && env.IS_GKE_CLUSTER_UP }} + with: + service_account_key: ${{ secrets.GKE_SA_KEY }} + project_id: ${{ secrets.GKE_PROJECT }} + + - name: Get GKE creds + uses: google-github-actions/get-gke-credentials@v2 + if: ${{ contains(github.ref, 'main') && github.event.pull_request.title != 'Feedback' && env.IS_GKE_CLUSTER_UP }} + with: + cluster_name: ${{ secrets.GKE_CLUSTER }} + location: ${{ secrets.GKE_ZONE }} + credentials: ${{ secrets.GKE_SA_KEY }} + + - name: Deploy to GKE + if: ${{ contains(github.ref, 'main') && github.event.pull_request.title != 'Feedback' && env.IS_GKE_CLUSTER_UP }} + run: |- + deployments=("collab-service" "matching-service" "question-service" "user-service" "frontend") + for dplymnt in "${deployments[@]}"; do + kubectl -n peerprep rollout restart deployment "$dplymnt" + done + + results: if: ${{ always() && !github.event.pull_request.draft }} runs-on: ubuntu-latest name: Final Results - needs: build-and-push-image + needs: build-push-deploy-image steps: - run: | - result="${{ needs.build-and-push-image.result }}" + result="${{ needs.build-push-deploy-image.result }}" if [[ $result == "success" || $result == "skipped" ]]; then exit 0 else diff --git a/k8s/03-collab-db-deployment.yaml b/k8s/03-collab-db-deployment.yaml index 8870199f31..55f5598443 100644 --- a/k8s/03-collab-db-deployment.yaml +++ b/k8s/03-collab-db-deployment.yaml @@ -56,7 +56,7 @@ spec: containerPort: 5432 volumeMounts: - name: collab-db-vol - mountPath: /data/collab-db + mountPath: /data volumeClaimTemplates: - metadata: name: collab-db-vol diff --git a/k8s/03-question-db-deployment.yaml b/k8s/03-question-db-deployment.yaml index 85a8505746..28b64e9899 100644 --- a/k8s/03-question-db-deployment.yaml +++ b/k8s/03-question-db-deployment.yaml @@ -56,7 +56,7 @@ spec: containerPort: 5432 volumeMounts: - name: question-db-vol - mountPath: /data/question-db + mountPath: /data volumeClaimTemplates: - metadata: name: question-db-vol diff --git a/k8s/03-user-db-deployment.yaml b/k8s/03-user-db-deployment.yaml index 3ff810409b..4114096a29 100644 --- a/k8s/03-user-db-deployment.yaml +++ b/k8s/03-user-db-deployment.yaml @@ -56,7 +56,7 @@ spec: containerPort: 5432 volumeMounts: - name: user-db-vol - mountPath: /data/user-db + mountPath: /data volumeClaimTemplates: - metadata: name: user-db-vol diff --git a/k8s/04-collab-svc-deployment.yaml b/k8s/04-collab-svc-deployment.yaml index a545b2e47a..b0c3b16daa 100644 --- a/k8s/04-collab-svc-deployment.yaml +++ b/k8s/04-collab-svc-deployment.yaml @@ -64,7 +64,7 @@ spec: containers: - name: collab-express image: ay2425s1cs3219g16/collab-express:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always envFrom: - secretRef: name: collaboration-secret diff --git a/k8s/04-collab-svc-hpa.yaml b/k8s/04-collab-svc-hpa.yaml index af31213f4c..6bdf779326 100644 --- a/k8s/04-collab-svc-hpa.yaml +++ b/k8s/04-collab-svc-hpa.yaml @@ -16,4 +16,4 @@ spec: name: cpu target: type: Utilization - averageUtilization: 30 + averageUtilization: 70 diff --git a/k8s/04-match-svc-deployment.yaml b/k8s/04-match-svc-deployment.yaml index 8d849ca10b..81a7c50bf4 100644 --- a/k8s/04-match-svc-deployment.yaml +++ b/k8s/04-match-svc-deployment.yaml @@ -77,7 +77,7 @@ spec: containers: - name: match-express image: ay2425s1cs3219g16/match-express:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always envFrom: - secretRef: name: matching-secret diff --git a/k8s/04-question-svc-deployment.yaml b/k8s/04-question-svc-deployment.yaml index ab59288f19..eca30a95d7 100644 --- a/k8s/04-question-svc-deployment.yaml +++ b/k8s/04-question-svc-deployment.yaml @@ -64,7 +64,7 @@ spec: containers: - name: question-express image: ay2425s1cs3219g16/question-express:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always envFrom: - secretRef: name: question-secret diff --git a/k8s/04-question-svc-hpa.yaml b/k8s/04-question-svc-hpa.yaml index 6764e3d237..6fd3b1e853 100644 --- a/k8s/04-question-svc-hpa.yaml +++ b/k8s/04-question-svc-hpa.yaml @@ -16,4 +16,4 @@ spec: name: cpu target: type: Utilization - averageUtilization: 30 + averageUtilization: 70 diff --git a/k8s/04-user-svc-deployment.yaml b/k8s/04-user-svc-deployment.yaml index f5cead51bd..2ba03034e8 100644 --- a/k8s/04-user-svc-deployment.yaml +++ b/k8s/04-user-svc-deployment.yaml @@ -64,7 +64,7 @@ spec: containers: - name: user-express image: ay2425s1cs3219g16/user-express:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always envFrom: - secretRef: name: user-secret diff --git a/k8s/04-user-svc-hpa.yaml b/k8s/04-user-svc-hpa.yaml index d64876db82..d5214a1814 100644 --- a/k8s/04-user-svc-hpa.yaml +++ b/k8s/04-user-svc-hpa.yaml @@ -16,4 +16,4 @@ spec: name: cpu target: type: Utilization - averageUtilization: 30 + averageUtilization: 70 diff --git a/k8s/05-frontend-deployment.yaml b/k8s/05-frontend-deployment.yaml index f7169558d2..23bb986d06 100644 --- a/k8s/05-frontend-deployment.yaml +++ b/k8s/05-frontend-deployment.yaml @@ -66,7 +66,7 @@ spec: containers: - name: frontend image: ay2425s1cs3219g16/frontend:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always envFrom: - secretRef: name: frontend-secret diff --git a/k8s/05-frontend-hpa.yaml b/k8s/05-frontend-hpa.yaml index 66a4d5ce6f..74c71025df 100644 --- a/k8s/05-frontend-hpa.yaml +++ b/k8s/05-frontend-hpa.yaml @@ -16,4 +16,4 @@ spec: name: cpu target: type: Utilization - averageUtilization: 30 + averageUtilization: 60 diff --git a/k8s/README.md b/k8s/README.md index 6e3c8b0673..190b45caf9 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -90,7 +90,7 @@ 2. Run the command to set up the ingress controller: ```sh - kubectl apply -f ./k8s/ingress/nginx-ingress.yaml + kubectl apply -f ./k8s/local ``` It should take a couple of minutes. Once done, you should run this command: @@ -147,7 +147,150 @@ A browser window should launch, directing you to the application's frontend. ## GKE Instructions + -To be added. +### Setup - +1. Authenticate or ensure you are added as a user to the Google Cloud Project: + + - Project ID: `cs3219-g16` + - Project Zone: `asia-southeast1-c` + +2. Install the `gcloud` C by following the instructions at this link: + + - [Installation Instructions](https://cloud.google.com/sdk/docs/install) + +3. Setup the CLI with the following commands: + + ```sh + gcloud auth login + + gcloud config set project cs3219-g16 + + gcloud config set compute/zone asia-southeast1-c + + gcloud components install gke-gcloud-auth-plugin + + export USE_GKE_GCLOUD_AUTH_PLUGIN=True + ``` + +4. Create the cluster with the following commands: + + ```sh + gcloud container clusters create \ + cs3219-g16 \ + --preemptible \ + --machine-type e2-small \ + --enable-autoscaling \ + --num-nodes 1 \ + --min-nodes 1 \ + --max-nodes 25 \ + --region=asia-southeast1-c + ``` + +5. Once the cluster has been created, run the commands below to configure `kubectl` and connect to the cluster: + + ```sh + gcloud container clusters get-credentials cs3219-g16 + + # You should see some output here + kubectl get nodes -o wide + ``` + +6. Run the script (ensure you are in a Bash shell like on Mac or Linux): + + ```sh + make k8s-up + ``` + + - Wait until the deployments all reach status running: + + ```sh + kubectl -n peerprep rollout status deployment frontend + ``` + +7. If you haven't already, visit the GCloud console -> 'Cloud Domains' and verify that a domain name has been created. + + - We currently have one as `peerprep-g16.net`. + - This can be created under 'Cloud Domains' -> 'Register Domain' in the GCloud console. + - We also associate a GCloud Global Web IP `web-ip` to this DNS record as an 'A' record. + - To set an IP DNS 'A' record, follow these steps: + 1. Create an IP: + + ```sh + gcloud compute addresses create web-ip --global + ``` + + 2. Verify that it exists: + + ```sh + gcloud compute addresses list + ``` + + 3. Grab the IP address: + + ```sh + gcloud compute addresses describe web-ip --format='value(address)' --global + ``` + + 4. Associate it via the console: + - Cloud DNS -> 'Zone Name': peerprep-g16.net -> 'Add standard' + - Paste the IP address + - 'Create' + +8. Install the `cert-manager` plugin: + + ```sh + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.yaml + ``` + +9. Create the ingress and secrets in the prod environment: + + ```sh + kubectl apply -f ./k8s/gcloud + ``` + + - After 15 minutes, you should be able to access the UI over HTTPS at this link: + - `https://peerprep-g16.net` + +10. Cleanup: + + - Delete the cluster: + + ```sh + gcloud container clusters delete cs3219-g16 + ``` + + - When done with the project, delete the web records: + + ```sh + gcloud dns record-sets delete peerprep-g16 --type A + + gcloud compute addresses delete web-ip --global + ``` + +### CD (Continuous Delivery via Github Actions) + +1. Setup the following in Github Actions by: + + - heading to the 'Settings' -> 'Secrets and variables' -> 'Actions' -> 'New repository secret' + - Adding the following keys: + + ```txt + GKE_SA_KEY: 'Service Accounts' page)> + GKE_PROJECT: cs3219-g16 + GKE_CLUSTER: cs3219-g16 + GKE_ZONE: asia-southeast1-c + ``` + + - If the `GKE_SA_KEY` is needed, contact us. + +2. Merge a PR to `main`. The following will happend: + + 1. An action will run under the 'actions' tab in Github. + + 2. This will build and push the service images and verify that the cluster is redeployed with the latest images: + + ```sh + kubectl -n peerprep get deployment + ``` diff --git a/k8s/gcloud-staging/01-issuer-le-staging.yaml b/k8s/gcloud-staging/01-issuer-le-staging.yaml new file mode 100644 index 0000000000..9d9c5b7c6a --- /dev/null +++ b/k8s/gcloud-staging/01-issuer-le-staging.yaml @@ -0,0 +1,18 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: letsencrypt-staging + namespace: peerprep + labels: + project: peerprep + peerprep.service: app-cert-issuer-staging +spec: + acme: + server: https://acme-staging-v02.api.letsencrypt.org/directory + email: ay2425s1.cs3219.g16@gmail.com # ❗ Replace this with your email address + privateKeySecretRef: + name: letsencrypt-staging + solvers: + - http01: + ingress: + name: peerprep-ingress \ No newline at end of file diff --git a/k8s/gcloud-staging/02-web-ssl-secret.yaml b/k8s/gcloud-staging/02-web-ssl-secret.yaml new file mode 100644 index 0000000000..278c096165 --- /dev/null +++ b/k8s/gcloud-staging/02-web-ssl-secret.yaml @@ -0,0 +1,13 @@ +# Placeholder Secret to store TLS keys + +apiVersion: v1 +kind: Secret +metadata: + name: web-ssl + namespace: peerprep + labels: + project: peerprep +type: kubernetes.io/tls +stringData: + tls.key: "" + tls.crt: "" \ No newline at end of file diff --git a/k8s/gcloud-staging/03-ingress.yaml b/k8s/gcloud-staging/03-ingress.yaml new file mode 100644 index 0000000000..3ed71905ea --- /dev/null +++ b/k8s/gcloud-staging/03-ingress.yaml @@ -0,0 +1,27 @@ +# ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: peerprep-ingress + namespace: peerprep + labels: + project: peerprep + peerprep.service: app-ingress + annotations: + # This tells Google Cloud to create an External Load Balancer to realize this Ingress + kubernetes.io/ingress.class: gce + # This enables HTTP connections from Internet clients + kubernetes.io/ingress.allow-http: "true" + # This tells Google Cloud to associate the External Load Balancer with the static IP which we created earlier + kubernetes.io/ingress.global-static-ip-name: web-ip + cert-manager.io/issuer: letsencrypt-staging +spec: + tls: + - secretName: web-ssl + hosts: + - peerprep-g16.net + defaultBackend: + service: + name: frontend + port: + number: 3000 \ No newline at end of file diff --git a/k8s/gcloud/01-issuer-le-prod.yaml b/k8s/gcloud/01-issuer-le-prod.yaml new file mode 100644 index 0000000000..b1e531d697 --- /dev/null +++ b/k8s/gcloud/01-issuer-le-prod.yaml @@ -0,0 +1,19 @@ +# issuer-lets-encrypt-production.yaml +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: letsencrypt-production + namespace: peerprep + labels: + project: peerprep + peerprep.service: app-cert-issuer-prod +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: ay2425s1.cs3219.g16@gmail.com # ❗ Replace this with your email address + privateKeySecretRef: + name: letsencrypt-production + solvers: + - http01: + ingress: + name: peerprep-ingress \ No newline at end of file diff --git a/k8s/gcloud/02-web-ssl-secret.yaml b/k8s/gcloud/02-web-ssl-secret.yaml new file mode 100644 index 0000000000..278c096165 --- /dev/null +++ b/k8s/gcloud/02-web-ssl-secret.yaml @@ -0,0 +1,13 @@ +# Placeholder Secret to store TLS keys + +apiVersion: v1 +kind: Secret +metadata: + name: web-ssl + namespace: peerprep + labels: + project: peerprep +type: kubernetes.io/tls +stringData: + tls.key: "" + tls.crt: "" \ No newline at end of file diff --git a/k8s/gcloud/03-ingress.yaml b/k8s/gcloud/03-ingress.yaml new file mode 100644 index 0000000000..5cd176924b --- /dev/null +++ b/k8s/gcloud/03-ingress.yaml @@ -0,0 +1,28 @@ +# ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: peerprep-ingress + namespace: peerprep + labels: + project: peerprep + peerprep.service: app-ingress + annotations: + # This tells Google Cloud to create an External Load Balancer to realize this Ingress + kubernetes.io/ingress.class: gce + # This enables HTTP connections from Internet clients + kubernetes.io/ingress.allow-http: "true" + # This tells Google Cloud to associate the External Load Balancer with the static IP which we created earlier + kubernetes.io/ingress.global-static-ip-name: web-ip + cert-manager.io/issuer: letsencrypt-production +spec: + tls: + - secretName: web-ssl + hosts: + - peerprep-g16.net + defaultBackend: + service: + name: frontend + port: + number: 3000 + \ No newline at end of file diff --git a/k8s/ingress/nginx-ingress.yaml b/k8s/local/nginx-ingress.yaml similarity index 100% rename from k8s/ingress/nginx-ingress.yaml rename to k8s/local/nginx-ingress.yaml diff --git a/scripts/k8s-test-load.sh b/scripts/k8s-test-load.sh index b274130921..3c19c24758 100755 --- a/scripts/k8s-test-load.sh +++ b/scripts/k8s-test-load.sh @@ -10,14 +10,16 @@ load_test_service() { exit 1 fi + pod_name="$service_name-service-load-test" + kubectl run -i \ - --tty "$service_name-service-load-test" \ + --tty $pod_name \ --rm \ -n $ns \ --image=busybox \ --labels="peerprep.network.$service_name-api=true" \ --restart=Never \ - -- /bin/sh -c "while sleep 0.01; do wget -q -O- http://$service_name-service:$port/health && echo; done" + -- /bin/sh -c "while true; do wget -q -O- http://$service_name-service:$port/health > /dev/null 2>&1; done" } load_test_service user 9001