Merge pull request #50 from AET-DevOps25/week8-backend

Week8 backend

Author: DuGuYifei

.github/workflows/deploy-aws.yml

@@ -2,6 +2,10 @@ name: Deploy to AWS
 
 on:
   workflow_dispatch:
+    inputs:
+      image:
+        description: "Image name and tag"
+        required: false
 
 jobs:
   deploy:
@@ -16,7 +20,7 @@ jobs:
         uses: appleboy/scp-action@v0.1.7
         with:
           host: ${{ vars.EC2_PUBLIC_IP }}
-          username: ${{ vars.AWS_EC2_USER }}
+          username: ${{ secrets.AWS_EC2_USER }}
           key: ${{ secrets.SERVER_SSH_KEY }}
           source: "./docker-compose.prod.deploy.yml"
           target: ~/srv/app/
@@ -26,18 +30,20 @@ jobs:
         uses: appleboy/ssh-action@v1.0.3
         with:
           host: ${{ vars.EC2_PUBLIC_IP }}
-          username: ${{ vars.AWS_EC2_USER }}
+          username: ${{ secrets.AWS_EC2_USER }}
           key: ${{ secrets.SERVER_SSH_KEY }}
           script: |
             cat > ~/srv/app/.env <<EOF
             # GitHub Container Registry
             GHCR_USER=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')
             GHCR_REPO=$(echo '${{ github.event.repository.name }}' | tr '[:upper:]' '[:lower:]')
             GHCR_TAG=$(
-            if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
-            echo "${{ github.sha }}"
+            if [[ -n "${{ github.event.inputs.image }}" ]]; then
+              echo "${{ github.event.inputs.image }}"
+            elif [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+              echo "${{ github.sha }}"
             else
-            echo "dev-${{ github.sha }}"
+              echo "dev-${{ github.sha }}"
             fi
             )
             DB_USERNAME=${{ secrets.DB_USERNAME }}
@@ -55,16 +61,19 @@ jobs:
             SPRING_PROFILES_ACTIVE=docker
             PYTHONDONTWRITEBYTECODE=1
             PYTHONUNBUFFERED=1
+            OLLAMA_BASE_URL="${{ secrets.OLLAMA_BASE_URL }}"
+            OLLAMA_MODEL="${{ vars.OLLAMA_MODEL }}"
+            OLLAMA_API_KEY="${{ secrets.OLLAMA_API_KEY }}"
             EOF
 
       # SSH - pull images and restart containers
       - name: Pull images and restart containers remotely
         uses: appleboy/ssh-action@v1.0.3
         with:
           host: ${{ vars.EC2_PUBLIC_IP }}
-          username: ${{ vars.AWS_EC2_USER }}
+          username: ${{ secrets.AWS_EC2_USER }}
           key: ${{ secrets.SERVER_SSH_KEY }}
           script: |
             cd ~/srv/app
-            docker compose -f docker-compose.prod.deploy.yml --env-file .env pull
-            docker compose -f docker-compose.prod.deploy.yml --env-file .env up -d
+            docker compose -f docker-compose.prod.deploy.yml pull
+            docker compose -f docker-compose.prod.deploy.yml up -d

.github/workflows/docker-publish.yml

@@ -5,16 +5,40 @@ on:
     branches:
       - main
   workflow_dispatch:
-    # 如果输入image tag，就跳过build
+    # if input image tag, then skip test and build
     inputs:
       image:
         description: "Image name and tag"
         required: false
 
 jobs:
-  build-and-push:
+  unit-test:
     if: ${{ github.event.inputs.image == '' || github.event.inputs.image == null }}
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4.4.1
+
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+
+      - name: Make gradlew executable
+        run: chmod +x gradlew
+
+      - name: Make protobuf executable
+        run: chmod +x service-application/protoc-gen-grpc-java-1.73.0-linux-x86_64.exe
+
+      - name: Run tests for all modules
+        run: ./gradlew test
+
+  build-and-push:
+    needs: unit-test
+    runs-on: ubuntu-latest
 
     permissions:
       contents: read

ansible/ansible.sh

@@ -0,0 +1 @@
+ansible-playbook -i hosts.ini deploy.yml

ansible/deploy.yml

@@ -8,13 +8,13 @@
         update_cache: yes
       when: ansible_os_family == "Debian"
 
-    - name: Install Docker and Docker Compose plugin (for Ubuntu)
-      apt:
-        name:
-          - docker.io
-          - docker-compose-plugin
-        state: present
+    - name: Install Docker using official script (for Ubuntu)
+      shell: |
+        curl -fsSL https://get.docker.com | bash
+      args:
+        executable: /bin/bash
       when: ansible_os_family == "Debian"
+      become: true
 
     - name: Add the user to the docker group
       user:
@@ -28,14 +28,14 @@
         state: directory
         mode: '0755'
 
-    - name: Copy .env to server
-      copy:
-        src: .env
-        dest: /srv/app/.env
-        mode: '0600'
+    - name: Create letsencrypt
+      file:
+        path: /srv/app/letsencrypt
+        state: directory
+        mode: '0755'
 
-    - name: Copy docker-compose.prod.yml to server
-      copy:
-        src: docker-compose.prod.yml
-        dest: /srv/app/docker-compose.prod.yml
-        mode: '0644'
+    - name: Create acme.json
+      file:
+        path: /srv/app/letsencrypt/acme.json
+        state: touch
+        mode: '0600'

ansible/hosts.ini.example

@@ -0,0 +1,2 @@
+[aws]
+xxx.xxx.xxx.xxx ansible_user=xxx ansible_ssh_private_key_file=/home/xxx/.pem_temp/xxx.pem

build.gradle

@@ -34,6 +34,8 @@ subprojects {
         implementation 'org.springframework.boot:spring-boot-starter-actuator'
         implementation 'io.micrometer:micrometer-registry-prometheus:1.15.1'
 
+        implementation("net.logstash.logback:logstash-logback-encoder:8.1")
+
         developmentOnly 'org.springframework.boot:spring-boot-devtools'
         annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor'
 

docker-compose.prod.deploy.yml

@@ -10,7 +10,7 @@ services:
       - "--entrypoints.web.http.redirections.entrypoint.to=websecure"
       - "--entrypoints.web.http.redirections.entrypoint.scheme=https"
       - "--providers.docker=true"
-      - "--providers.docker.exposedByDefault=false"
+      - "--providers.docker.exposedbydefault=false"
       - "--certificatesresolvers.letsencrypt.acme.httpchallenge=true"
       - "--certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web"
       - "--certificatesresolvers.letsencrypt.acme.email=admin@aihr.com"
@@ -20,16 +20,11 @@ services:
     ports:
       - "80:80" # HTTP
       - "443:443" # HTTPS
-      - "8080:8080" # Traefik dashboard
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /home/ubuntu/srv/app/letsencrypt:/letsencrypt
     networks:
       - ai-hr-network
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.traefik.rule=Host(`traefik.localhost`)"
-      - "traefik.http.routers.traefik.service=api@internal"
-      - "traefik.http.routers.traefik.entrypoints=web"
 
   # PostgreSQL database
   postgresql:
@@ -48,7 +43,7 @@ services:
       test:
         [
           "CMD-SHELL",
-          "PGPASSWORD=$$POSTGRES_PASSWORD psql -U $$POSTGRES_USER -d ai_db -tAc \"SELECT to_regclass('public.embeddings');\" | grep -q embeddings"
+          "PGPASSWORD=$$POSTGRES_PASSWORD psql -U $$POSTGRES_USER -d postgres -tAc \"SELECT 1 FROM pg_database WHERE datname='ai_db';\" | grep -q 1"
         ]
       interval: 10s
       timeout: 5s
@@ -89,8 +84,8 @@ services:
       retries: 3
     labels:
       - "traefik.enable=true"
-      - "traefik.http.routers.auth.rule=Host(`localhost`) && PathPrefix(`/api/v1/auth`)"
-      - "traefik.http.routers.auth.entrypoints=web"
+      - "traefik.http.routers.auth.rule=Host(`ec2-34-236-4-221.compute-1.amazonaws.com`) && PathPrefix(`/api/v1/auth`)"
+      - "traefik.http.routers.auth.entrypoints=websecure"
       - "traefik.http.services.auth.loadbalancer.server.port=8080"
 
   # Job management service
@@ -100,8 +95,6 @@ services:
     depends_on:
       postgresql:
         condition: service_healthy
-      service-auth:
-        condition: service_healthy
     networks:
       - ai-hr-network
     environment:
@@ -128,8 +121,8 @@ services:
       retries: 3
     labels:
       - "traefik.enable=true"
-      - "traefik.http.routers.job.rule=Host(`localhost`) && PathPrefix(`/api/v1/jobs`)"
-      - "traefik.http.routers.job.entrypoints=web"
+      - "traefik.http.routers.job.rule=Host(`ec2-34-236-4-221.compute-1.amazonaws.com`) && PathPrefix(`/api/v1/jobs`)"
+      - "traefik.http.routers.job.entrypoints=websecure"
       - "traefik.http.services.job.loadbalancer.server.port=8081"
 
   # Application management service
@@ -139,10 +132,6 @@ services:
     depends_on:
       postgresql:
         condition: service_healthy
-      service-auth:
-        condition: service_healthy
-      service-job:
-        condition: service_healthy
     networks:
       - ai-hr-network
     environment:
@@ -174,8 +163,8 @@ services:
       retries: 3
     labels:
       - "traefik.enable=true"
-      - "traefik.http.routers.application.rule=Host(`localhost`) && (PathPrefix(`/api/v1/applications`) || PathPrefix(`/api/v1/chat`) || PathPrefix(`/api/v1/assessments`) || PathPrefix(`/api/v1/files`))"
-      - "traefik.http.routers.application.entrypoints=web"
+      - "traefik.http.routers.application.rule=Host(`ec2-34-236-4-221.compute-1.amazonaws.com`) && (PathPrefix(`/api/v1/applications`) || PathPrefix(`/api/v1/chat`) || PathPrefix(`/api/v1/assessments`) || PathPrefix(`/api/v1/files`))"
+      - "traefik.http.routers.application.entrypoints=websecure"
       - "traefik.http.services.application.loadbalancer.server.port=8082"
 
   # GenAI service
@@ -190,8 +179,12 @@ services:
     environment:
       - PYTHONDONTWRITEBYTECODE=1
       - PYTHONUNBUFFERED=1
-      - OLLAMA_BASE_URL=${OLLAMA_BASE_URL} \
-      - OLLAMA_MODEL=${OLLAMA_MODEL} \
+      - DB_HOST=${DB_HOST:-postgresql}
+      - DB_PORT=${DB_PORT:-5432}
+      - DB_USERNAME=${DB_USERNAME:-postgres}
+      - DB_PASSWORD=${DB_PASSWORD:-postgres}
+      - OLLAMA_BASE_URL=${OLLAMA_BASE_URL}
+      - OLLAMA_MODEL=${OLLAMA_MODEL}
       - OLLAMA_API_KEY=${OLLAMA_API_KEY}
 
   # Frontend
@@ -202,8 +195,10 @@ services:
       - ai-hr-network
     labels:
       - "traefik.enable=true"
-      - "traefik.http.routers.frontend.rule=Host(`localhost`)"
-      - "traefik.http.routers.frontend.entrypoints=web"
+      - "traefik.http.routers.frontend.rule=Host(`ec2-34-236-4-221.compute-1.amazonaws.com`)"
+      - "traefik.http.routers.frontend.entrypoints=websecure"
+      - "traefik.http.routers.frontend.tls=true"
+      - "traefik.http.routers.frontend.tls.certresolver=letsencrypt"
       - "traefik.http.services.frontend.loadbalancer.server.port=80"
 
 networks:

docker-compose.yml

@@ -42,7 +42,7 @@ services:
       test:
         [
           "CMD-SHELL",
-          "PGPASSWORD=$$POSTGRES_PASSWORD psql -U $$POSTGRES_USER -d ai_db -tAc \"SELECT to_regclass('public.embeddings');\" | grep -q embeddings"
+          "PGPASSWORD=$$POSTGRES_PASSWORD psql -U $$POSTGRES_USER -d postgres -tAc \"SELECT 1 FROM pg_database WHERE datname='ai_db';\" | grep -q 1"
         ]
       interval: 10s
       timeout: 5s
@@ -186,6 +186,10 @@ services:
     environment:
       - PYTHONDONTWRITEBYTECODE=1
       - PYTHONUNBUFFERED=1
+      - DB_HOST=${DB_HOST:-postgresql}
+      - DB_PORT=${DB_PORT:-5432}
+      - DB_USERNAME=${DB_USERNAME:-postgres}
+      - DB_PASSWORD=${DB_PASSWORD:-postgres}
       - OLLAMA_BASE_URL=${OLLAMA_BASE_URL}
       - OLLAMA_MODEL=${OLLAMA_MODEL}
       - OLLAMA_API_KEY=${OLLAMA_API_KEY}

helm/aihr/charts/postgresql/templates/secret.yaml

@@ -28,6 +28,13 @@ spec:
         - name: application
           image: "{{ default $defaultRepo .Values.image.repository }}:{{ default $tag .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
           ports:
             - name: http
               containerPort: {{ .Values.service.port }}