Skip to content

Commit 568eed6

Browse files
Benji918Benji918
committed
Add GitHub Actions for CI/CD and improve analysis tasks
This commit introduces custom GitHub Actions for deploying, migrating, and starting the application, including modularized setup steps for Python and dependencies. Additionally, it refines analysis tasks by improving error handling, logging, and consistency checks, while adding a new `BankStatementSanitizer` class for sensitive data handling.
1 parent 18a3670 commit 568eed6

File tree

13 files changed

+825
-168
lines changed

13 files changed

+825
-168
lines changed

‎.env.example‎

Lines changed: 141 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,144 @@ SENTRY_DSN=your-sentry-dsn-here
4747
PROMETHEUS_METRICS=true
4848

4949
# Rate Limiting
50-
RATE_LIMIT_PER_MINUTE=60
50+
RATE_LIMIT_PER_MINUTE=60
51+
52+
name: Development Deployment Workflow
53+
54+
on:
55+
push:
56+
branches:
57+
- dev
58+
59+
jobs:
60+
dev-deployment:
61+
runs-on: ubuntu-latest
62+
63+
steps:
64+
- name: Checkout code
65+
uses: actions/checkout@v4
66+
67+
- name: Setup Python
68+
uses: actions/setup-python@v4
69+
with:
70+
python-version: '3.11'
71+
72+
- name: Deploy To Droplet
73+
uses: appleboy/ssh-action@v1
74+
with:
75+
host: ${{ secrets.DIGITALOCEAN_DROPLET_IP }}
76+
username: root
77+
key: ${{ secrets.DIGITALOCEAN_SSH_PRIVATE_KEY }}
78+
script: |
79+
set -e
80+
echo "🔄 Starting deployment..."
81+
82+
# Install PM2 if not already installed
83+
if ! command -v pm2 &> /dev/null; then
84+
echo "Installing PM2..."
85+
npm install -g pm2
86+
fi
87+
88+
# Check if the directory exists
89+
if [ -d "$HOME/Bank-Statment-Anaylser" ]; then
90+
echo "Directory 'Bank-Statment-Anaylser' exists....."
91+
fi
92+
93+
echo "Pulling repository changes..."
94+
cd $HOME/Bank-Statment-Anaylser
95+
96+
# Create .env file on the droplet
97+
echo "${{ secrets.ENV_FILE }}" > .env
98+
99+
# Ensure we're on the right branch
100+
git reset --hard HEAD
101+
git clean -fd
102+
git fetch origin dev
103+
git checkout dev
104+
git pull origin dev
105+
echo "✓ Git changes pulled"
106+
107+
# Create or reuse existing environment
108+
if [ ! -d "venv" ]; then
109+
python3.11 -m venv venv
110+
echo "✓ Virtual environment created"
111+
else
112+
echo "✓ Using existing virtual environment"
113+
fi
114+
115+
source venv/bin/activate
116+
echo "✓ Activated virtual environment"
117+
118+
REQUIREMENTS_FILE="/root/.previous_requirements_hash"
119+
120+
if [ ! -f "requirements.txt" ]; then
121+
echo "❌ requirements.txt not found!"
122+
exit 1
123+
fi
124+
125+
# Calculate hash of current requirements.txt
126+
CURRENT_HASH=$(md5sum requirements.txt | cut -d' ' -f1)
127+
128+
# Check if hash file exists and compare hashes
129+
if [ ! -f "$REQUIREMENTS_FILE" ] || [ "$(cat $REQUIREMENTS_FILE)" != "$CURRENT_HASH" ]; then
130+
echo "📦 Requirements changed, installing dependencies..."
131+
pip install -r requirements.txt
132+
# Save new hash
133+
echo "$CURRENT_HASH" > "$REQUIREMENTS_FILE"
134+
else
135+
echo "✓ Requirements unchanged, skipping installation"
136+
fi
137+
138+
ALEMBIC_CWD="$PWD/venv/bin/alembic"
139+
140+
if [ ! -f "$ALEMBIC_CWD" ]; then
141+
echo "Installing alembic..."
142+
pip install alembic
143+
fi
144+
145+
146+
# Check for multiple Alembic heads and merge them if necessary
147+
echo "Checking Alembic migrations..."
148+
alembic_heads=$($ALEMBIC_CWD heads | grep -v '^$' | wc -l)
149+
if [ "$alembic_heads" -gt 1 ]; then
150+
echo "Multiple Alembic heads detected. Merging heads..."
151+
$ALEMBIC_CWD merge heads -m "merge heads for deployment" || { echo "Failed to merge Alembic heads"; exit 1; }
152+
fi
153+
154+
$ALEMBIC_CWD upgrade head || { echo "Failed to run Alembic migrations"; exit 1; }
155+
echo "✓ Database migrations completed"
156+
157+
# Delete any existing pm2 processes
158+
pm2 delete all 2>/dev/null || true
159+
160+
# Verify ecosystem file exists
161+
if [ ! -f "ecosystem.config.js" ]; then
162+
echo "❌ ecosystem.config.js not found!"
163+
exit 1
164+
fi
165+
166+
echo "Starting application with pm2"
167+
pm2 start ecosystem.config.js || { echo "❌ PM2 failed to start application"; pm2 logs --lines 50; exit 1; }
168+
169+
# Save PM2 process list
170+
pm2 save
171+
172+
# Ensure PM2 starts on system reboot
173+
pm2 startup
174+
175+
# Check if the application is running
176+
if pm2 show bank-statement-analyzer | grep -q "online"; then
177+
echo "✅ Application successfully started with PM2 at $(date)"
178+
else
179+
echo "❌ Application failed to start"
180+
pm2 logs bank-statement-analyzer --lines 50
181+
exit 1
182+
fi
183+
184+
if pm2 show celery-worker | grep -q "online"; then
185+
echo "✅ Celery workers successfully started with PM2"
186+
else
187+
echo "❌ Celery workers failed to start"
188+
pm2 logs celery-worker --lines 50
189+
exit 1
190+
fi

‎.github/actions/deploy-code/action.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: 'Deploy Code'
2+
description: 'Deploy application code to the server'
3+
4+
inputs:
5+
host:
6+
description: 'Target server host/IP'
7+
required: true
8+
username:
9+
description: 'SSH username'
10+
required: true
11+
ssh-key:
12+
description: 'SSH private key'
13+
required: true
14+
env-file:
15+
description: 'Environment file contents'
16+
required: true
17+
branch:
18+
description: 'Git branch to deploy'
19+
required: true
20+
app-directory:
21+
description: 'Application directory name'
22+
required: true
23+
24+
runs:
25+
using: 'composite'
26+
steps:
27+
- name: Pull Repository Changes
28+
uses: appleboy/ssh-action@v1
29+
with:
30+
host: ${{ inputs.host }}
31+
username: ${{ inputs.username }}
32+
key: ${{ inputs.ssh-key }}
33+
script: |
34+
set -e
35+
echo "📥 Pulling repository changes..."
36+
37+
cd $HOME/${{ inputs.app-directory }}
38+
39+
# Create .env file on the droplet
40+
echo "${{ inputs.env-file }}" > .env
41+
echo "✓ Environment file created"
42+
43+
# Ensure we're on the right branch
44+
git reset --hard HEAD
45+
git clean -fd
46+
git fetch origin ${{ inputs.branch }}
47+
git checkout ${{ inputs.branch }}
48+
git pull origin ${{ inputs.branch }}
49+
echo "✓ Git changes pulled successfully"

‎.github/actions/deploy-to-server/action.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: 'Deploy To Server'
2+
description: 'Deploy application to remote server via SSH'
3+
4+
inputs:
5+
host:
6+
description: 'Target Server/VM'
7+
required: true
8+
9+
username:
10+
description: 'SSH username'
11+
required: true
12+
default: 'root'
13+
14+
ssh-key:
15+
description: 'SSH private key'
16+
required: true
17+
18+
env-file:
19+
description: 'Environment file contents'
20+
required: true
21+
22+
branch:
23+
description: 'Git branch to deploy'
24+
required: true
25+
default: 'dev'
26+
27+
app-directory:
28+
description: 'Application directory name'
29+
required: true
30+
31+
32+
33+
runs:
34+
using: 'composite'
35+
steps:
36+
- name: Setup Dependencies
37+
uses: ./.github/actions/setup-dependencies
38+
with:
39+
host: ${{ inputs.host }}
40+
username: ${{ inputs.username }}
41+
app-directory: ${{ inputs.app-directory }}
42+
43+
44+
- name: Deploy Code
45+
uses: ./.github/actions/deploy-code
46+
with:
47+
host: ${{ inputs.host }}
48+
username: ${{ inputs.username }}
49+
ssh-key: ${{ inputs.ssh-key }}
50+
env-file: ${{ inputs.env-file }}
51+
branch: ${{ inputs.branch }}
52+
app-directory: ${{ inputs.app-directory }}
53+
54+
55+
- name: Setup Python Environment
56+
uses: ./.github/actions/setup-python-env
57+
with:
58+
host: ${{ inputs.host }}
59+
username: ${{ inputs.username }}
60+
ssh-key: ${{ inputs.ssh-key }}
61+
app-directory: ${{ inputs.app-directory }}
62+
63+
64+
- name: Run Database Migrations
65+
uses: ./.github/actions/run-migrations
66+
with:
67+
host: ${{ inputs.host }}
68+
username: ${{ inputs.username }}
69+
ssh-key: ${{ inputs.ssh-key }}
70+
app-directory: ${{ inputs.app-directory }}
71+
72+
- name: Start Application
73+
uses: ./.github/actions/start-application
74+
with:
75+
host: ${{ inputs.host }}
76+
username: ${{ inputs.username }}
77+
ssh-key: ${{ inputs.ssh-key }}
78+
app-directory: ${{ inputs.app-directory }}
79+
80+
81+
82+

‎.github/actions/run-migrations/action.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: 'Run Database Migrations'
2+
description: 'Run Alembic database migrations'
3+
4+
inputs:
5+
host:
6+
description: 'Target server host/IP'
7+
required: true
8+
username:
9+
description: 'SSH username'
10+
required: true
11+
ssh-key:
12+
description: 'SSH private key'
13+
required: true
14+
app-directory:
15+
description: 'Application directory name'
16+
required: true
17+
18+
runs:
19+
using: 'composite'
20+
steps:
21+
- name: Execute Database Migrations
22+
uses: appleboy/ssh-action@v1
23+
with:
24+
host: ${{ inputs.host }}
25+
username: ${{ inputs.username }}
26+
key: ${{ inputs.ssh-key }}
27+
script: |
28+
set -e
29+
echo "🗄️ Running database migrations..."
30+
31+
cd $HOME/${{ inputs.app-directory }}
32+
source venv/bin/activate
33+
34+
ALEMBIC_CWD="$PWD/venv/bin/alembic"
35+
36+
# Check for multiple Alembic heads and merge them if necessary
37+
echo "Checking Alembic migrations..."
38+
alembic_heads=$($ALEMBIC_CWD heads | grep -v '^$' | wc -l)
39+
if [ "$alembic_heads" -gt 1 ]; then
40+
echo "Multiple Alembic heads detected. Merging heads..."
41+
$ALEMBIC_CWD merge heads -m "merge heads for deployment" || {
42+
echo "Failed to merge Alembic heads";
43+
exit 1;
44+
}
45+
echo "✓ Alembic heads merged"
46+
fi
47+
48+
$ALEMBIC_CWD upgrade head || {
49+
echo "Failed to run Alembic migrations";
50+
exit 1;
51+
}
52+
echo "✓ Database migrations completed"

‎.github/actions/setup-dependencies/action.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: 'Setup Dependencies'
2+
description: 'Install required system dependencies on the server'
3+
4+
inputs:
5+
host:
6+
description: 'Target server host/IP'
7+
required: true
8+
username:
9+
description: 'SSH username'
10+
required: true
11+
ssh-key:
12+
description: 'SSH private key'
13+
required: true
14+
15+
app-directory:
16+
description: 'Application directory name'
17+
required: true
18+
19+
20+
21+
runs:
22+
using: 'composite'
23+
steps:
24+
- name: Install PM2 and Check Directory
25+
uses: appleboy/ssh-action@v1
26+
with:
27+
host: ${{ inputs.host }}
28+
username: ${{ inputs.username }}
29+
key: ${{ inputs.ssh-key }}
30+
script: |
31+
set -e
32+
echo "🔄 Setting up dependencies..."
33+
34+
# Install PM2 if not already installed
35+
if ! command -v pm2 &> /dev/null; then
36+
echo "Installing PM2..."
37+
npm install -g pm2
38+
echo "✓ PM2 installed"
39+
else
40+
echo "✓ PM2 already installed"
41+
fi
42+
43+
# Check if the directory exists
44+
if [ -d "$HOME/${{ inputs.app-directory }}" ]; then
45+
echo "✓ Directory '${{ inputs.app-directory }} exists"
46+
else
47+
echo "❌ Directory '${{ inputs.app-directory }}' does not exist"
48+
exit 1
49+
fi

0 commit comments

Comments
 (0)