Skip to content

Commit 9cf45e5

Browse files
jjaeroongjjaeroong
authored
[OFFNAL-27/refactor][jjaeroong]: cicd 가중치 설정 수정
[OFFNAL-27/refactor][jjaeroong]: cicd 가중치 설정 수정
2 parents 9c4e06b + c0ef238 commit 9cf45e5

File tree

1 file changed

+128
-110
lines changed

1 file changed

+128
-110
lines changed

‎.github/workflows/deploy-prod.yml‎

Lines changed: 128 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -228,12 +228,30 @@ jobs:
228228
set_weight() {
229229
local blue_weight=$1
230230
local green_weight=$2
231-
231+
232232
cp "$UPSTREAM_FILE" "$UPSTREAM_FILE.bak"
233-
234-
sed -i -E "s/(server spring_blue:8080[[:space:]]+weight=)[0-9]+/\1${blue_weight}/" "$UPSTREAM_FILE"
235-
sed -i -E "s/(server spring_green:8080[[:space:]]+weight=)[0-9]+/\1${green_weight}/" "$UPSTREAM_FILE"
236-
233+
234+
# weight=0 방지
235+
local bw="$blue_weight"
236+
local gw="$green_weight"
237+
if [ "$bw" -eq 0 ]; then bw=1; fi
238+
if [ "$gw" -eq 0 ]; then gw=1; fi
239+
240+
# weight 숫자 갱신
241+
sed -i -E "s/(server spring_blue:8080[[:space:]]+weight=)[0-9]+/\1${bw}/" "$UPSTREAM_FILE"
242+
sed -i -E "s/(server spring_green:8080[[:space:]]+weight=)[0-9]+/\1${gw}/" "$UPSTREAM_FILE"
243+
244+
if [ "$blue_weight" -eq 100 ] && [ "$green_weight" -eq 0 ]; then
245+
sed -i -E "s/(server spring_green:8080[^;]*);/\1 down;/" "$UPSTREAM_FILE"
246+
sed -i -E "s/(server spring_blue:8080[^;]*) down;/\1;/" "$UPSTREAM_FILE"
247+
elif [ "$blue_weight" -eq 0 ] && [ "$green_weight" -eq 100 ]; then
248+
sed -i -E "s/(server spring_blue:8080[^;]*);/\1 down;/" "$UPSTREAM_FILE"
249+
sed -i -E "s/(server spring_green:8080[^;]*) down;/\1;/" "$UPSTREAM_FILE"
250+
else
251+
sed -i -E "s/(server spring_blue:8080[^;]*) down;/\1;/" "$UPSTREAM_FILE"
252+
sed -i -E "s/(server spring_green:8080[^;]*) down;/\1;/" "$UPSTREAM_FILE"
253+
fi
254+
237255
if sudo docker-compose exec nginx nginx -t; then
238256
sudo docker-compose exec nginx nginx -s reload
239257
return 0
@@ -248,140 +266,140 @@ jobs:
248266
# ----------------------------
249267
# Canary 모니터링 함수 ( /login/page 기준, 추후 기준 api 변경)
250268
# ----------------------------
251-
monitor_canary() {
252-
local duration=$1
253-
local endpoint="/login/page"
269+
monitor_canary() {
270+
local duration=$1
271+
local endpoint="/login/page"
254272
255-
256-
local host_header
257-
host_header=$(grep -m1 -E "^[[:space:]]*server_name[[:space:]]+" "$UPSTREAM_FILE" \
258-
| awk '{print $2}' | tr -d ';')
259273
260-
# 혹시 못 찾으면 fallback (그래도 dev/prod 둘 다 이걸로는 보통 안전)
274+
local host_header
275+
host_header=$(grep -m1 -E "^[[:space:]]*server_name[[:space:]]+" "$UPSTREAM_FILE" \
276+
| awk '{print $2}' | tr -d ';')
277+
278+
# 혹시 못 찾으면 fallback
261279
if [ -z "$host_header" ]; then
262280
host_header="localhost"
263281
fi
264-
282+
265283
echo "모니터링 중 (${duration}초)... endpoint=${endpoint}, host=${host_header}"
266-
284+
267285
local fail_count=0
268286
local total_checks=$((duration / 3))
269-
287+
270288
for i in $(seq 1 $total_checks); do
271289
sleep 3
272290
273291
# 80은 301로 https로 보내므로 200~399면 OK로 처리
274292
local code
275-
code=$(curl -sS -o /dev/null -w "%{http_code}" -H "Host: ${host_header}" "http://localhost${endpoint}" || echo "000")
276-
277-
if [ "$code" -ge 200 ] && [ "$code" -lt 400 ]; then
278-
echo "✓ /login/page OK (code=$code) [$i/$total_checks]"
279-
else
280-
fail_count=$((fail_count + 1))
281-
echo "❌ /login/page FAIL (code=$code) (fail=$fail_count) [$i/$total_checks]"
282-
fi
283-
284-
if [ "$fail_count" -gt 3 ]; then
285-
echo "에러 과다 발생 (/login/page 기준)"
286-
return 1
287-
fi
288-
done
289-
290-
echo "모니터링 완료 (/login/page 기준)"
291-
return 0
293+
code=$(curl -sS -o /dev/null -w "%{http_code}" -H "Host: ${host_header}" "http://localhost${endpoint}" || echo "000")
294+
295+
if [ "$code" -ge 200 ] && [ "$code" -lt 400 ]; then
296+
echo "✓ /login/page OK (code=$code) [$i/$total_checks]"
297+
else
298+
fail_count=$((fail_count + 1))
299+
echo "❌ /login/page FAIL (code=$code) (fail=$fail_count) [$i/$total_checks]"
300+
fi
301+
302+
if [ "$fail_count" -gt 3 ]; then
303+
echo "에러 과다 발생 (/login/page 기준)"
304+
return 1
305+
fi
306+
done
307+
308+
echo "모니터링 완료 (/login/page 기준)"
309+
return 0
292310
}
293-
294-
# ----------------------------
295-
# 9) Canary Phase 1: 10% 트래픽
296-
# ----------------------------
297-
echo "🐤 1단계: 10% 트래픽 전환"
298-
299-
if [ "$NEW_COLOR" = "blue" ]; then
311+
312+
# ----------------------------
313+
# 9) Canary Phase 1: 10% 트래픽
314+
# ----------------------------
315+
echo "1단계: 10% 트래픽 전환"
316+
317+
if [ "$NEW_COLOR" = "blue" ]; then
300318
if ! set_weight 10 90; then
301-
echo "1단계 weight 설정 실패"
302-
sudo docker-compose stop "$NEW_SERVICE" || true
303-
sudo docker-compose rm -f "$NEW_SERVICE" || true
304-
exit 1
319+
echo "1단계 weight 설정 실패"
320+
sudo docker-compose stop "$NEW_SERVICE" || true
321+
sudo docker-compose rm -f "$NEW_SERVICE" || true
322+
exit 1
305323
fi
306-
else
324+
else
307325
if ! set_weight 90 10; then
308-
echo "1단계 weight 설정 실패"
309-
sudo docker-compose stop "$NEW_SERVICE" || true
310-
sudo docker-compose rm -f "$NEW_SERVICE" || true
311-
exit 1
326+
echo "1단계 weight 설정 실패"
327+
sudo docker-compose stop "$NEW_SERVICE" || true
328+
sudo docker-compose rm -f "$NEW_SERVICE" || true
329+
exit 1
312330
fi
313-
fi
314-
315-
if ! monitor_canary 30; then
331+
fi
332+
333+
if ! monitor_canary 30; then
316334
echo "❌ 1단계 모니터링 실패, 롤백"
317-
335+
318336
# 트래픽 복구
319337
if [ "$CURRENT_COLOR" = "blue" ]; then
320-
set_weight 100 0 || true
338+
set_weight 100 0 || true
321339
else
322-
set_weight 0 100 || true
340+
set_weight 0 100 || true
323341
fi
324-
342+
325343
sudo docker-compose stop "$NEW_SERVICE" || true
326344
sudo docker-compose rm -f "$NEW_SERVICE" || true
327345
exit 1
328-
fi
329-
330-
# ----------------------------
331-
# 10) Full Switch: 100% 트래픽
332-
# ----------------------------
333-
echo "🚀 2단계: 100% 트래픽 전환"
334-
335-
if [ "$NEW_COLOR" = "blue" ]; then
346+
fi
347+
348+
# ----------------------------
349+
# 10) Full Switch: 100% 트래픽
350+
# ----------------------------
351+
echo "2단계: 100% 트래픽 전환"
352+
353+
if [ "$NEW_COLOR" = "blue" ]; then
336354
if ! set_weight 100 0; then
337-
echo "❌ 전체 전환 실패"
338-
# 10%로 되돌림
339-
set_weight 10 90 || true
340-
exit 1
355+
echo "전체 전환 실패"
356+
# 10%로 되돌림
357+
set_weight 10 90 || true
358+
exit 1
341359
fi
342-
else
360+
else
343361
if ! set_weight 0 100; then
344-
echo "❌ 전체 전환 실패"
345-
# 10%로 되돌림
346-
set_weight 90 10 || true
347-
exit 1
362+
echo "전체 전환 실패"
363+
# 10%로 되돌림
364+
set_weight 90 10 || true
365+
exit 1
348366
fi
349-
fi
350-
351-
rm -f "$UPSTREAM_FILE.bak"
352-
353-
# ----------------------------
354-
# 11) 최종 검증
355-
# ----------------------------
356-
echo "최종 검증 중..."
357-
sleep 10
358-
359-
if ! curl -fsS "http://localhost:${NEW_PORT}/actuator/health" > /dev/null 2>&1; then
360-
echo "⚠️ 최종 Health check 실패"
361-
fi
362-
363-
# ----------------------------
364-
# 12) 기존 서비스 종료 (60초 유예)
365-
# ----------------------------
366-
echo "$CURRENT_COLOR 종료 대기 중 (60초 유예)..."
367-
for i in {1..12}; do
367+
fi
368+
369+
rm -f "$UPSTREAM_FILE.bak"
370+
371+
# ----------------------------
372+
# 11) 최종 검증
373+
# ----------------------------
374+
echo "최종 검증 중..."
375+
sleep 10
376+
377+
if ! curl -fsS "http://localhost:${NEW_PORT}/actuator/health" > /dev/null 2>&1; then
378+
echo "최종 Health check 실패"
379+
fi
380+
381+
# ----------------------------
382+
# 12) 기존 서비스 종료 (60초 유예)
383+
# ----------------------------
384+
echo "$CURRENT_COLOR 종료 대기 중 (60초 유예)..."
385+
for i in {1..12}; do
368386
echo -n "."
369387
sleep 5
370-
done
371-
echo ""
372-
373-
sudo docker-compose stop -t 30 "$CURRENT_SERVICE" || true
374-
sudo docker-compose rm -f "$CURRENT_SERVICE" || true
375-
376-
# ----------------------------
377-
# 13) 배포 완료
378-
# ----------------------------
379-
echo ""
380-
echo "=========================================="
381-
echo "✅ 배포 완료"
382-
echo "=========================================="
383-
echo "시간: $(date '+%Y-%m-%d %H:%M:%S')"
384-
echo "활성: $NEW_COLOR ($NEW_SERVICE:$NEW_PORT)"
385-
echo "종료: $CURRENT_COLOR ($CURRENT_SERVICE)"
386-
echo "단계: 10% → 100%"
387-
echo "=========================================="
388+
done
389+
echo ""
390+
391+
sudo docker-compose stop -t 30 "$CURRENT_SERVICE" || true
392+
sudo docker-compose rm -f "$CURRENT_SERVICE" || true
393+
394+
# ----------------------------
395+
# 13) 배포 완료
396+
# ----------------------------
397+
echo ""
398+
echo "=========================================="
399+
echo "✅ 배포 완료"
400+
echo "=========================================="
401+
echo "시간: $(date '+%Y-%m-%d %H:%M:%S')"
402+
echo "활성: $NEW_COLOR ($NEW_SERVICE:$NEW_PORT)"
403+
echo "종료: $CURRENT_COLOR ($CURRENT_SERVICE)"
404+
echo "단계: 10% → 100%"
405+
echo "=========================================="

0 commit comments

Comments
 (0)