Skip to content

Commit dd96bef

Browse files
condo-botsitozzz
authored andcommitted
fix(condo): INFRA-1234 support legacy -d option
1 parent 77f6bba commit dd96bef

File tree

3 files changed

+132
-72
lines changed

3 files changed

+132
-72
lines changed

.github/workflows/_nodejs.condo.core.tests.yml

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
on:
22
workflow_call:
33
inputs:
4-
domain_name:
4+
shard_index:
55
required: true
6-
type: string
6+
type: number
7+
shard_total:
8+
required: true
9+
type: number
710
image_name:
811
required: true
912
type: string
@@ -56,12 +59,12 @@ jobs:
5659
-e GH_REF_NAME=$GH_REF_NAME \
5760
-v ./test_logs:/app/test_logs --network="host" \
5861
${{ secrets.DOCKER_REGISTRY }}/${{ inputs.image_name }} \
59-
bash -c "./bin/run_condo_domain_tests.sh -d ${{ inputs.domain_name }}"
62+
bash -c "./bin/run_condo_domain_tests.sh -s ${{ inputs.shard_index }} -t ${{ inputs.shard_total }}"
6063
env:
6164
PROMETHEUS_RW_SERVER_URL: ${{ secrets.K6_PROMETHEUS_RW_SERVER_URL }}
6265
PROMETHEUS_USER: ${{ secrets.K6_PROMETHEUS_USER }}
6366
PROMETHEUS_PASSWORD: ${{ secrets.K6_PROMETHEUS_PASSWORD }}
64-
GH_REF_NAME: ${{ github.event.pull_request.head.ref || github.ref_name }}
67+
GH_REF_NAME: ${{ github.head_ref || github.ref_name }}
6568

6669
- name: Collect docker logs on failure
6770
if: failure()
@@ -73,7 +76,7 @@ jobs:
7376
uses: actions/upload-artifact@v4
7477
if: failure()
7578
with:
76-
name: logs-${{ inputs.domain_name }}-${{ env.GITHUB_RUN_ID }}-${{ env.GITHUB_RUN_NUMBER }}
79+
name: logs-shard-${{ inputs.shard_index }}-of-${{ inputs.shard_total }}-${{ env.GITHUB_RUN_ID }}-${{ env.GITHUB_RUN_NUMBER }}
7780
path: |
7881
./test_logs/*
7982
*.log

.github/workflows/nodejs.condo.ci.yml

Lines changed: 4 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@ on:
55
branches:
66
- main
77

8-
# NOTE: change pull_request_target to pull_request if you want to change the CI in your branch
9-
pull_request_target:
8+
pull_request:
109
types:
1110
- opened
1211
- reopened
@@ -424,25 +423,11 @@ jobs:
424423
# TODO: remove this back
425424
fail-fast: false
426425
matrix:
427-
domain:
428-
- organization
429-
- user
430-
- scope
431-
- property
432-
- acquiring
433-
- billing
434-
- miniapp
435-
- banking
436-
- ticket
437-
- meter
438-
- contact
439-
- resident
440-
- notification
441-
- common
442-
- others
426+
shard_index: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
443427
uses: ./.github/workflows/_nodejs.condo.core.tests.yml
444428
with:
445-
domain_name: ${{ matrix.domain }}
429+
shard_index: ${{ matrix.shard_index }}
430+
shard_total: 16
446431
runs-on: test-runners
447432
image_name: ${{ needs.build-image.outputs.DOCKER_IMAGE }}
448433
secrets: inherit

bin/run_condo_domain_tests.sh

100644100755
Lines changed: 120 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,62 @@
33
set -ex
44

55
domain_name=""
6+
shard_index=""
7+
shard_total=""
68

79
usage() {
8-
echo "Usage: $0 -d <domain_name>"
10+
echo "Usage: $0 -d <domain_name> | -s <shard_index> -t <shard_total>"
911
exit 1
1012
}
1113

12-
while getopts "d:" opt; do
14+
setup_and_start_services() {
15+
node bin/prepare.js -f condo -r condo -c condo
16+
17+
export NEWS_ITEMS_SENDING_DELAY_SEC=2
18+
export NEWS_ITEM_SENDING_TTL_SEC=2
19+
export NODE_OPTIONS="--max_old_space_size=4192"
20+
export WORKER_CONCURRENCY=100
21+
export DATABASE_POOL_MAX=10
22+
23+
node -e 'console.log(v8.getHeapStatistics().heap_size_limit/(1024*1024))'
24+
25+
# NOTE(pahaz): Keystone not in dev mode trying to check dist/admin folder
26+
mkdir -p ./apps/condo/dist/admin
27+
28+
yarn workspace @app/condo start 2>&1 > /app/test_logs/condo.dev.log &
29+
30+
node bin/wait-apps-apis.js -f condo
31+
32+
# check migrations
33+
yarn workspace @app/condo makemigrations --check &> /dev/null
34+
35+
source bin/validate-db-schema-ts-to-match-graphql-api.sh
36+
37+
yarn workspace @app/condo worker 2>&1 > /app/test_logs/condo.worker.log &
38+
sleep 3
39+
40+
# And check background processes!
41+
[[ $(jobs | wc -l | tr -d ' ') != '2' ]] && exit 2
42+
sleep 3
43+
}
44+
45+
stop_services() {
46+
# Note: we need to stop background worker! because packages tests use the same redis queue
47+
kill $(jobs -p) || echo 'background worker and dev server is already killed!'
48+
killall node || echo 'no node processes'
49+
}
50+
51+
while getopts ":d:s:t:" opt; do
1352
case $opt in
1453
d)
1554
domain_name="$OPTARG"
1655
;;
56+
s)
57+
shard_index="$OPTARG"
58+
;;
59+
t)
60+
shard_total="$OPTARG"
61+
;;
1762
\?)
1863
echo "Invalid option: -$OPTARG" >&2
1964
usage
@@ -25,62 +70,89 @@ while getopts "d:" opt; do
2570
esac
2671
done
2772

28-
if [ -z "$domain_name" ]; then
29-
echo "-d is a required argument!"
73+
if [ -n "$domain_name" ] && { [ -n "$shard_index" ] || [ -n "$shard_total" ]; }; then
74+
echo "Use either legacy domain mode (-d) or shard mode (-s/-t), not both"
3075
usage
3176
fi
3277

33-
node bin/prepare.js -f condo -r condo -c condo
34-
35-
export NEWS_ITEMS_SENDING_DELAY_SEC=2
36-
export NEWS_ITEM_SENDING_TTL_SEC=2
37-
export NODE_OPTIONS="--max_old_space_size=4192"
38-
export WORKER_CONCURRENCY=100
39-
export DATABASE_POOL_MAX=10
40-
41-
node -e 'console.log(v8.getHeapStatistics().heap_size_limit/(1024*1024))'
42-
43-
# NOTE(pahaz): Keystone not in dev mode trying to check dist/admin folder
44-
mkdir -p ./apps/condo/dist/admin
45-
46-
yarn workspace @app/condo start 2>&1 > /app/test_logs/condo.dev.log &
78+
if [ -n "$domain_name" ]; then
79+
setup_and_start_services
80+
81+
if [ "$domain_name" != 'others' ]; then
82+
# TESTS
83+
yarn workspace @app/condo test --workerIdleMemoryLimit="256MB" --testTimeout=15000 -w=3 --forceExit --silent=false --verbose --bail --testPathPattern '/domains/'$domain_name'/schema/(.*)[.]test.js$' 2>&1 > '/app/test_logs/condo.'$domain_name'.tests.log'
84+
# SPECS
85+
if [ -n "$(find apps/condo/domains/$domain_name -name '*spec.[j|t]s' 2>/dev/null)" ]; then
86+
yarn workspace @app/condo test --workerIdleMemoryLimit="256MB" --testTimeout=15000 -w=2 --forceExit --silent=false --verbose --bail --testPathPattern '/domains/'$domain_name'/(.*)[.]spec.[j|t]s$' 2>&1 > '/app/test_logs/condo.'$domain_name'.specs.log'
87+
else
88+
echo "Files matching (.*)[.]spec.[j|t]s in directory apps/condo/domains/$domain_name not found! Skipping..."
89+
fi
90+
else
91+
# TESTS
92+
yarn workspace @app/condo test --workerIdleMemoryLimit="256MB" --testTimeout=15000 -w=3 --forceExit --silent=false --verbose --bail --testPathPattern '/schema/(.*)[.]test.js$' --testPathIgnorePatterns='/domains/(organization|user|scope|property|acquiring|billing|miniapp|banking|ticket|meter|contact|resident|notification|common)/' 2>&1 > /app/test_logs/condo.others.tests.log
93+
yarn workspace @app/condo test --workerIdleMemoryLimit="256MB" --testTimeout=15000 -w=3 --forceExit --silent=false --verbose --bail --testPathPattern '(.*)[.]test.js$' --testPathIgnorePatterns='/schema/(.*)[.]test.js$' 2>&1 > /app/test_logs/condo.5.test.others.log
94+
# SPECS
95+
yarn workspace @app/condo test --workerIdleMemoryLimit="256MB" --testTimeout=15000 -w=2 --forceExit --silent=false --verbose --bail --testPathPattern '(.*)[.]spec.[j|t]s$' --testPathIgnorePatterns='/domains/(organization|user|scope|property|acquiring|billing|miniapp|banking|ticket|meter|contact|resident|notification|common)/' 2>&1 > /app/test_logs/condo.others.specs.log
96+
fi
4797

48-
node bin/wait-apps-apis.js -f condo
98+
stop_services
4999

50-
# check migrations
51-
yarn workspace @app/condo makemigrations --check &> /dev/null
100+
if [ "$domain_name" = 'others' ]; then
101+
# TODO: INFRA-155 Remove it completely by rewriting a task tests or migrate to jest.setup or smth
102+
REDIS_URL='[{"port":7001,"host":"127.0.0.1"},{"port":7002,"host":"127.0.0.1"},{"port":7003,"host":"127.0.0.1"}]' yarn workspace @open-condo/keystone test
103+
yarn workspace @app/condo lint-schema
104+
fi
52105

53-
source bin/validate-db-schema-ts-to-match-graphql-api.sh
106+
exit 0
107+
fi
54108

55-
yarn workspace @app/condo worker 2>&1 > /app/test_logs/condo.worker.log &
56-
sleep 3
109+
if [ -z "$shard_index" ] || [ -z "$shard_total" ]; then
110+
echo "-s and -t are required arguments!"
111+
usage
112+
fi
57113

58-
# And check background processes!
59-
[[ $(jobs | wc -l | tr -d ' ') != '2' ]] && exit 2
60-
sleep 3
114+
if ! [[ "$shard_index" =~ ^[0-9]+$ ]] || ! [[ "$shard_total" =~ ^[0-9]+$ ]]; then
115+
echo "-s and -t must be positive numbers"
116+
exit 1
117+
fi
61118

62-
if [ $domain_name != "others" ]; then
63-
# TESTS
64-
yarn workspace @app/condo test --workerIdleMemoryLimit="256MB" --testTimeout=15000 -w=3 --forceExit --silent=false --verbose --bail --testPathPattern '/domains/'$domain_name'/schema/(.*)[.]test.js$' 2>&1 > '/app/test_logs/condo.'$domain_name'.tests.log'
65-
# SPECS
66-
if [ -n "$(find apps/condo/domains/$domain_name -name '*spec.[j|t]s' 2>/dev/null)" ]; then
67-
yarn workspace @app/condo test --workerIdleMemoryLimit="256MB" --testTimeout=15000 -w=2 --forceExit --silent=false --verbose --bail --testPathPattern '/domains/'$domain_name'/(.*)[.]spec.[j|t]s$' 2>&1 > '/app/test_logs/condo.'$domain_name'.specs.log'
68-
else
69-
echo "Files matching (.*)[.]spec.[j|t]s in directory apps/condo/domains/$domain_name not found! Skipping..."
70-
fi
71-
# Note: we need to stop background worker! because packages tests use the same redis queue
72-
kill $(jobs -p) || echo 'background worker and dev server is already killed!'
73-
killall node || echo 'no node processes'
74-
else
75-
# TESTS
76-
yarn workspace @app/condo test --workerIdleMemoryLimit="256MB" --testTimeout=15000 -w=3 --forceExit --silent=false --verbose --bail --testPathPattern '/schema/(.*)[.]test.js$' --testPathIgnorePatterns='/domains/(organization|user|scope|property|acquiring|billing|miniapp|banking|ticket|meter|contact|resident|notification|common)/' 2>&1 > /app/test_logs/condo.others.tests.log
77-
yarn workspace @app/condo test --workerIdleMemoryLimit="256MB" --testTimeout=15000 -w=3 --forceExit --silent=false --verbose --bail --testPathPattern '(.*)[.]test.js$' --testPathIgnorePatterns='/schema/(.*)[.]test.js$' 2>&1 > /app/test_logs/condo.5.test.others.log
78-
# SPECS
79-
yarn workspace @app/condo test --workerIdleMemoryLimit="256MB" --testTimeout=15000 -w=2 --forceExit --silent=false --verbose --bail --testPathPattern '(.*)[.]spec.[j|t]s$' --testPathIgnorePatterns='/domains/(organization|user|scope|property|acquiring|billing|miniapp|banking|ticket|meter|contact|resident|notification|common)/' 2>&1 > /app/test_logs/condo.others.specs.log
80-
# Note: we need to stop background worker! because packages tests use the same redis queue
81-
kill $(jobs -p) || echo 'background worker and dev server is already killed!'
82-
killall node || echo 'no node processes'
119+
if [ "$shard_index" -lt 1 ] || [ "$shard_total" -lt 1 ] || [ "$shard_index" -gt "$shard_total" ]; then
120+
echo "Invalid shard values: shard_index=$shard_index shard_total=$shard_total"
121+
exit 1
122+
fi
83123

124+
setup_and_start_services
125+
126+
# TESTS (.test.js)
127+
yarn workspace @app/condo test \
128+
--workerIdleMemoryLimit="256MB" \
129+
--testTimeout=15000 \
130+
-w=3 \
131+
--forceExit \
132+
--silent=false \
133+
--verbose \
134+
--bail \
135+
--testPathPattern '(.*)[.]test.js$' \
136+
--shard="$shard_index/$shard_total" \
137+
2>&1 > "/app/test_logs/condo.shard-${shard_index}-of-${shard_total}.tests.log"
138+
139+
# SPECS (.spec.js|.spec.ts)
140+
yarn workspace @app/condo test \
141+
--workerIdleMemoryLimit="256MB" \
142+
--testTimeout=15000 \
143+
-w=2 \
144+
--forceExit \
145+
--silent=false \
146+
--verbose \
147+
--bail \
148+
--testPathPattern '(.*)[.]spec.[j|t]s$' \
149+
--shard="$shard_index/$shard_total" \
150+
2>&1 > "/app/test_logs/condo.shard-${shard_index}-of-${shard_total}.specs.log"
151+
152+
stop_services
153+
154+
# Run non-domain tests that were previously executed in "others"
155+
if [ "$shard_index" -eq 1 ]; then
84156
# TODO: INFRA-155 Remove it completely by rewriting a task tests or migrate to jest.setup or smth
85157
REDIS_URL='[{"port":7001,"host":"127.0.0.1"},{"port":7002,"host":"127.0.0.1"},{"port":7003,"host":"127.0.0.1"}]' yarn workspace @open-condo/keystone test
86158
yarn workspace @app/condo lint-schema

0 commit comments

Comments
 (0)