Skip to content

Commit 08d65ca

Browse files
pinglinpinglin
committed
ci(workflows): run integration test on only one dummy model
1 parent 9351fb9 commit 08d65ca

File tree

5 files changed

+278
-140
lines changed

5 files changed

+278
-140
lines changed

‎.github/workflows/integration-test.yml‎

Lines changed: 97 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ on:
77
- "**"
88

99
jobs:
10-
1110
compose:
1211
runs-on: ubuntu-latest
1312
steps:
@@ -138,10 +137,106 @@ jobs:
138137
fetch-depth: 0
139138
fetch-tags: true
140139

140+
- name: Load .env file
141+
uses: cardinalby/export-env-action@v2
142+
with:
143+
envFile: instill-core/.env
144+
141145
- name: Run model integration test
142146
working-directory: instill-core
143147
run: |
144-
make model-integration-test
148+
set -x
149+
echo "=== Starting model integration test ==="
150+
151+
# Load .env to ensure all variables are set
152+
set -a
153+
source .env
154+
set +a
155+
156+
# Start registry and wait for it
157+
echo "Starting registry..."
158+
EDITION=docker-ce:test docker compose up registry -d
159+
160+
echo "Waiting for registry to be ready..."
161+
timeout=60
162+
while ! curl -sf http://localhost:5001/v2/ >/dev/null 2>&1; do
163+
if [ $timeout -le 0 ]; then
164+
echo "Registry failed to start"
165+
docker logs registry
166+
exit 1
167+
fi
168+
sleep 1
169+
timeout=$((timeout-1))
170+
done
171+
echo "Registry is ready"
172+
173+
# Build and push models
174+
echo "Building and pushing models..."
175+
make build-and-push-models
176+
177+
# Verify models were pushed
178+
echo "Verifying models in registry..."
179+
curl -s http://localhost:5001/v2/_catalog | jq '.'
180+
181+
# Start all services
182+
echo "Starting Instill Core with model init..."
183+
make compose-dev EDITION=docker-ce:test INITMODEL_ENABLED=true INITMODEL_INVENTORY=${PWD}/integration-test/models/inventory.json
184+
185+
# Wait for services to stabilize
186+
echo "Waiting for services to be healthy..."
187+
sleep 30
188+
189+
# Check services status
190+
echo "=== Docker containers status ==="
191+
docker ps -a
192+
193+
# Check model-backend-init-model
194+
echo "=== Checking model-backend-init-model ==="
195+
docker logs model-backend-init-model || echo "Init model container not started yet"
196+
197+
# Check Ray status
198+
echo "=== Checking Ray status ==="
199+
docker exec ray ray status || echo "Ray status check failed"
200+
201+
# Check Ray applications
202+
echo "=== Checking Ray applications ==="
203+
docker run --rm --network instill-network curlimages/curl:latest \
204+
curl -s http://ray:8265/api/serve/applications/ | jq '.' || echo "Ray API call failed"
205+
206+
# Now wait for models to deploy
207+
echo "Waiting for models to deploy..."
208+
make wait-models-deploy
209+
210+
- name: Debug on failure
211+
if: failure()
212+
working-directory: instill-core
213+
run: |
214+
echo "=== DEBUGGING FAILED TEST ==="
215+
216+
echo -e "\n=== Docker containers ==="
217+
docker ps -a
218+
219+
echo -e "\n=== System resources ==="
220+
docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"
221+
222+
echo -e "\n=== model-backend logs ==="
223+
docker logs model-backend --tail 100 || true
224+
225+
echo -e "\n=== model-backend-worker logs ==="
226+
docker logs model-backend-worker --tail 100 || true
227+
228+
echo -e "\n=== model-backend-init-model logs ==="
229+
docker logs model-backend-init-model --tail 100 || true
230+
231+
echo -e "\n=== Ray logs ==="
232+
docker logs ray --tail 200 || true
233+
234+
echo -e "\n=== Registry logs ==="
235+
docker logs registry --tail 50 || true
236+
237+
echo -e "\n=== Ray Serve applications ==="
238+
docker run --rm --network instill-network curlimages/curl:latest \
239+
curl -s http://ray:8265/api/serve/applications/ | jq '.' || true
145240
146241
- name: Tear down Instill Core
147242
if: always()

‎Makefile‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,12 +153,24 @@ build-and-push-models: # Helper target to build and push models
153153
.PHONY: wait-models-deploy
154154
wait-models-deploy: # Helper target to wait for model deployment
155155
@model_count=$$(jq length integration-test/models/inventory.json); \
156-
timeout=1800; elapsed=0; spinner='|/-\\'; i=0; \
156+
echo "Waiting for $$model_count model(s) to deploy..."; \
157+
timeout=1800; elapsed=0; spinner='|/-\\'; i=0; debug_interval=30; \
157158
while [ "$$(docker run --rm --network instill-network curlimages/curl:latest curl -s http://${RAY_HOST}:${RAY_PORT_DASHBOARD}/api/serve/applications/ | jq ".applications | to_entries | map(select(.key | contains(\"dummy-\")) | .value.status) | length == $$model_count and all(. == \"RUNNING\")")" != "true" ]; do \
158-
running_count=$$(docker run --rm --network instill-network curlimages/curl:latest curl -s http://${RAY_HOST}:${RAY_PORT_DASHBOARD}/api/serve/applications/ | jq '.applications | to_entries | map(select(.key | contains("dummy-")) | .value.status) | map(select(. == "RUNNING")) | length'); \
159+
ray_response=$$(docker run --rm --network instill-network curlimages/curl:latest curl -s http://${RAY_HOST}:${RAY_PORT_DASHBOARD}/api/serve/applications/); \
160+
running_count=$$(echo "$$ray_response" | jq '.applications | to_entries | map(select(.key | contains("dummy-")) | .value.status) | map(select(. == "RUNNING")) | length'); \
159161
printf "\r[Waiting %3ds/%ds] %s models still deploying... (%d/%d RUNNING)" "$$elapsed" "$$timeout" "$${spinner:$$((i % 4)):1}" "$$running_count" "$$model_count"; \
162+
if [ $$((elapsed % debug_interval)) -eq 0 ] && [ "$$elapsed" -gt 0 ]; then \
163+
echo ""; \
164+
echo "=== Debug info at $${elapsed}s ==="; \
165+
echo "Ray applications response:"; \
166+
echo "$$ray_response" | jq '.applications | to_entries | map({key: .key, status: .value.status, message: .value.message})' || echo "$$ray_response"; \
167+
echo "Expected models from inventory:"; \
168+
jq -r '.[].id' integration-test/models/inventory.json; \
169+
fi; \
160170
sleep 1; elapsed=$$((elapsed+1)); \
161171
if [ "$$elapsed" -ge "$$timeout" ]; then \
172+
echo "\n=== TIMEOUT - Final Ray state ==="; \
173+
docker run --rm --network instill-network curlimages/curl:latest curl -s http://${RAY_HOST}:${RAY_PORT_DASHBOARD}/api/serve/applications/ | jq '.'; \
162174
echo "\nTimeout waiting for models to deploy!"; exit 1; \
163175
fi; \
164176
i=$$((i + 1)); \

‎integration-test/models/inventory-all.json‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
[
2+
{
3+
"id": "dummy-cls",
4+
"owner_type": "users",
5+
"owner_id": "admin",
6+
"description": "A classification model for integration-test",
7+
"model_definition": "model-definitions/container",
8+
"visibility": "VISIBILITY_PUBLIC",
9+
"task": "TASK_CLASSIFICATION",
10+
"region": "REGION_LOCAL",
11+
"hardware": "CPU",
12+
"version": "dev",
13+
"configuration": {}
14+
},
15+
{
16+
"id": "dummy-det",
17+
"owner_type": "users",
18+
"owner_id": "admin",
19+
"description": "A object detection model for integration-test",
20+
"model_definition": "model-definitions/container",
21+
"visibility": "VISIBILITY_PUBLIC",
22+
"task": "TASK_DETECTION",
23+
"region": "REGION_LOCAL",
24+
"hardware": "CPU",
25+
"version": "dev",
26+
"configuration": {}
27+
},
28+
{
29+
"id": "dummy-instance-segmentation",
30+
"owner_type": "users",
31+
"owner_id": "admin",
32+
"description": "A instance segmentation model for integration-test",
33+
"model_definition": "model-definitions/container",
34+
"visibility": "VISIBILITY_PUBLIC",
35+
"task": "TASK_INSTANCE_SEGMENTATION",
36+
"region": "REGION_LOCAL",
37+
"hardware": "CPU",
38+
"version": "dev",
39+
"configuration": {}
40+
},
41+
{
42+
"id": "dummy-keypoint",
43+
"owner_type": "users",
44+
"owner_id": "admin",
45+
"description": "A keypoint detection model for integration-test",
46+
"model_definition": "model-definitions/container",
47+
"visibility": "VISIBILITY_PUBLIC",
48+
"task": "TASK_KEYPOINT",
49+
"region": "REGION_LOCAL",
50+
"hardware": "CPU",
51+
"version": "dev",
52+
"configuration": {}
53+
},
54+
{
55+
"id": "dummy-semantic-segmentation",
56+
"owner_type": "users",
57+
"owner_id": "admin",
58+
"description": "A semantic segmentation model for integration-test",
59+
"model_definition": "model-definitions/container",
60+
"visibility": "VISIBILITY_PUBLIC",
61+
"region": "REGION_LOCAL",
62+
"hardware": "CPU",
63+
"task": "TASK_SEMANTIC_SEGMENTATION",
64+
"version": "dev",
65+
"configuration": {}
66+
},
67+
{
68+
"id": "dummy-completion",
69+
"owner_type": "users",
70+
"owner_id": "admin",
71+
"description": "A completion model for integration-test",
72+
"model_definition": "model-definitions/container",
73+
"visibility": "VISIBILITY_PUBLIC",
74+
"task": "TASK_COMPLETION",
75+
"region": "REGION_LOCAL",
76+
"hardware": "CPU",
77+
"version": "dev",
78+
"configuration": {}
79+
},
80+
{
81+
"id": "dummy-chat",
82+
"owner_type": "users",
83+
"owner_id": "admin",
84+
"description": "A chat model for integration-test",
85+
"model_definition": "model-definitions/container",
86+
"visibility": "VISIBILITY_PUBLIC",
87+
"task": "TASK_CHAT",
88+
"region": "REGION_LOCAL",
89+
"hardware": "CPU",
90+
"version": "dev",
91+
"configuration": {}
92+
},
93+
{
94+
"id": "dummy-text-to-image",
95+
"owner_type": "users",
96+
"owner_id": "admin",
97+
"description": "A text to image model for integration-test",
98+
"model_definition": "model-definitions/container",
99+
"visibility": "VISIBILITY_PUBLIC",
100+
"task": "TASK_TEXT_TO_IMAGE",
101+
"region": "REGION_LOCAL",
102+
"hardware": "CPU",
103+
"version": "dev",
104+
"configuration": {}
105+
},
106+
{
107+
"id": "dummy-multimodal-chat",
108+
"owner_type": "users",
109+
"owner_id": "admin",
110+
"description": "A multimodal chat model for integration-test",
111+
"model_definition": "model-definitions/container",
112+
"visibility": "VISIBILITY_PUBLIC",
113+
"task": "TASK_CHAT",
114+
"region": "REGION_LOCAL",
115+
"hardware": "CPU",
116+
"version": "dev",
117+
"configuration": {}
118+
},
119+
{
120+
"id": "dummy-text-embedding",
121+
"owner_type": "users",
122+
"owner_id": "admin",
123+
"description": "A embedding model for integration-test",
124+
"model_definition": "model-definitions/container",
125+
"visibility": "VISIBILITY_PUBLIC",
126+
"task": "TASK_EMBEDDING",
127+
"region": "REGION_LOCAL",
128+
"hardware": "CPU",
129+
"version": "dev",
130+
"configuration": {}
131+
},
132+
{
133+
"id": "dummy-multimodal-embedding",
134+
"owner_type": "users",
135+
"owner_id": "admin",
136+
"description": "A multimodal embedding model for integration-test",
137+
"model_definition": "model-definitions/container",
138+
"visibility": "VISIBILITY_PUBLIC",
139+
"task": "TASK_EMBEDDING",
140+
"region": "REGION_LOCAL",
141+
"hardware": "CPU",
142+
"version": "dev",
143+
"configuration": {}
144+
}
145+
]

0 commit comments

Comments
 (0)