Skip to content

Commit bbe7565

Browse files
pinglinpinglin
committed
ci(workflows): run integration test on only one dummy model
1 parent 9351fb9 commit bbe7565

File tree

6 files changed

+228
-142
lines changed

6 files changed

+228
-142
lines changed

‎.github/workflows/integration-test.yml‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ on:
77
- "**"
88

99
jobs:
10-
1110
compose:
1211
runs-on: ubuntu-latest
1312
steps:
@@ -138,10 +137,46 @@ jobs:
138137
fetch-depth: 0
139138
fetch-tags: true
140139

140+
- name: Load .env file
141+
uses: cardinalby/export-env-action@v2
142+
with:
143+
envFile: instill-core/.env
144+
141145
- name: Run model integration test
142146
working-directory: instill-core
143147
run: |
144-
make model-integration-test
148+
make model-integration-test
149+
150+
- name: Debug on failure
151+
if: always()
152+
working-directory: instill-core
153+
run: |
154+
echo "=== DEBUGGING FAILED TEST ==="
155+
156+
echo -e "\n=== Docker containers ==="
157+
docker ps -a
158+
159+
echo -e "\n=== System resources ==="
160+
docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"
161+
162+
echo -e "\n=== model-backend logs ==="
163+
docker logs model-backend --tail 100 || true
164+
165+
echo -e "\n=== model-backend-worker logs ==="
166+
docker logs model-backend-worker --tail 100 || true
167+
168+
echo -e "\n=== model-backend-init-model logs ==="
169+
docker logs model-backend-init-model --tail 100 || true
170+
171+
echo -e "\n=== Ray logs ==="
172+
docker logs ray --tail 200 || true
173+
174+
echo -e "\n=== Registry logs ==="
175+
docker logs registry --tail 50 || true
176+
177+
echo -e "\n=== Ray Serve applications ==="
178+
docker run --rm --network instill-network curlimages/curl:latest \
179+
curl -s http://ray:8265/api/serve/applications/ | jq '.' || true
145180
146181
- name: Tear down Instill Core
147182
if: always()

‎Makefile‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -153,13 +153,33 @@ build-and-push-models: # Helper target to build and push models
153153
.PHONY: wait-models-deploy
154154
wait-models-deploy: # Helper target to wait for model deployment
155155
@model_count=$$(jq length integration-test/models/inventory.json); \
156-
timeout=1800; elapsed=0; spinner='|/-\\'; i=0; \
156+
echo "Waiting for $$model_count model(s) to deploy..."; \
157+
echo "Expected models:"; \
158+
jq -r '.[].id' integration-test/models/inventory.json; \
159+
echo ""; \
160+
timeout=1800; elapsed=0; spinner='|/-\\'; i=0; debug_interval=10; \
161+
ray_response=$$(docker run --rm --network instill-network curlimages/curl:latest curl -s http://${RAY_HOST}:${RAY_PORT_DASHBOARD}/api/serve/applications/); \
162+
echo "=== Initial Ray state ==="; \
163+
echo "$$ray_response" | jq '.applications | to_entries | map({key: .key, status: .value.status, message: .value.message})' || echo "$$ray_response"; \
164+
echo ""; \
157165
while [ "$$(docker run --rm --network instill-network curlimages/curl:latest curl -s http://${RAY_HOST}:${RAY_PORT_DASHBOARD}/api/serve/applications/ | jq ".applications | to_entries | map(select(.key | contains(\"dummy-\")) | .value.status) | length == $$model_count and all(. == \"RUNNING\")")" != "true" ]; do \
158-
running_count=$$(docker run --rm --network instill-network curlimages/curl:latest curl -s http://${RAY_HOST}:${RAY_PORT_DASHBOARD}/api/serve/applications/ | jq '.applications | to_entries | map(select(.key | contains("dummy-")) | .value.status) | map(select(. == "RUNNING")) | length'); \
166+
ray_response=$$(docker run --rm --network instill-network curlimages/curl:latest curl -s http://${RAY_HOST}:${RAY_PORT_DASHBOARD}/api/serve/applications/); \
167+
running_count=$$(echo "$$ray_response" | jq '.applications | to_entries | map(select(.key | contains("dummy-")) | .value.status) | map(select(. == "RUNNING")) | length'); \
159168
printf "\r[Waiting %3ds/%ds] %s models still deploying... (%d/%d RUNNING)" "$$elapsed" "$$timeout" "$${spinner:$$((i % 4)):1}" "$$running_count" "$$model_count"; \
169+
if [ $$((elapsed % debug_interval)) -eq 0 ] && [ "$$elapsed" -gt 0 ]; then \
170+
echo ""; \
171+
echo "=== Debug at $${elapsed}s ==="; \
172+
echo "$$ray_response" | jq '.applications | to_entries | map({key: .key, status: .value.status, message: .value.message})' || echo "$$ray_response"; \
173+
fi; \
160174
sleep 1; elapsed=$$((elapsed+1)); \
161175
if [ "$$elapsed" -ge "$$timeout" ]; then \
162-
echo "\nTimeout waiting for models to deploy!"; exit 1; \
176+
echo ""; \
177+
echo "=== TIMEOUT - Final Ray state ==="; \
178+
docker run --rm --network instill-network curlimages/curl:latest curl -s http://${RAY_HOST}:${RAY_PORT_DASHBOARD}/api/serve/applications/ | jq '.'; \
179+
echo ""; \
180+
echo "=== Model Backend Init Model Logs ==="; \
181+
docker logs model-backend-init-model --tail 100 || echo "Container not found"; \
182+
echo "Timeout waiting for models to deploy!"; exit 1; \
163183
fi; \
164184
i=$$((i + 1)); \
165185
done; \

‎docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -493,7 +493,7 @@ services:
493493
tail -f /dev/null'
494494
volumes:
495495
- /var/lib/containers:/var/lib/containers
496-
shm_size: 8gb
496+
shm_size: 4gb
497497
healthcheck:
498498
test: ["CMD", "ray", "status"]
499499
interval: 5s

‎integration-test/models/inventory-all.json‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
[
2+
{
3+
"id": "dummy-cls",
4+
"owner_type": "users",
5+
"owner_id": "admin",
6+
"description": "A classification model for integration-test",
7+
"model_definition": "model-definitions/container",
8+
"visibility": "VISIBILITY_PUBLIC",
9+
"task": "TASK_CLASSIFICATION",
10+
"region": "REGION_LOCAL",
11+
"hardware": "CPU",
12+
"version": "dev",
13+
"configuration": {}
14+
},
15+
{
16+
"id": "dummy-det",
17+
"owner_type": "users",
18+
"owner_id": "admin",
19+
"description": "A object detection model for integration-test",
20+
"model_definition": "model-definitions/container",
21+
"visibility": "VISIBILITY_PUBLIC",
22+
"task": "TASK_DETECTION",
23+
"region": "REGION_LOCAL",
24+
"hardware": "CPU",
25+
"version": "dev",
26+
"configuration": {}
27+
},
28+
{
29+
"id": "dummy-instance-segmentation",
30+
"owner_type": "users",
31+
"owner_id": "admin",
32+
"description": "A instance segmentation model for integration-test",
33+
"model_definition": "model-definitions/container",
34+
"visibility": "VISIBILITY_PUBLIC",
35+
"task": "TASK_INSTANCE_SEGMENTATION",
36+
"region": "REGION_LOCAL",
37+
"hardware": "CPU",
38+
"version": "dev",
39+
"configuration": {}
40+
},
41+
{
42+
"id": "dummy-keypoint",
43+
"owner_type": "users",
44+
"owner_id": "admin",
45+
"description": "A keypoint detection model for integration-test",
46+
"model_definition": "model-definitions/container",
47+
"visibility": "VISIBILITY_PUBLIC",
48+
"task": "TASK_KEYPOINT",
49+
"region": "REGION_LOCAL",
50+
"hardware": "CPU",
51+
"version": "dev",
52+
"configuration": {}
53+
},
54+
{
55+
"id": "dummy-semantic-segmentation",
56+
"owner_type": "users",
57+
"owner_id": "admin",
58+
"description": "A semantic segmentation model for integration-test",
59+
"model_definition": "model-definitions/container",
60+
"visibility": "VISIBILITY_PUBLIC",
61+
"region": "REGION_LOCAL",
62+
"hardware": "CPU",
63+
"task": "TASK_SEMANTIC_SEGMENTATION",
64+
"version": "dev",
65+
"configuration": {}
66+
},
67+
{
68+
"id": "dummy-completion",
69+
"owner_type": "users",
70+
"owner_id": "admin",
71+
"description": "A completion model for integration-test",
72+
"model_definition": "model-definitions/container",
73+
"visibility": "VISIBILITY_PUBLIC",
74+
"task": "TASK_COMPLETION",
75+
"region": "REGION_LOCAL",
76+
"hardware": "CPU",
77+
"version": "dev",
78+
"configuration": {}
79+
},
80+
{
81+
"id": "dummy-chat",
82+
"owner_type": "users",
83+
"owner_id": "admin",
84+
"description": "A chat model for integration-test",
85+
"model_definition": "model-definitions/container",
86+
"visibility": "VISIBILITY_PUBLIC",
87+
"task": "TASK_CHAT",
88+
"region": "REGION_LOCAL",
89+
"hardware": "CPU",
90+
"version": "dev",
91+
"configuration": {}
92+
},
93+
{
94+
"id": "dummy-text-to-image",
95+
"owner_type": "users",
96+
"owner_id": "admin",
97+
"description": "A text to image model for integration-test",
98+
"model_definition": "model-definitions/container",
99+
"visibility": "VISIBILITY_PUBLIC",
100+
"task": "TASK_TEXT_TO_IMAGE",
101+
"region": "REGION_LOCAL",
102+
"hardware": "CPU",
103+
"version": "dev",
104+
"configuration": {}
105+
},
106+
{
107+
"id": "dummy-multimodal-chat",
108+
"owner_type": "users",
109+
"owner_id": "admin",
110+
"description": "A multimodal chat model for integration-test",
111+
"model_definition": "model-definitions/container",
112+
"visibility": "VISIBILITY_PUBLIC",
113+
"task": "TASK_CHAT",
114+
"region": "REGION_LOCAL",
115+
"hardware": "CPU",
116+
"version": "dev",
117+
"configuration": {}
118+
},
119+
{
120+
"id": "dummy-text-embedding",
121+
"owner_type": "users",
122+
"owner_id": "admin",
123+
"description": "A embedding model for integration-test",
124+
"model_definition": "model-definitions/container",
125+
"visibility": "VISIBILITY_PUBLIC",
126+
"task": "TASK_EMBEDDING",
127+
"region": "REGION_LOCAL",
128+
"hardware": "CPU",
129+
"version": "dev",
130+
"configuration": {}
131+
},
132+
{
133+
"id": "dummy-multimodal-embedding",
134+
"owner_type": "users",
135+
"owner_id": "admin",
136+
"description": "A multimodal embedding model for integration-test",
137+
"model_definition": "model-definitions/container",
138+
"visibility": "VISIBILITY_PUBLIC",
139+
"task": "TASK_EMBEDDING",
140+
"region": "REGION_LOCAL",
141+
"hardware": "CPU",
142+
"version": "dev",
143+
"configuration": {}
144+
}
145+
]

‎integration-test/models/inventory.json‎

Lines changed: 0 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -11,135 +11,5 @@
1111
"hardware": "CPU",
1212
"version": "dev",
1313
"configuration": {}
14-
},
15-
{
16-
"id": "dummy-det",
17-
"owner_type": "users",
18-
"owner_id": "admin",
19-
"description": "A object detection model for integration-test",
20-
"model_definition": "model-definitions/container",
21-
"visibility": "VISIBILITY_PUBLIC",
22-
"task": "TASK_DETECTION",
23-
"region": "REGION_LOCAL",
24-
"hardware": "CPU",
25-
"version": "dev",
26-
"configuration": {}
27-
},
28-
{
29-
"id": "dummy-instance-segmentation",
30-
"owner_type": "users",
31-
"owner_id": "admin",
32-
"description": "A instance segmentation model for integration-test",
33-
"model_definition": "model-definitions/container",
34-
"visibility": "VISIBILITY_PUBLIC",
35-
"task": "TASK_INSTANCE_SEGMENTATION",
36-
"region": "REGION_LOCAL",
37-
"hardware": "CPU",
38-
"version": "dev",
39-
"configuration": {}
40-
},
41-
{
42-
"id": "dummy-keypoint",
43-
"owner_type": "users",
44-
"owner_id": "admin",
45-
"description": "A keypoint detection model for integration-test",
46-
"model_definition": "model-definitions/container",
47-
"visibility": "VISIBILITY_PUBLIC",
48-
"task": "TASK_KEYPOINT",
49-
"region": "REGION_LOCAL",
50-
"hardware": "CPU",
51-
"version": "dev",
52-
"configuration": {}
53-
},
54-
{
55-
"id": "dummy-semantic-segmentation",
56-
"owner_type": "users",
57-
"owner_id": "admin",
58-
"description": "A semantic segmentation model for integration-test",
59-
"model_definition": "model-definitions/container",
60-
"visibility": "VISIBILITY_PUBLIC",
61-
"region": "REGION_LOCAL",
62-
"hardware": "CPU",
63-
"task": "TASK_SEMANTIC_SEGMENTATION",
64-
"version": "dev",
65-
"configuration": {}
66-
},
67-
{
68-
"id": "dummy-completion",
69-
"owner_type": "users",
70-
"owner_id": "admin",
71-
"description": "A completion model for integration-test",
72-
"model_definition": "model-definitions/container",
73-
"visibility": "VISIBILITY_PUBLIC",
74-
"task": "TASK_COMPLETION",
75-
"region": "REGION_LOCAL",
76-
"hardware": "CPU",
77-
"version": "dev",
78-
"configuration": {}
79-
},
80-
{
81-
"id": "dummy-chat",
82-
"owner_type": "users",
83-
"owner_id": "admin",
84-
"description": "A chat model for integration-test",
85-
"model_definition": "model-definitions/container",
86-
"visibility": "VISIBILITY_PUBLIC",
87-
"task": "TASK_CHAT",
88-
"region": "REGION_LOCAL",
89-
"hardware": "CPU",
90-
"version": "dev",
91-
"configuration": {}
92-
},
93-
{
94-
"id": "dummy-text-to-image",
95-
"owner_type": "users",
96-
"owner_id": "admin",
97-
"description": "A text to image model for integration-test",
98-
"model_definition": "model-definitions/container",
99-
"visibility": "VISIBILITY_PUBLIC",
100-
"task": "TASK_TEXT_TO_IMAGE",
101-
"region": "REGION_LOCAL",
102-
"hardware": "CPU",
103-
"version": "dev",
104-
"configuration": {}
105-
},
106-
{
107-
"id": "dummy-multimodal-chat",
108-
"owner_type": "users",
109-
"owner_id": "admin",
110-
"description": "A multimodal chat model for integration-test",
111-
"model_definition": "model-definitions/container",
112-
"visibility": "VISIBILITY_PUBLIC",
113-
"task": "TASK_CHAT",
114-
"region": "REGION_LOCAL",
115-
"hardware": "CPU",
116-
"version": "dev",
117-
"configuration": {}
118-
},
119-
{
120-
"id": "dummy-text-embedding",
121-
"owner_type": "users",
122-
"owner_id": "admin",
123-
"description": "A embedding model for integration-test",
124-
"model_definition": "model-definitions/container",
125-
"visibility": "VISIBILITY_PUBLIC",
126-
"task": "TASK_EMBEDDING",
127-
"region": "REGION_LOCAL",
128-
"hardware": "CPU",
129-
"version": "dev",
130-
"configuration": {}
131-
},
132-
{
133-
"id": "dummy-multimodal-embedding",
134-
"owner_type": "users",
135-
"owner_id": "admin",
136-
"description": "A multimodal embedding model for integration-test",
137-
"model_definition": "model-definitions/container",
138-
"visibility": "VISIBILITY_PUBLIC",
139-
"task": "TASK_EMBEDDING",
140-
"region": "REGION_LOCAL",
141-
"hardware": "CPU",
142-
"version": "dev",
143-
"configuration": {}
14414
}
14515
]

0 commit comments

Comments
 (0)