Skip to content

Commit b574be1

Browse files
pinglinpinglin
committed
ci(workflows,Makefile): add model deploy integration test
1 parent d0c53ce commit b574be1

File tree

51 files changed

+898
-114
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+898
-114
lines changed

‎.github/workflows/helm-integration-test-console.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
- name: Maximize build space
1616
uses: easimon/maximize-build-space@master
1717
with:
18-
root-reserve-mb: 2048
18+
root-reserve-mb: 5120
1919
remove-dotnet: "true"
2020
remove-android: "true"
2121
remove-haskell: "true"

‎.github/workflows/integration-test-latest.yml‎

Lines changed: 0 additions & 23 deletions
This file was deleted.

‎.github/workflows/integration-test-model-deploy.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: Integration Test Reusable (backend)
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
target:
7+
required: true
8+
type: string
9+
10+
jobs:
11+
integration-test:
12+
runs-on: ubuntu-latest
13+
timeout-minutes: 120
14+
steps:
15+
- name: Maximize build space
16+
uses: easimon/maximize-build-space@master
17+
with:
18+
root-reserve-mb: 5120
19+
remove-dotnet: "true"
20+
remove-android: "true"
21+
remove-haskell: "true"
22+
remove-codeql: "true"
23+
remove-docker-images: "true"
24+
build-mount-path: "/var/lib/docker"
25+
26+
- name: Restart docker
27+
run: sudo service docker restart
28+
29+
- name: Setup Python
30+
uses: actions/setup-python@v5
31+
with:
32+
python-version: "3.11"
33+
34+
- name: Checkout Python SDK
35+
uses: actions/checkout@v4
36+
with:
37+
repository: instill-ai/python-sdk
38+
path: python-sdk
39+
40+
- name: Install Python SDK (editable mode)
41+
run: |
42+
cd python-sdk
43+
git submodule update --init --remote --recursive
44+
pip install --upgrade pip
45+
pip install -e .
46+
# Verify installation and environment
47+
python -c "import instill; print('Instill SDK version:', instill.__version__)"
48+
which instill
49+
50+
- name: Checkout repo
51+
uses: actions/checkout@v4
52+
with:
53+
repository: instill-ai/instill-core
54+
path: instill-core
55+
56+
- name: Run model deployment integration test (${{ inputs.target }})
57+
run: |
58+
cd instill-core
59+
make integration-test-model-deploy-${{ inputs.target }}
60+
61+
- name: Make down Instill Core
62+
run: |
63+
make down

‎.github/workflows/integration-test-release.yml‎

Lines changed: 0 additions & 23 deletions
This file was deleted.

‎.github/workflows/integration-test.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
name: Integration Test
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches:
7+
- "**"
8+
9+
jobs:
10+
determine-target:
11+
runs-on: ubuntu-latest
12+
outputs:
13+
target: ${{ steps.set-target.outputs.target }}
14+
steps:
15+
- name: Set target based on branch
16+
id: set-target
17+
run: |
18+
if [[ "${{ github.ref }}" == "refs/heads/release-please--branches--main" ]]; then
19+
echo "target=release" >> $GITHUB_OUTPUT
20+
else
21+
echo "target=latest" >> $GITHUB_OUTPUT
22+
fi
23+
24+
backend:
25+
needs: determine-target
26+
strategy:
27+
fail-fast: false
28+
matrix:
29+
# Include artifact-backend only for latest builds
30+
component:
31+
[mgmt-backend, pipeline-backend, model-backend, artifact-backend]
32+
uses: ./.github/workflows/integration-test-backend.yml
33+
with:
34+
component: ${{ matrix.component }}
35+
target: ${{ needs.determine-target.outputs.target }}
36+
37+
model-deploy:
38+
needs: determine-target
39+
uses: ./.github/workflows/integration-test-model-deploy.yml
40+
with:
41+
target: ${{ needs.determine-target.outputs.target }}
42+
43+
# TODO: Enable console integration test once the issue is fixed
44+
# console:
45+
# needs: determine-target
46+
# uses: ./.github/workflows/integration-test-console.yml
47+
# with:
48+
# target: ${{ needs.determine-target.outputs.target }}

‎Makefile‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
.DEFAULT_GOAL:=help
22

3+
SHELL := /bin/bash
4+
35
#============================================================================
46

57
# load environment variables
@@ -181,6 +183,22 @@ ps: ## List all service containers
181183
top: ## Display all running service processes
182184
@EDITION= DEFAULT_USER_UID= docker compose top
183185

186+
.PHONY: integration-test-model-deploy-latest
187+
integration-test-model-deploy-latest: # Run integration test on the latest Instill Core to build, push and deploy dummy models
188+
@make latest EDITION=local-ce:test
189+
@make build-and-push-models
190+
@make latest EDITION=local-ce:test INITMODEL_ENABLED=true INITMODEL_PATH=${PWD}/integration-test/models/inventory.json
191+
@make wait-models-deploy
192+
@make down
193+
194+
.PHONY: integration-test-model-deploy-release
195+
integration-test-model-deploy-release: # Run integration test on the released Instill Core to build, push and deploy dummy models
196+
@make all EDITION=local-ce:test
197+
@make build-and-push-models
198+
@make all EDITION=local-ce:test INITMODEL_ENABLED=true INITMODEL_PATH=${PWD}/integration-test/models/inventory.json
199+
@make wait-models-deploy
200+
@make down
201+
184202
.PHONY: integration-test-latest
185203
integration-test-latest: # Run integration test on the latest Instill Core
186204
@make build-latest
@@ -196,7 +214,7 @@ integration-test-latest: # Run integration test on the latest Instill Core
196214
@make down
197215

198216
.PHONY: integration-test-release
199-
integration-test-release: # Run integration test on the release Instill Core
217+
integration-test-release: # Run integration test on the released Instill Core
200218
@make build-release
201219
@make all EDITION=local-ce:test COMPONENT_ENV=${COMPONENT_TEST_ENV}
202220
@docker run --rm \
@@ -318,7 +336,7 @@ console-integration-test-latest: # Run console integration test on the latest
318336
@make down
319337

320338
.PHONY: console-integration-test-release
321-
console-integration-test-release: # Run console integration test on the release Instill Core
339+
console-integration-test-release: # Run console integration test on the released Instill Core
322340
@make all EDITION=local-ce:test INSTILL_CORE_HOST=${API_GATEWAY_HOST} COMPONENT_ENV=${COMPONENT_TEST_ENV}
323341
@docker run --rm \
324342
-e NEXT_PUBLIC_GENERAL_API_VERSION=v1beta \
@@ -491,6 +509,25 @@ endif
491509
@pkill -f "port-forward"
492510
@make down
493511

512+
.PHONY: build-and-push-models
513+
build-and-push-models: # Helper target to build and push models
514+
@./integration-test/scripts/build_and_push_models.sh "$(PWD)/integration-test/models" "localhost:5001"
515+
516+
.PHONY: wait-models-deploy
517+
wait-models-deploy: # Helper target to wait for model deployment
518+
@model_count=$$(jq length integration-test/models/inventory.json); \
519+
timeout=600; elapsed=0; spinner='|/-\\'; i=0; \
520+
while [ "$$(curl -s http://localhost:8265/api/serve/applications/ | jq ".applications | to_entries | map(select(.key | contains(\"dummy-\")) | .value.status) | length == $$model_count and all(. == \"RUNNING\")")" != "true" ]; do \
521+
running_count=$$(curl -s http://localhost:8265/api/serve/applications/ | jq '.applications | to_entries | map(select(.key | contains("dummy-")) | .value.status) | map(select(. == "RUNNING")) | length'); \
522+
printf "\r[Waiting %3ds/%ds] %s models still deploying... (%d/%d RUNNING)" "$$elapsed" "$$timeout" "$${spinner:$$((i % 4)):1}" "$$running_count" "$$model_count"; \
523+
sleep 1; elapsed=$$((elapsed+1)); \
524+
if [ "$$elapsed" -ge "$$timeout" ]; then \
525+
echo "\nTimeout waiting for models to deploy!"; exit 1; \
526+
fi; \
527+
i=$$((i + 1)); \
528+
done; \
529+
echo "\nAll $$model_count models deployed and running."
530+
494531
.PHONY: help
495532
help: ## Show this help
496533
@echo "\nMake Application with Instill Core"

‎docker-compose.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,8 @@ services:
314314
CFG_INITMODEL_ENABLED: ${INITMODEL_ENABLED}
315315
CFG_INITMODEL_PATH: ${INITMODEL_PATH}
316316
entrypoint: ./model-backend-init-model
317+
volumes:
318+
- ${INITMODEL_PATH:-/dev/null}:${INITMODEL_PATH:-/dev/null}
317319
depends_on:
318320
model_backend:
319321
condition: service_healthy
@@ -563,6 +565,8 @@ services:
563565
restart: unless-stopped
564566
volumes:
565567
- ${CONFIG_DIR_PATH}/registry/config.yaml:/etc/docker/registry/config.yml
568+
ports:
569+
- ${REGISTRY_EXTERNAL_PORT}:${REGISTRY_PORT}
566570
depends_on:
567571
redis:
568572
condition: service_healthy

‎integration-test/models/dummy-chat/README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
---
2+
Task: TextGenerationChat
3+
Tags:
4+
- TextGenerationChat
5+
- Test
6+
---
7+
8+
# Test repo
9+
This is a dummy text generation chat model for testing purpose

‎integration-test/models/dummy-chat/instill.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
build:
2+
gpu: false
3+
python_version: "3.11" # support only 3.11
4+
python_packages:

‎integration-test/models/dummy-chat/model.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import time
2+
from instill.helpers.ray_config import instill_deployment, InstillDeployable
3+
from instill.helpers import (
4+
parse_task_chat_to_chat_input,
5+
construct_task_chat_output,
6+
)
7+
8+
9+
@instill_deployment
10+
class Chat:
11+
def __init__(self):
12+
pass
13+
14+
async def __call__(self, request):
15+
inputs = await parse_task_chat_to_chat_input(request=request)
16+
17+
input_len = len(inputs)
18+
19+
finish_reasons = [["length"] for _ in range(input_len)]
20+
indexes = [list(range(input_len))]
21+
created = [[int(time.time())] for _ in range(input_len)]
22+
messages = [
23+
[{"content": inputs[i].messages[-1]["content"], "role": "assistant"}]
24+
for i in range(input_len)
25+
]
26+
27+
return construct_task_chat_output(
28+
request=request,
29+
finish_reasons=finish_reasons,
30+
indexes=indexes,
31+
created_timestamps=created,
32+
messages=messages,
33+
)
34+
35+
36+
entrypoint = InstillDeployable(Chat).get_deployment_handle()

0 commit comments

Comments
 (0)