ci(workflows,Makefile): add model deploy integration test

Author: pinglin

.github/workflows/helm-integration-test-console.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Maximize build space
         uses: easimon/maximize-build-space@master
         with:
-          root-reserve-mb: 2048
+          root-reserve-mb: 5120
           remove-dotnet: "true"
           remove-android: "true"
           remove-haskell: "true"

.github/workflows/integration-test-latest.yml

@@ -0,0 +1,69 @@
+name: Integration Test Reusable (backend)
+
+on:
+  workflow_call:
+    inputs:
+      target:
+        required: true
+        type: string
+
+jobs:
+  integration-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 5120
+          remove-dotnet: "true"
+          remove-android: "true"
+          remove-haskell: "true"
+          remove-codeql: "true"
+          remove-docker-images: "true"
+          build-mount-path: "/var/lib/docker"
+
+      - name: Restart docker
+        run: sudo service docker restart
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Checkout Python SDK
+        uses: actions/checkout@v4
+        with:
+          repository: instill-ai/python-sdk
+          path: python-sdk
+
+      - name: Install Python SDK (editable mode)
+        run: |
+          cd python-sdk
+          git submodule update --init --remote --recursive
+          pip install --upgrade pip
+          pip install -e .
+          # Verify installation and environment
+          python -c "import instill; print('Instill SDK version:', instill.__version__)"
+          which instill
+          echo "Python path:"
+          python -c "import sys; print('\n'.join(sys.path))"
+
+      - name: Verify SDK installed
+        run: |
+          python -m site
+          pip list | grep instill
+          python -c "import instill; print(instill.__version__)"
+
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          repository: instill-ai/instill-core
+
+      - name: Run model deployment integration test (${{ inputs.target }})
+        run: |
+          make integration-test-model-deploy-${{ inputs.target }}
+
+      - name: Make down Instill Core
+        run: |
+          make down

.github/workflows/integration-test-model-deploy.yml

@@ -0,0 +1,48 @@
+name: Integration Test
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "**"
+
+jobs:
+  determine-target:
+    runs-on: ubuntu-latest
+    outputs:
+      target: ${{ steps.set-target.outputs.target }}
+    steps:
+      - name: Set target based on branch
+        id: set-target
+        run: |
+          if [[ "${{ github.ref }}" == "refs/heads/release-please--branches--main" ]]; then
+            echo "target=release" >> $GITHUB_OUTPUT
+          else
+            echo "target=latest" >> $GITHUB_OUTPUT
+          fi
+
+  backend:
+    needs: determine-target
+    strategy:
+      fail-fast: false
+      matrix:
+        # Include artifact-backend only for latest builds
+        component:
+          [mgmt-backend, pipeline-backend, model-backend, artifact-backend]
+    uses: ./.github/workflows/integration-test-backend.yml
+    with:
+      component: ${{ matrix.component }}
+      target: ${{ needs.determine-target.outputs.target }}
+
+  model-deploy:
+    needs: determine-target
+    uses: ./.github/workflows/integration-test-model-deploy.yml
+    with:
+      target: ${{ needs.determine-target.outputs.target }}
+
+  # TODO: Enable console integration test once the issue is fixed
+  # console:
+  #   needs: determine-target
+  #   uses: ./.github/workflows/integration-test-console.yml
+  #   with:
+  #     target: ${{ needs.determine-target.outputs.target }}

.github/workflows/integration-test-release.yml

@@ -181,6 +181,23 @@ ps:				## List all service containers
 top:			## Display all running service processes
 	@EDITION= DEFAULT_USER_UID= docker compose top
 
+.PHONY: integration-test-model-deploy-latest
+integration-test-model-deploy-latest:       	 	# Run integration test on the latest Instill Core to build, push and deploy dummy models
+	@make check-instill-sdk
+	@make build-and-push-models
+	@make latest EDITION=local-ce:test
+	@make latest EDITION=local-ce:test INITMODEL_ENABLED=true INITMODEL_PATH=${PWD}/integration-test/models/inventory.json
+	@make wait-models-deploy
+	@make down
+
+.PHONY: integration-test-model-deploy-release
+integration-test-model-deploy-release:       	 	# Run integration test on the released Instill Core to build, push and deploy dummy models
+	@make all EDITION=local-ce:test
+	@make build-and-push-models
+	@make all EDITION=local-ce:test INITMODEL_ENABLED=true INITMODEL_PATH=${PWD}/integration-test/models/inventory.json
+	@make wait-models-deploy
+	@make down
+
 .PHONY: integration-test-latest
 integration-test-latest:			# Run integration test on the latest Instill Core
 	@make build-latest
@@ -196,7 +213,7 @@ integration-test-latest:			# Run integration test on the latest Instill Core
 	@make down
 
 .PHONY: integration-test-release
-integration-test-release:			# Run integration test on the release Instill Core
+integration-test-release:			# Run integration test on the released Instill Core
 	@make build-release
 	@make all EDITION=local-ce:test COMPONENT_ENV=${COMPONENT_TEST_ENV}
 	@docker run --rm \
@@ -318,7 +335,7 @@ console-integration-test-latest:			# Run console integration test on the latest
 	@make down
 
 .PHONY: console-integration-test-release
-console-integration-test-release:			# Run console integration test on the release Instill Core
+console-integration-test-release:			# Run console integration test on the released Instill Core
 	@make all EDITION=local-ce:test INSTILL_CORE_HOST=${API_GATEWAY_HOST} COMPONENT_ENV=${COMPONENT_TEST_ENV}
 	@docker run --rm \
 		-e NEXT_PUBLIC_GENERAL_API_VERSION=v1beta \
@@ -491,6 +508,38 @@ endif
 	@pkill -f "port-forward"
 	@make down
 
+.PHONY: build-and-push-models
+build-and-push-models:	# Helper target to build and push models
+	@python -c "import instill; print(instill.__version__)"
+	@python3 -c 'import instill; import os.path; print(os.path.dirname(os.path.dirname(instill.__file__)))'
+	@SDK_PATH=$$(python3 -c 'import instill; import os.path; print(os.path.dirname(os.path.dirname(instill.__file__)))' 2>/dev/null || echo ""); \
+	./integration-test/scripts/build_and_push_models.sh \
+		"$(PWD)/integration-test/models" \
+		"localhost:5001" \
+		"$$SDK_PATH"
+
+.PHONY: wait-models-deploy
+wait-models-deploy:		# Helper target to wait for model deployment
+	@model_count=$$(jq length integration-test/models/inventory.json); \
+	timeout=600; elapsed=0; spinner='|/-\\'; i=0; \
+	while [ "$$(curl -s http://localhost:8265/api/serve/applications/ | jq ".applications | to_entries | map(select(.key | contains(\"dummy-\")) | .value.status) | length == $$model_count and all(. == \"RUNNING\")")" != "true" ]; do \
+		running_count=$$(curl -s http://localhost:8265/api/serve/applications/ | jq '.applications | to_entries | map(select(.key | contains("dummy-")) | .value.status) | map(select(. == "RUNNING")) | length'); \
+		printf "\r[Waiting %3ds/%ds] %c models still deploying... (%d/%d RUNNING)" "$$elapsed" "$$timeout" "$${spinner:i++%4:1}" "$$running_count" "$$model_count"; \
+		sleep 1; elapsed=$$((elapsed+1)); \
+		if [ "$$elapsed" -ge "$$timeout" ]; then \
+			echo "\nTimeout waiting for models to deploy!"; exit 1; \
+		fi; \
+	done; \
+	echo "\nAll $$model_count models deployed and running."
+
+.PHONY: check-instill-sdk
+check-instill-sdk:		# Helper target to check if instill-sdk is installed and show version or path if in editable mode
+	@echo "Checking for instill-sdk installation..." >&2
+	@if ! python3 -m pip show instill-sdk > /dev/null 2>&1; then \
+		echo "instill-sdk is NOT installed." >&2; \
+		false; \
+	fi
+
 .PHONY: help
 help:       	## Show this help
 	@echo "\nMake Application with Instill Core"

.github/workflows/integration-test.yml

@@ -314,6 +314,8 @@ services:
       CFG_INITMODEL_ENABLED: ${INITMODEL_ENABLED}
       CFG_INITMODEL_PATH: ${INITMODEL_PATH}
     entrypoint: ./model-backend-init-model
+    volumes:
+      - ${INITMODEL_PATH:-/dev/null}:${INITMODEL_PATH:-/dev/null}
     depends_on:
       model_backend:
         condition: service_healthy
@@ -563,6 +565,8 @@ services:
     restart: unless-stopped
     volumes:
       - ${CONFIG_DIR_PATH}/registry/config.yaml:/etc/docker/registry/config.yml
+    ports:
+      - ${REGISTRY_EXTERNAL_PORT}:${REGISTRY_PORT}
     depends_on:
       redis:
         condition: service_healthy

Makefile

@@ -0,0 +1,9 @@
+---
+Task: TextGenerationChat
+Tags:
+  - TextGenerationChat
+  - Test
+---
+
+# Test repo
+This is a dummy text generation chat model for testing purpose

docker-compose.yml

@@ -0,0 +1,4 @@
+build:
+  gpu: false
+  python_version: "3.11"  # support only 3.11
+  python_packages:

integration-test/models/dummy-chat/README.md

@@ -0,0 +1,36 @@
+import time
+from instill.helpers.ray_config import instill_deployment, InstillDeployable
+from instill.helpers import (
+    parse_task_chat_to_chat_input,
+    construct_task_chat_output,
+)
+
+
+@instill_deployment
+class Chat:
+    def __init__(self):
+        pass
+
+    async def __call__(self, request):
+        inputs = await parse_task_chat_to_chat_input(request=request)
+
+        input_len = len(inputs)
+
+        finish_reasons = [["length"] for _ in range(input_len)]
+        indexes = [list(range(input_len))]
+        created = [[int(time.time())] for _ in range(input_len)]
+        messages = [
+            [{"content": inputs[i].messages[-1]["content"], "role": "assistant"}]
+            for i in range(input_len)
+        ]
+
+        return construct_task_chat_output(
+            request=request,
+            finish_reasons=finish_reasons,
+            indexes=indexes,
+            created_timestamps=created,
+            messages=messages,
+        )
+
+
+entrypoint = InstillDeployable(Chat).get_deployment_handle()