chore: clean e2e tests

Author: sangjanai

engine/e2e-test/main.py

@@ -2,27 +2,16 @@
 import sys
 
 ### e2e tests are expensive, have to keep engines tests in order
-from test_api_engine_list import TestApiEngineList
-from test_api_engine_install import TestApiEngineInstall
-from test_api_engine_get import TestApiEngineGet
-
-### models, keeps in order, note that we only uninstall engine after finishing all models test
-from test_api_model_pull_direct_url import TestApiModelPullDirectUrl
-from test_api_model_start_stop import TestApiModelStartStop
-from test_api_model_get import TestApiModelGet
-from test_api_model_list import TestApiModelList
-from test_api_model_update import TestApiModelUpdate
-from test_api_model_delete import TestApiModelDelete
+from test_api_engine import TestApiEngine
+from test_api_model import TestApiModel
 from test_api_model_import import TestApiModelImport
-from test_api_engine_uninstall import TestApiEngineUninstall
 
 ###
 from test_cli_engine_get import TestCliEngineGet
 from test_cli_engine_install import TestCliEngineInstall
 from test_cli_engine_list import TestCliEngineList
 from test_cli_engine_uninstall import TestCliEngineUninstall
-from test_cli_model_delete import TestCliModelDelete
-from test_cli_model_pull_direct_url import TestCliModelPullDirectUrl
+from test_cli_model import TestCliModel
 from test_cli_server_start import TestCliServerStart
 from test_cortex_update import TestCortexUpdate
 from test_create_log_folder import TestCreateLogFolder

…ne/e2e-test/test_api_engine_uninstall.py engine/e2e-test/test_api_engine.pyengine/e2e-test/test_api_engine_uninstall.py renamed to engine/e2e-test/test_api_engine.py engine/e2e-test/test_api_engine_uninstall.py renamed to engine/e2e-test/test_api_engine.py

@@ -1,29 +1,54 @@
-import time
-
 import pytest
 import requests
+import time
 from test_runner import (
-    run,
-    start_server_if_needed,
+    start_server,
     stop_server,
     wait_for_websocket_download_success_event,
 )
 
-
-class TestApiEngineUninstall:
+class TestApiEngine:
 
     @pytest.fixture(autouse=True)
     def setup_and_teardown(self):
         # Setup
-        start_server_if_needed()
+        success = start_server()
+        if not success:
+            raise Exception("Failed to start server")
 
         yield
 
         # Teardown
         stop_server()
 
+    # engines list
+    def test_engines_list_api_run_successfully(self):
+        response = requests.get("http://localhost:3928/engines")
+        assert response.status_code == 200
+    
+    # engines get
+    def test_engines_get_llamacpp_should_be_successful(self):
+        response = requests.get("http://localhost:3928/engines/llama-cpp")
+        assert response.status_code == 200
+        
+    # engines install
+    def test_engines_install_llamacpp_specific_version_and_variant(self):
+        data = {"version": "v0.1.35-27.10.24", "variant": "linux-amd64-avx-cuda-11-7"}
+        response = requests.post(
+            "http://localhost:3928/v1/engines/llama-cpp/install", json=data
+        )
+        assert response.status_code == 200
+
+    def test_engines_install_llamacpp_specific_version_and_null_variant(self):
+        data = {"version": "v0.1.35-27.10.24"}
+        response = requests.post(
+            "http://localhost:3928/v1/engines/llama-cpp/install", json=data
+        )
+        assert response.status_code == 200
+    
+    # engines uninstall
     @pytest.mark.asyncio
-    async def test_engines_uninstall_llamacpp_should_be_successful(self):
+    async def test_engines_install_uninstall_llamacpp_should_be_successful(self):
         response = requests.post("http://localhost:3928/v1/engines/llama-cpp/install")
         assert response.status_code == 200
         await wait_for_websocket_download_success_event(timeout=None)
@@ -33,7 +58,7 @@ async def test_engines_uninstall_llamacpp_should_be_successful(self):
         assert response.status_code == 200
 
     @pytest.mark.asyncio
-    async def test_engines_uninstall_llamacpp_with_only_version_should_be_failed(self):
+    async def test_engines_install_uninstall_llamacpp_with_only_version_should_be_failed(self):
         # install first
         data = {"variant": "mac-arm64"}
         install_response = requests.post(
@@ -50,7 +75,7 @@ async def test_engines_uninstall_llamacpp_with_only_version_should_be_failed(sel
         assert response.json()["message"] == "No variant provided"
 
     @pytest.mark.asyncio
-    async def test_engines_uninstall_llamacpp_with_variant_should_be_successful(self):
+    async def test_engines_install_uninstall_llamacpp_with_variant_should_be_successful(self):
         # install first
         data = {"variant": "mac-arm64"}
         install_response = requests.post(
@@ -62,7 +87,7 @@ async def test_engines_uninstall_llamacpp_with_variant_should_be_successful(self
         response = requests.delete("http://127.0.0.1:3928/v1/engines/llama-cpp/install")
         assert response.status_code == 200
 
-    def test_engines_uninstall_llamacpp_with_specific_variant_and_version_should_be_successful(
+    def test_engines_install_uninstall_llamacpp_with_specific_variant_and_version_should_be_successful(
         self,
     ):
         data = {"variant": "mac-arm64", "version": "v0.1.35"}
@@ -76,3 +101,5 @@ def test_engines_uninstall_llamacpp_with_specific_variant_and_version_should_be_
             "http://localhost:3928/v1/engines/llama-cpp/install", json=data
         )
         assert response.status_code == 200
+
+    

engine/e2e-test/test_api_engine_get.py

@@ -1,33 +1,29 @@
 import pytest
 import requests
+import time
 from test_runner import (
     run,
     start_server,
     stop_server,
     wait_for_websocket_download_success_event,
 )
 
-
-class TestApiModelPullDirectUrl:
+class TestApiModel:
     @pytest.fixture(autouse=True)
     def setup_and_teardown(self):
-        # Setup
-        stop_server()
+        # Setup        
         success = start_server()
         if not success:
             raise Exception("Failed to start server")
         # Delete model if exists
-        run(
-            "Delete model",
-            [
-                "models",
-                "delete",
-                "afrideva:zephyr-smol_llama-100m-sft-full-GGUF:zephyr-smol_llama-100m-sft-full.q2_k.gguf",
-            ],
-        )
         yield
 
         # Teardown
+        stop_server()
+        
+    # Pull with direct url    
+    @pytest.mark.asyncio
+    async def test_model_pull_with_direct_url_should_be_success(self):
         run(
             "Delete model",
             [
@@ -36,10 +32,7 @@ def setup_and_teardown(self):
                 "afrideva:zephyr-smol_llama-100m-sft-full-GGUF:zephyr-smol_llama-100m-sft-full.q2_k.gguf",
             ],
         )
-        stop_server()
-
-    @pytest.mark.asyncio
-    async def test_model_pull_with_direct_url_should_be_success(self):
+        
         myobj = {
             "model": "https://huggingface.co/afrideva/zephyr-smol_llama-100m-sft-full-GGUF/blob/main/zephyr-smol_llama-100m-sft-full.q2_k.gguf"
         }
@@ -54,6 +47,15 @@ async def test_model_pull_with_direct_url_should_be_success(self):
             get_model_response.json()["model"]
             == "afrideva:zephyr-smol_llama-100m-sft-full-GGUF:zephyr-smol_llama-100m-sft-full.q2_k.gguf"
         )
+        
+        run(
+            "Delete model",
+            [
+                "models",
+                "delete",
+                "afrideva:zephyr-smol_llama-100m-sft-full-GGUF:zephyr-smol_llama-100m-sft-full.q2_k.gguf",
+            ],
+        )
 
     @pytest.mark.asyncio
     async def test_model_pull_with_direct_url_should_have_desired_name(self):
@@ -73,3 +75,58 @@ async def test_model_pull_with_direct_url_should_have_desired_name(self):
             get_model_response.json()["name"]
             == "smol_llama_100m"
         )
+        
+        run(
+            "Delete model",
+            [
+                "models",
+                "delete",
+                "afrideva:zephyr-smol_llama-100m-sft-full-GGUF:zephyr-smol_llama-100m-sft-full.q2_k.gguf",
+            ],
+        )
+        
+    async def test_models_start_stop_should_be_successful(self):
+        print("Install engine")
+        response = requests.post("http://localhost:3928/v1/engines/llama-cpp/install")
+        assert response.status_code == 200
+        await wait_for_websocket_download_success_event(timeout=None)
+        # TODO(sang) need to fix for cuda download
+        time.sleep(30)
+
+        print("Pull model")
+        json_body = {"model": "tinyllama:gguf"}
+        response = requests.post("http://localhost:3928/v1/models/pull", json=json_body)
+        assert response.status_code == 200, f"Failed to pull model: tinyllama:gguf"
+        await wait_for_websocket_download_success_event(timeout=None)
+        
+        # get API
+        print("Get model")
+        response = requests.get("http://localhost:3928/v1/models/tinyllama:gguf")
+        assert response.status_code == 200
+        
+        # list API
+        print("List model")
+        response = requests.get("http://localhost:3928/v1/models")
+        assert response.status_code == 200
+
+        print("Start model")
+        json_body = {"model": "tinyllama:gguf"}
+        response = requests.post(
+            "http://localhost:3928/v1/models/start", json=json_body
+        )
+        assert response.status_code == 200, f"status_code: {response.status_code}"
+
+        print("Stop model")
+        response = requests.post("http://localhost:3928/v1/models/stop", json=json_body)
+        assert response.status_code == 200, f"status_code: {response.status_code}"
+                
+        # update API
+        print("Update model")
+        body_json = {'model': 'tinyllama:gguf'}
+        response = requests.patch("http://localhost:3928/v1/models/tinyllama:gguf", json = body_json)        
+        assert response.status_code == 200
+
+        # delete API
+        print("Delete model")
+        response = requests.delete("http://localhost:3928/v1/models/tinyllama:gguf")
+        assert response.status_code == 200