test: add auto testcase for test project

add auto testcase  for test project, contains assistant, inference, retrieval, tool service

Author: taskingaijc

.gitignore

@@ -67,3 +67,4 @@ target/
 
 .swagger-codegen-ignore
 /.swagger-codegen/
+.DS_Store

test-requirements.txt

@@ -3,3 +3,15 @@ nose>=1.3.7
 pluggy>=0.3.1
 py>=1.4.31
 randomize>=0.13
+pytest==7.4.3
+allure-pytest==2.13.2
+pytest-ordering==0.6
+pytest-xdist==3.5.0
+PyYAML==6.0.1
+pytest-assume==2.4.3
+pytest-asyncio==0.21.1
+asyncio==3.4.3
+pytest-tornasync>=0.6.0
+pytest-trio==0.8.0
+pytest-twisted==1.14.0
+Twisted==23.10.0

test/__init__.py

@@ -1 +0,0 @@
-# coding: utf-8

test/common/app_auth.py

@@ -0,0 +1,25 @@
+import aiohttp
+from typing import List, Dict
+from test.common.utils import ResponseWrapper
+from test.config import Config
+
+AUTH_BASE_URL = Config.SPACE_SERVICE_URL + "/v1/auth"
+
+
+async def login(data: Dict):
+    # data = {email: str, password: str}
+    async with aiohttp.ClientSession() as session:
+        response = await session.post(AUTH_BASE_URL + "/login", json=data)
+        return ResponseWrapper(response.status, await response.json())
+
+BASE_URL = Config.PROJECT_SERVICE_URL
+
+
+# For GET /{space_id}/projects/list
+async def list_projects(token: str, space_id: str, data: Dict):
+    # data = {"offset": int, "limit": int}
+    headers = {"Authorization": "Bearer " + token}
+    async with aiohttp.ClientSession(headers=headers) as session:
+        request_url = f"{BASE_URL}/v1/{space_id}/projects/list"
+        response = await session.get(request_url, params=data)
+        return ResponseWrapper(response.status, await response.json())

test/common/logger.py

@@ -0,0 +1,61 @@
+import logging
+import time
+import os
+
+BASE_PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+LOG_PATH = os.path.join(BASE_PATH, "log")
+if not os.path.exists(LOG_PATH):
+    os.mkdir(LOG_PATH)
+
+
+class Logger:
+
+    def __init__(self):
+        self.log_name = os.path.join(LOG_PATH, "{}.log".format(time.strftime("%Y%m%d")))
+        self.logger = logging.getLogger("log")
+        self.logger.setLevel(logging.DEBUG)
+
+        self.formatter = logging.Formatter(
+            '[%(asctime)s][%(filename)s %(lineno)d][%(levelname)s]: %(message)s')
+
+        self.file_logger = logging.FileHandler(self.log_name, mode='a', encoding="UTF-8")
+        self.console = logging.StreamHandler()
+        self.console.setLevel(logging.DEBUG)
+        self.file_logger.setLevel(logging.DEBUG)
+        self.file_logger.setFormatter(self.formatter)
+        self.console.setFormatter(self.formatter)
+        self.logger.addHandler(self.file_logger)
+        self.logger.addHandler(self.console)
+
+
+logger = Logger().logger
+
+
+def logger_info_base(http_code: str, http_status: str, res):
+    logger.info("http_code ==>> except_res：{}， real_res：【 {} 】".format(http_code, res.status_code))
+    logger.info("http_status ==>> except_res：{}， real_res：【 {} 】".format(http_status, res.json()["status"]))
+
+
+def logger_info(http_code: str, http_status: str, res):
+    logger_info_base(http_code, http_status, res)
+    logger.info("total_count ==>> real_res：【 {} 】".format(res.json()["total_count"]))
+    logger.info("fetched_count ==>> real_res：【 {} 】".format(res.json()["fetched_count"]))
+
+
+def logger_ready_or_creating_info(http_code: str, http_status: str, res):
+    logger_info_base(http_code, http_status, res)
+    logger.info("data_status ==>> real_res：【 {} 】".format(res.json()["data"]["status"]))
+
+
+def logger_success_info(http_code: str, http_status: str, data_status, res):
+    logger_info_base(http_code, http_status, res)
+    logger.info("data_status ==>> except_res：{}， real_res：【 {} 】".format(data_status, res.json()["data"]["status"]))
+
+
+def logger_error_info(http_code: str, http_status: str, data_status, res):
+    logger_info_base(http_code, http_status, res)
+    logger.info("data_status ==>> except_res：{}， real_res：【 {} 】".format(data_status, res.json()["error"]["code"]))
+
+
+

test/common/read_data.py

@@ -0,0 +1,59 @@
+import yaml
+import json
+import os
+from configparser import ConfigParser
+from test.common.logger import logger
+
+
+class MyConfigParser(ConfigParser):
+
+    def __init__(self, defaults=None):
+        ConfigParser.__init__(self, defaults=defaults)
+
+    def optionxform(self, option_str):
+        return option_str
+
+    def get_sections(self):
+        return self._sections
+
+
+class ReadFileData:
+
+    def __init__(self):
+        self.base_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+    def load_yaml(self, yaml_file_name):
+        file_path = os.path.join(self.base_path, "data", yaml_file_name)
+        logger.info("load {} file......".format(file_path))
+        with open(file_path, encoding='utf-8') as f:
+            _data = yaml.safe_load(f)
+        logger.info("read data ==>>  {} ".format(_data))
+        return _data
+
+    def load_json(self):
+        file_path = os.path.join(self.base_path, "config", "setting.json")
+        logger.info("load {} file......".format(file_path))
+        with open(file_path, encoding='utf-8') as f:
+            _data = json.load(f)
+        logger.info("read data ==>>  {} ".format(_data))
+        return _data
+
+    def load_ini(self):
+        file_path = os.path.join(self.base_path, "config", "setting.ini")
+        logger.info("load {} file......".format(file_path))
+        config = MyConfigParser()
+        config.read(file_path, encoding="UTF-8")
+        _data = config.get_sections()
+        # print("read data ==>>  {} ".format(_data))
+        return _data
+
+    def write_yml(self, yaml_file_name, _data):
+        file_path = os.path.join(self.base_path, "data", yaml_file_name)
+        logger.info("load {} file......".format(file_path))
+        with open(file_path,  'wb') as f:
+            logger.info("write data ==>>  {} ".format(_data))
+            yaml.safe_dump(_data, f, default_flow_style=False,
+                           explicit_start=True, allow_unicode=True, encoding='utf-8')
+
+
+data = ReadFileData()

test/common/utils.py

@@ -0,0 +1,192 @@
+from typing import Dict, Any
+import asyncio
+import random
+import string
+import pytest
+
+from test.common.logger import logger
+
+
+class ResponseWrapper:
+    def __init__(self, status: int, json_data: Dict):
+        self.status_code = status
+        self._json_data = json_data
+
+    def json(self):
+        return self._json_data
+
+
+def list_to_dict(data: list):
+    d = {}
+    for i in data:
+        if isinstance(i, dict):
+            for k, v in i.items():
+                d[k] = v
+        else:
+            d.update(i)
+    return d
+
+
+def get_random_email():
+    domain = ["163.com", "126.com", "yeah.net", "vip.163.com", "vip.126.com", "188.com", "vip.188.com", "qq.com",
+              "gmail.com", "yahoo.com", "sina.com", "sina.cn", "sohu.com", "sogou.com", "outlook.com", "189.com",
+              "wo.cn", "139.com", "ailiyun.com", "icloud.com", "tianya.cn", "renren.com", "tom.com"]
+    username_length = random.randint(5, 10)
+    username = ''.join(random.choices(string.ascii_lowercase, k=username_length))
+    domain_name = random.choice(domain)
+    email = username + "@" + domain_name
+    return email
+
+
+def get_password():
+    return ''.join(random.choices(string.ascii_letters, k=7))+str(random.randint(0, 9))
+
+
+def assume(res, except_dict: Dict[str, Any]):
+    pytest.assume(res.status_code == int(except_dict["except_http_code"]))
+    pytest.assume(res.json()["status"] == except_dict["except_status"])
+
+
+def assume_success(res, except_dict: Dict[str, Any]):
+    assume(res, except_dict)
+    pytest.assume(res.json()["data"]["status"] == except_dict["except_data_status"])
+
+
+def assume_error(res, except_dict: Dict[str, Any]):
+    assume(res, except_dict)
+    pytest.assume(res.json()["error"]["code"] == except_dict["except_error_code"])
+
+
+def assume_count(res, except_dict: Dict[str, Any]):
+    assume(res, except_dict)
+    pytest.assume(res.json()["total_count"] >= 0)
+    pytest.assume(res.json()["fetched_count"] >= 0)
+    pytest.assume(len(res.json()["data"]) == res.json()["fetched_count"])
+
+
+def assume_assistant(res, assistant_dict: Dict[str, Any]):
+    for key, value in assistant_dict.items():
+        if key == "error":
+            pass
+        else:
+            pytest.assume(res[key] == assistant_dict[key])
+
+
+def assume_collection(res, collection_dict: Dict[str, Any]):
+    for key, value in collection_dict.items():
+        if key == "configs":
+            for k, v in value.items():
+                pytest.assume(res.json()["data"][key][k] == collection_dict[key][k])
+        elif key == "error":
+            pass
+        else:
+            pytest.assume(res.json()["data"][key] == collection_dict[key])
+
+
+def assume_record(res, record_dict: Dict[str, Any]):
+    for key, value in record_dict.items():
+        if key == "type":
+            pytest.assume(res.json()["data"][key] == record_dict[key])
+        elif key == "error":
+            pass
+        elif key == "text":
+            pytest.assume(res.json()["data"]["content"][key] == record_dict[key])
+
+
+def assume_chunk(res, chunk_dict: Dict[str, Any]):
+    for key, value in chunk_dict.items():
+        if key == "error":
+            pass
+        else:
+            pytest.assume(len(res.json()["data"]) == res.json()["fetched_count"] == chunk_dict["top_k"])
+
+
+async def mq_execute_create_collection(n, func,  *args, **kwargs):
+    if n == 10:
+        logger.info(func+"timeout")
+        pytest.assume(False)
+    else:
+        from test.retrieval_service.collection import get_collection
+        res = await get_collection(*args, **kwargs)
+        if res.json()["data"]["status"] == "ready" and res.json()["data"]["num_records"] >= 0 and res.json()["data"]["num_chunks"] >= 0:
+            return
+        else:
+            await asyncio.sleep(1)
+            logger.info(f"collection status is {res.json()['data']['status']}")
+            logger.info(f"collection num_records is {res.json()['data']['num_records']}")
+            logger.info(f"collection num_chunks is {res.json()['data']['num_chunks']}")
+            await mq_execute_create_collection(n+1, func, *args, **kwargs)
+
+
+async def mq_execute_delete_collection(n, func,  *args, **kwargs):
+    if n == 10:
+        logger.info(func + "timeout")
+        pytest.assume(False)
+    else:
+        from test.retrieval_service.collection import get_collection
+        res = await get_collection(*args, **kwargs)
+        if res.status_code == 404:
+            logger.info(f"project[{args[0]}]collection[{args[1]}] is deleted")
+            return
+        else:
+            await asyncio.sleep(1)
+            await mq_execute_delete_collection(n+1, func, *args, **kwargs)
+
+
+async def mq_execute_create_record(n, func, *args, **kwargs):
+    if n == 10:
+        logger.info(func + "timeout")
+        pytest.assume(False)
+    else:
+        from test.retrieval_service.record import get_record
+        res = await get_record(*args, **kwargs)
+        if res.json()["data"]["status"] == "ready" and res.json()["data"]["num_chunks"] >= 0:
+            logger.info(f"project[{args[0]}]collection[{args[1]}]record[{args[2]}] status is ready")
+            return
+        elif res.json()["data"]["status"] == "partial" and res.json()["data"]["num_chunks"] >= 0:
+            logger.info(f"project[{args[0]}]collection[{args[1]}]record[{args[2]}] is partial")
+            return
+        else:
+            await asyncio.sleep(1)
+            logger.info(f"record status is {res.json()['data']['status']}")
+            logger.info(f"record num_chunks is {res.json()['data']['num_chunks']}")
+            await mq_execute_create_record(n + 1, func, *args, **kwargs)
+
+
+async def mq_execute_delete_record(n, func, *args, **kwargs):
+    if n == 10:
+        logger.info(func + "timeout")
+        pytest.assume(False)
+    else:
+        from test.retrieval_service.record import get_record
+        res = await get_record(*args, **kwargs)
+        if res.status_code == 404:
+            logger.info(f"project[{args[0]}]collection[{args[1]}]record[{args[2]}] is deleted")
+            return
+        else:
+            await asyncio.sleep(1)
+            await mq_execute_delete_record(n + 1, func, *args, **kwargs)
+
+
+async def mq_execute_delete_project(n, func, *args, **kwargs):
+    if n == 10:
+        logger.info(func + "timeout")
+        pytest.assume(False)
+    else:
+        from test.retrieval_service.collection import list_collections
+        collections_res = await list_collections(*args, **kwargs)
+        if collections_res.status_code == 400:
+            return
+        else:
+            await asyncio.sleep(1)
+            await mq_execute_delete_project(n + 1, func, *args, **kwargs)
+
+
+def get_random():
+    return ''.join(random.choices(string.ascii_letters, k=7))+str(random.randint(0, 9))
+
+
+def get_project_id(i: int):
+    return (get_random() for j in range(i))
+
+

test/config.py

@@ -0,0 +1,8 @@
+import os
+
+# text_model_id = os.getenv("text_model_id")
+# chat_model_id = os.getenv("chat_model_id")
+
+text_model_id = "jTJxv3iQ"
+chat_model_id = "Pxjizpc5"
+TASKINGAI_API_KEY = "taHAGKN5nNYdfAw4VtDVDXq7WXriyzsd"