Add plugin info config and interfaces (#601)

* Add plugin info config and interfaces

* Remove test data

Author: wu-clan

.pre-commit-config.yaml

@@ -2,8 +2,8 @@ repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
-      - id: check-added-large-files
-      - id: end-of-file-fixer
+#      - id: check-added-large-files
+#      - id: end-of-file-fixer
       - id: check-yaml
       - id: check-toml
 

backend/app/admin/api/v1/sys/__init__.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+# !/usr/bin/env python3
 # -*- coding: utf-8 -*-
 from fastapi import APIRouter
 

backend/app/admin/api/v1/sys/plugin.py

@@ -1,72 +1,79 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-import io
-import os.path
-import zipfile
-
-from typing import Annotated
+from typing import Annotated, Any
 
 from fastapi import APIRouter, Depends, File, UploadFile
 from fastapi.params import Query
 from starlette.responses import StreamingResponse
 
-from backend.common.exception import errors
-from backend.common.response.response_schema import ResponseModel, response_base
+from backend.app.admin.service.plugin_service import plugin_service
+from backend.common.response.response_code import CustomResponseCode
+from backend.common.response.response_schema import ResponseModel, ResponseSchemaModel, response_base
+from backend.common.security.jwt import DependsJwtAuth
 from backend.common.security.permission import RequestPermission
 from backend.common.security.rbac import DependsRBAC
-from backend.core.path_conf import PLUGIN_DIR
-from backend.plugin.tools import install_requirements_async
 
 router = APIRouter()
 
 
+@router.get('', summary='获取所有插件', dependencies=[DependsJwtAuth])
+async def get_all_plugins() -> ResponseSchemaModel[list[dict[str, Any]]]:
+    plugins = await plugin_service.get_all()
+    return response_base.success(data=plugins)
+
+
+@router.post(
+    '/install/zip',
+    summary='安装 zip 插件',
+    description='使用插件 zip 压缩包进行安装',
+    dependencies=[
+        Depends(RequestPermission('sys:plugin:install')),
+        DependsRBAC,
+    ],
+)
+async def install_zip_plugin(file: Annotated[UploadFile, File()]) -> ResponseModel:
+    await plugin_service.install_zip(file=file)
+    return response_base.success(res=CustomResponseCode.PLUGIN_INSTALL_SUCCESS)
+
+
 @router.post(
-    '/install',
-    summary='安装插件',
-    description='需使用插件 zip 压缩包进行安装',
+    '/install/git',
+    summary='安装 git 插件',
+    description='使用插件 git 仓库地址进行安装，不限制平台；如果需要凭证，需在 git 仓库地址中添加凭证信息',
     dependencies=[
         Depends(RequestPermission('sys:plugin:install')),
         DependsRBAC,
     ],
 )
-async def install_plugin(file: Annotated[UploadFile, File()]) -> ResponseModel:
-    contents = await file.read()
-    file_bytes = io.BytesIO(contents)
-    if not zipfile.is_zipfile(file_bytes):
-        raise errors.ForbiddenError(msg='插件压缩包格式非法')
-    with zipfile.ZipFile(file_bytes) as zf:
-        # 校验压缩包
-        plugin_dir_in_zip = f'{file.filename[:-4]}/backend/plugin/'
-        members_in_plugin_dir = [name for name in zf.namelist() if name.startswith(plugin_dir_in_zip)]
-        if not members_in_plugin_dir:
-            raise errors.ForbiddenError(msg='插件压缩包内容非法')
-        plugin_name = members_in_plugin_dir[1].replace(plugin_dir_in_zip, '').replace('/', '')
-        if (
-            len(members_in_plugin_dir) <= 3
-            or f'{plugin_dir_in_zip}{plugin_name}/plugin.toml' not in members_in_plugin_dir
-            or f'{plugin_dir_in_zip}{plugin_name}/README.md' not in members_in_plugin_dir
-        ):
-            raise errors.ForbiddenError(msg='插件压缩包内缺少必要文件')
+async def install_git_plugin(repo_url: Annotated[str, Query(description='插件 git 仓库地址')]) -> ResponseModel:
+    await plugin_service.install_git(repo_url=repo_url)
+    return response_base.success(res=CustomResponseCode.PLUGIN_INSTALL_SUCCESS)
+
 
-        # 插件是否可安装
-        full_plugin_path = os.path.join(PLUGIN_DIR, plugin_name)
-        if os.path.exists(full_plugin_path):
-            raise errors.ForbiddenError(msg='此插件已安装')
-        else:
-            os.makedirs(full_plugin_path, exist_ok=True)
+@router.post(
+    '/uninstall',
+    summary='卸载插件',
+    description='此操作会直接删除插件依赖，但不会直接删除插件，而是将插件移动到备份目录',
+    dependencies=[
+        Depends(RequestPermission('sys:plugin:uninstall')),
+        DependsRBAC,
+    ],
+)
+async def uninstall_plugin(plugin: Annotated[str, Query(description='插件名称')]) -> ResponseModel:
+    await plugin_service.uninstall(plugin=plugin)
+    return response_base.success(res=CustomResponseCode.PLUGIN_UNINSTALL_SUCCESS)
 
-        # 解压（安装）
-        members = []
-        for member in zf.infolist():
-            if member.filename.startswith(plugin_dir_in_zip):
-                new_filename = member.filename.replace(plugin_dir_in_zip, '')
-                if new_filename:
-                    member.filename = new_filename
-                    members.append(member)
-        zf.extractall(PLUGIN_DIR, members)
-        if os.path.exists(os.path.join(full_plugin_path, 'requirements.txt')):
-            await install_requirements_async()
 
+@router.post(
+    '/status',
+    summary='更新插件状态',
+    dependencies=[
+        Depends(RequestPermission('sys:plugin:status')),
+        DependsRBAC,
+    ],
+)
+async def update_plugin_status(plugin: Annotated[str, Query(description='插件名称')]) -> ResponseModel:
+    await plugin_service.update_status(plugin=plugin)
     return response_base.success()
 
 
@@ -79,19 +86,7 @@ async def install_plugin(file: Annotated[UploadFile, File()]) -> ResponseModel:
     ],
 )
 async def build_plugin(plugin: Annotated[str, Query(description='插件名称')]) -> StreamingResponse:
-    plugin_dir = os.path.join(PLUGIN_DIR, plugin)
-    if not os.path.exists(plugin_dir):
-        raise errors.ForbiddenError(msg='插件不存在')
-
-    bio = io.BytesIO()
-    with zipfile.ZipFile(bio, 'w') as zf:
-        for root, dirs, files in os.walk(plugin_dir):
-            dirs[:] = [d for d in dirs if d != '__pycache__']
-            for file in files:
-                file_path = os.path.join(root, file)
-                arcname = os.path.relpath(file_path, start=plugin_dir)
-                zf.write(file_path, arcname)
-
+    bio = await plugin_service.build(plugin=plugin)
     bio.seek(0)
     return StreamingResponse(
         bio,

backend/app/admin/service/plugin_service.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import io
+import json
+import os
+import shutil
+import zipfile
+
+from typing import Any
+
+from dulwich import porcelain
+from fastapi import UploadFile
+
+from backend.common.enums import StatusType
+from backend.common.exception import errors
+from backend.common.log import log
+from backend.core.conf import settings
+from backend.core.path_conf import PLUGIN_DIR
+from backend.database.redis import redis_client
+from backend.plugin.tools import install_requirements_async, uninstall_requirements_async
+from backend.utils.re_verify import is_git_url
+from backend.utils.timezone import timezone
+
+
+class PluginService:
+    """插件服务类"""
+
+    @staticmethod
+    async def get_all() -> list[dict[str, Any]]:
+        """获取所有插件"""
+        keys = []
+        result = []
+
+        async for key in redis_client.scan_iter(f'{settings.PLUGIN_REDIS_PREFIX}:info:*'):
+            keys.append(key)
+
+        for info in await redis_client.mget(*keys):
+            result.append(json.loads(info))
+
+        return result
+
+    @staticmethod
+    async def install_zip(*, file: UploadFile) -> None:
+        """
+        通过 zip 压缩包安装插件
+
+        :param file: 插件 zip 压缩包
+        :return:
+        """
+        contents = await file.read()
+        file_bytes = io.BytesIO(contents)
+        if not zipfile.is_zipfile(file_bytes):
+            raise errors.ForbiddenError(msg='插件压缩包格式非法')
+        with zipfile.ZipFile(file_bytes) as zf:
+            # 校验压缩包
+            plugin_dir_in_zip = file.filename[:-4]
+            members_in_plugin_dir = [name for name in zf.namelist() if name.startswith(plugin_dir_in_zip)]
+            if not members_in_plugin_dir:
+                raise errors.ForbiddenError(msg='插件压缩包内容非法')
+            plugin_name = members_in_plugin_dir[1].replace(plugin_dir_in_zip, '').replace('/', '')
+            if (
+                len(members_in_plugin_dir) <= 3
+                or f'{plugin_dir_in_zip}/plugin.toml' not in members_in_plugin_dir
+                or f'{plugin_dir_in_zip}/README.md' not in members_in_plugin_dir
+            ):
+                raise errors.ForbiddenError(msg='插件压缩包内缺少必要文件')
+
+            # 插件是否可安装
+            full_plugin_path = os.path.join(PLUGIN_DIR, plugin_name)
+            if os.path.exists(full_plugin_path):
+                raise errors.ForbiddenError(msg='此插件已安装')
+            else:
+                os.makedirs(full_plugin_path, exist_ok=True)
+
+            # 解压（安装）
+            members = []
+            for member in zf.infolist():
+                if member.filename.startswith(plugin_dir_in_zip):
+                    new_filename = member.filename.replace(plugin_dir_in_zip, '')
+                    if new_filename:
+                        member.filename = new_filename
+                        members.append(member)
+            zf.extractall(PLUGIN_DIR, members)
+
+        await install_requirements_async(plugin_name)
+
+    @staticmethod
+    async def install_git(*, repo_url: str):
+        """
+        通过 git 安装插件
+
+        :param repo_url: git 存储库的 URL
+        :return:
+        """
+        match = is_git_url(repo_url)
+        if not match:
+            raise errors.ForbiddenError(msg='Git 仓库地址格式非法')
+        repo_name = match.group('repo')
+        plugins = await redis_client.lrange(settings.PLUGIN_REDIS_PREFIX, 0, -1)
+        if repo_name in plugins:
+            raise errors.ForbiddenError(msg=f'{repo_name} 插件已安装')
+        try:
+            porcelain.clone(repo_url, os.path.join(PLUGIN_DIR, repo_name), checkout=True)
+        except Exception as e:
+            log.error(f'插件安装失败: {e}')
+            raise errors.ServerError(msg='插件安装失败，请稍后重试') from e
+        else:
+            await install_requirements_async(repo_name)
+
+    @staticmethod
+    async def uninstall(*, plugin: str):
+        """
+        卸载插件
+
+        :param plugin: 插件名称
+        :return:
+        """
+        plugin_dir = os.path.join(PLUGIN_DIR, plugin)
+        if not os.path.exists(plugin_dir):
+            raise errors.ForbiddenError(msg='插件不存在')
+        await uninstall_requirements_async(plugin)
+        bacup_dir = os.path.join(PLUGIN_DIR, f'{plugin}.{timezone.now().strftime("%Y%m%d%H%M%S")}.backup')
+        shutil.move(plugin_dir, bacup_dir)
+
+    @staticmethod
+    async def update_status(*, plugin: str):
+        """
+        更新插件状态
+
+        :param plugin: 插件名称
+        :return:
+        """
+        plugin_info = await redis_client.get(f'{settings.PLUGIN_REDIS_PREFIX}:info:{plugin}')
+        if not plugin_info:
+            raise errors.ForbiddenError(msg='插件不存在')
+        plugin_info = json.loads(plugin_info)
+        new_status = (
+            StatusType.enable.value
+            if plugin_info.get('plugin', {}).get('enable') == StatusType.disable.value
+            else StatusType.disable.value
+        )
+        plugin_info['plugin']['enable'] = new_status
+        await redis_client.set(
+            f'{settings.PLUGIN_REDIS_PREFIX}:info:{plugin}', json.dumps(plugin_info, ensure_ascii=False)
+        )
+        await redis_client.hset(f'{settings.PLUGIN_REDIS_PREFIX}:status', plugin, str(new_status))
+
+    @staticmethod
+    async def build(*, plugin: str) -> io.BytesIO:
+        """
+        打包插件为 zip 压缩包
+
+        :param plugin: 插件名称
+        :return:
+        """
+        plugin_dir = os.path.join(PLUGIN_DIR, plugin)
+        if not os.path.exists(plugin_dir):
+            raise errors.ForbiddenError(msg='插件不存在')
+
+        bio = io.BytesIO()
+        with zipfile.ZipFile(bio, 'w') as zf:
+            for root, dirs, files in os.walk(plugin_dir):
+                dirs[:] = [d for d in dirs if d != '__pycache__']
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    arcname = os.path.relpath(file_path, start=plugin_dir)
+                    zf.write(file_path, arcname)
+
+        return bio
+
+
+plugin_service: PluginService = PluginService()

backend/common/response/response_code.py

@@ -39,6 +39,10 @@ class CustomResponseCode(CustomCodeBase):
     HTTP_503 = (503, '服务器暂时无法处理请求')
     HTTP_504 = (504, '网关超时')
 
+    # Plugin
+    PLUGIN_INSTALL_SUCCESS = (200, '插件安装成功，请根据插件说明（README.md）进行相关配置并重启服务')
+    PLUGIN_UNINSTALL_SUCCESS = (200, '插件卸载成功，请根据插件说明（README.md）移除相关配置并重启服务')
+
 
 class CustomErrorCode(CustomCodeBase):
     """自定义错误状态码"""

backend/core/conf.py

@@ -181,9 +181,10 @@ class Settings(BaseSettings):
         'confirm_password',
     ]
 
-    # 插件配置
+    # Plugin 配置
     PLUGIN_PIP_CHINA: bool = True
     PLUGIN_PIP_INDEX_URL: str = 'https://mirrors.aliyun.com/pypi/simple/'
+    PLUGIN_REDIS_PREFIX: str = 'fba:plugin'
 
     # App Admin
     # .env OAuth2
@@ -200,11 +201,11 @@ class Settings(BaseSettings):
     CAPTCHA_LOGIN_EXPIRE_SECONDS: int = 60 * 5  # 3 分钟
 
     # App Task
-    # .env Redis 配置
+    # .env Redis
     CELERY_BROKER_REDIS_DATABASE: int
     CELERY_BACKEND_REDIS_DATABASE: int
 
-    # .env RabbitMQ 配置
+    # .env RabbitMQ
     # docker run -d --hostname fba-mq --name fba-mq  -p 5672:5672 -p 15672:15672 rabbitmq:latest
     CELERY_RABBITMQ_HOST: str
     CELERY_RABBITMQ_PORT: int
@@ -238,11 +239,9 @@ class Settings(BaseSettings):
     }
 
     # Plugin Code Generator
-    # 代码下载
     CODE_GENERATOR_DOWNLOAD_ZIP_FILENAME: str = 'fba_generator'
 
     # Plugin Config
-    # 参数配置
     CONFIG_BUILT_IN_TYPES: list[str] = ['website', 'protocol', 'policy']
 
     @model_validator(mode='before')

backend/plugin/code_generator/plugin.toml

@@ -1,2 +1,8 @@
+[plugin]
+summary = '代码生成'
+version = '0.0.1'
+description = '生成通用业务代码'
+author = 'wu-clan'
+
 [app]
 router = ['v1']