From c0ffc0aaf518f68d634944320cc9cc15373af0d5 Mon Sep 17 00:00:00 2001 From: liqiang-fit2cloud Date: Wed, 9 Apr 2025 09:57:11 +0800 Subject: [PATCH 001/124] security: fix reverse shell vulnerability in function library. --- apps/common/util/function_code.py | 2 +- installer/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/common/util/function_code.py b/apps/common/util/function_code.py index 31797a9f425..30ce3a33d20 100644 --- a/apps/common/util/function_code.py +++ b/apps/common/util/function_code.py @@ -88,7 +88,7 @@ def _exec_sandbox(self, _code, _id): os.system(f"chown {self.user}:{self.user} {exec_python_file}") kwargs = {'cwd': BASE_DIR} subprocess_result = subprocess.run( - ['su', '-c', python_directory + ' ' + exec_python_file, self.user], + ['su', '-s', python_directory, '-c', "exec(open('" + exec_python_file + "').read())", self.user], text=True, capture_output=True, **kwargs) os.remove(exec_python_file) diff --git a/installer/Dockerfile b/installer/Dockerfile index a2dc3f66264..d2c1eefb6fa 100644 --- a/installer/Dockerfile +++ b/installer/Dockerfile @@ -70,7 +70,7 @@ RUN chmod 755 /opt/maxkb/app/installer/run-maxkb.sh && \ useradd --no-create-home --home /opt/maxkb/app/sandbox sandbox -g root && \ chown -R sandbox:root /opt/maxkb/app/sandbox && \ chmod g-x /usr/local/bin/* /usr/bin/* /bin/* /usr/sbin/* /sbin/* /usr/lib/postgresql/15/bin/* && \ - chmod g+x /usr/local/bin/python* /bin/sh + chmod g+x /usr/local/bin/python* EXPOSE 8080 From f4212eba97e482eda5b44836c57c7471c2c95a46 Mon Sep 17 00:00:00 2001 From: Yitong Date: Wed, 9 Apr 2025 10:23:30 +0800 Subject: [PATCH 002/124] refactor: add sdlc Dockerfile --- installer/Dockerfile-sdlc | 101 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 installer/Dockerfile-sdlc diff --git a/installer/Dockerfile-sdlc b/installer/Dockerfile-sdlc new file mode 100644 index 00000000000..c93b8ea08ef --- /dev/null +++ b/installer/Dockerfile-sdlc @@ -0,0 +1,101 @@ +# 基础镜像使用 Python + PostgreSQL +FROM python:3.11-slim-bullseye AS python-pg-base + +# 设置时区和安装基础依赖 +ARG DEPENDENCIES=" \ + libexpat1-dev \ + libffi-dev \ + curl \ + ca-certificates \ + vim \ + gettext \ + postgresql-15-pgvector \ + python3-pip" + +RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + echo "Asia/Shanghai" > /etc/timezone && \ + apt-get update && apt-get install -y --no-install-recommends $DEPENDENCIES && \ + apt-get clean all && \ + rm -rf /var/lib/apt/lists/* + +# 向量模型构建阶段 +FROM python-pg-base AS vector-model +COPY installer/install_model.py install_model.py +RUN pip3 install --upgrade pip setuptools && \ + pip install pycrawlers transformers && \ + python3 install_model.py + +# Web 构建阶段 +FROM node:18-alpine3.18 AS web-build +COPY ui ui +RUN cd ui && \ + npm install && \ + npm run build && \ + rm -rf ./node_modules + +# 应用构建阶段 +FROM python-pg-base AS stage-build + +COPY . /opt/maxkb/app +RUN mkdir -p /opt/maxkb/app /opt/maxkb/model /opt/maxkb/conf && \ + rm -rf /opt/maxkb/app/ui + +COPY --from=web-build ui /opt/maxkb/app/ui +WORKDIR /opt/maxkb/app +RUN python3 -m venv /opt/py3 && \ + pip install poetry==1.8.5 --break-system-packages && \ + poetry config virtualenvs.create false && \ + . /opt/py3/bin/activate && \ + if [ "$(uname -m)" = "x86_64" ]; then sed -i 's/^torch.*/torch = {version = "^2.6.0+cpu", source = "pytorch"}/g' pyproject.toml; fi && \ + poetry install && \ + export MAXKB_CONFIG_TYPE=ENV && python3 /opt/maxkb/app/apps/manage.py compilemessages + +# 最终阶段 +FROM python-pg-base +ARG DOCKER_IMAGE_TAG=dev \ + BUILD_AT \ + GITHUB_COMMIT + +# 设置环境变量 +ENV MAXKB_VERSION="${DOCKER_IMAGE_TAG} (build at ${BUILD_AT}, commit: ${GITHUB_COMMIT})" \ + MAXKB_CONFIG_TYPE=ENV \ + MAXKB_DB_NAME=maxkb \ + MAXKB_DB_HOST=127.0.0.1 \ + MAXKB_DB_PORT=5432 \ + MAXKB_DB_USER=root \ + MAXKB_DB_PASSWORD=Password123@postgres \ + MAXKB_DB_MAX_OVERFLOW=80 \ + MAXKB_EMBEDDING_MODEL_NAME=/opt/maxkb/model/embedding/shibing624_text2vec-base-chinese \ + MAXKB_EMBEDDING_MODEL_PATH=/opt/maxkb/model/embedding \ + MAXKB_SANDBOX=1 \ + LANG=en_US.UTF-8 \ + PATH=/opt/py3/bin:$PATH \ + POSTGRES_USER=root \ + POSTGRES_PASSWORD=Password123@postgres \ + POSTGRES_MAX_CONNECTIONS=1000 \ + PIP_TARGET=/opt/maxkb/app/sandbox/python-packages \ + PYTHONPATH=/opt/maxkb/app/sandbox/python-packages \ + PYTHONUNBUFFERED=1 + +WORKDIR /opt/maxkb/app +COPY --from=stage-build /opt/maxkb /opt/maxkb +COPY --from=stage-build /opt/py3 /opt/py3 +COPY --from=vector-model /opt/maxkb/model /opt/maxkb/model + +RUN chmod 755 /opt/maxkb/app/installer/run-maxkb.sh && \ + cp -r /opt/maxkb/model/base/hub /opt/maxkb/model/tokenizer && \ + cp -f /opt/maxkb/app/installer/run-maxkb.sh /usr/bin/run-maxkb.sh && \ + cp -f /opt/maxkb/app/installer/init.sql /docker-entrypoint-initdb.d && \ + curl -L --connect-timeout 120 -m 1800 https://resource.fit2cloud.com/maxkb/ffmpeg/get-ffmpeg-linux | sh && \ + mkdir -p /opt/maxkb/app/sandbox/python-packages && \ + find /opt/maxkb/app -mindepth 1 -not -name 'sandbox' -exec chmod 700 {} + && \ + chmod 755 /tmp && \ + useradd --no-create-home --home /opt/maxkb/app/sandbox sandbox -g root && \ + chown -R sandbox:root /opt/maxkb/app/sandbox && \ + chmod g-x /usr/local/bin/* /usr/bin/* /bin/* /usr/sbin/* /sbin/* /usr/lib/postgresql/15/bin/* && \ + chmod g+x /usr/local/bin/python* /bin/sh + +EXPOSE 8080 + +ENTRYPOINT ["bash", "-c"] +CMD [ "/usr/bin/run-maxkb.sh" ] \ No newline at end of file From 46364f2cbab8029d4ec428763448fc5a1e2b07e5 Mon Sep 17 00:00:00 2001 From: Yitong Date: Wed, 9 Apr 2025 13:38:43 +0800 Subject: [PATCH 003/124] refactor: add sdlc Dockerfile --- installer/Dockerfile-sdlc | 47 +++++++------------------- installer/milvus-start.sh | 4 +++ installer/run-build-sdlc-base-image.sh | 6 ++++ 3 files changed, 22 insertions(+), 35 deletions(-) create mode 100644 installer/milvus-start.sh create mode 100644 installer/run-build-sdlc-base-image.sh diff --git a/installer/Dockerfile-sdlc b/installer/Dockerfile-sdlc index c93b8ea08ef..2ec990a58f7 100644 --- a/installer/Dockerfile-sdlc +++ b/installer/Dockerfile-sdlc @@ -1,40 +1,19 @@ -# 基础镜像使用 Python + PostgreSQL -FROM python:3.11-slim-bullseye AS python-pg-base - -# 设置时区和安装基础依赖 -ARG DEPENDENCIES=" \ - libexpat1-dev \ - libffi-dev \ - curl \ - ca-certificates \ - vim \ - gettext \ - postgresql-15-pgvector \ - python3-pip" - -RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ - echo "Asia/Shanghai" > /etc/timezone && \ - apt-get update && apt-get install -y --no-install-recommends $DEPENDENCIES && \ - apt-get clean all && \ - rm -rf /var/lib/apt/lists/* - -# 向量模型构建阶段 -FROM python-pg-base AS vector-model -COPY installer/install_model.py install_model.py -RUN pip3 install --upgrade pip setuptools && \ - pip install pycrawlers transformers && \ - python3 install_model.py - -# Web 构建阶段 +FROM sdlc-vector-model:v1.0.1 AS vector-model FROM node:18-alpine3.18 AS web-build COPY ui ui RUN cd ui && \ npm install && \ npm run build && \ rm -rf ./node_modules +FROM sdlc-python-pg:python3.11-pg15.8 AS stage-build + +ARG DEPENDENCIES=" \ + python3-pip" -# 应用构建阶段 -FROM python-pg-base AS stage-build +RUN apt-get update && \ + apt-get install -y --no-install-recommends $DEPENDENCIES && \ + apt-get clean all && \ + rm -rf /var/lib/apt/lists/* COPY . /opt/maxkb/app RUN mkdir -p /opt/maxkb/app /opt/maxkb/model /opt/maxkb/conf && \ @@ -50,13 +29,11 @@ RUN python3 -m venv /opt/py3 && \ poetry install && \ export MAXKB_CONFIG_TYPE=ENV && python3 /opt/maxkb/app/apps/manage.py compilemessages -# 最终阶段 -FROM python-pg-base +FROM sdlc-python-pg:python3.11-pg15.8 ARG DOCKER_IMAGE_TAG=dev \ BUILD_AT \ GITHUB_COMMIT -# 设置环境变量 ENV MAXKB_VERSION="${DOCKER_IMAGE_TAG} (build at ${BUILD_AT}, commit: ${GITHUB_COMMIT})" \ MAXKB_CONFIG_TYPE=ENV \ MAXKB_DB_NAME=maxkb \ @@ -80,7 +57,7 @@ ENV MAXKB_VERSION="${DOCKER_IMAGE_TAG} (build at ${BUILD_AT}, commit: ${GITHUB_C WORKDIR /opt/maxkb/app COPY --from=stage-build /opt/maxkb /opt/maxkb COPY --from=stage-build /opt/py3 /opt/py3 -COPY --from=vector-model /opt/maxkb/model /opt/maxkb/model +COPY --from=vector-model /opt/maxkb/app/model /opt/maxkb/model RUN chmod 755 /opt/maxkb/app/installer/run-maxkb.sh && \ cp -r /opt/maxkb/model/base/hub /opt/maxkb/model/tokenizer && \ @@ -98,4 +75,4 @@ RUN chmod 755 /opt/maxkb/app/installer/run-maxkb.sh && \ EXPOSE 8080 ENTRYPOINT ["bash", "-c"] -CMD [ "/usr/bin/run-maxkb.sh" ] \ No newline at end of file +CMD [ "/usr/bin/run-maxkb.sh" ] diff --git a/installer/milvus-start.sh b/installer/milvus-start.sh new file mode 100644 index 00000000000..13704c19bf5 --- /dev/null +++ b/installer/milvus-start.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +curl -sfL https://raw.githubusercontent.com/milvus-io/milvus/master/scripts/standalone_embed.sh -o standalone_embed.sh +bash standalone_embed.sh start \ No newline at end of file diff --git a/installer/run-build-sdlc-base-image.sh b/installer/run-build-sdlc-base-image.sh new file mode 100644 index 00000000000..8700dae817b --- /dev/null +++ b/installer/run-build-sdlc-base-image.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cd ../ +docker rmi -f sdlc-python-pg:python3.11-pg15.8 +docker build -f installer/Dockerfile-python-pg -t sdlc-python-pg:python3.11-pg15.8 . +docker rmi -f sdlc-vector-model:v1.0.1 +docker build -f Dockerfile-vector-model -t sdlc-vector-model:v1.0.1 . From 0b60a03e5df917b91b153227f8cf8d6f7510f292 Mon Sep 17 00:00:00 2001 From: ivy <188476399@qq.com> Date: Fri, 11 Apr 2025 16:56:41 +0800 Subject: [PATCH 004/124] perf: refine copywriting --- ui/src/locales/lang/en-US/views/system.ts | 1 + ui/src/locales/lang/zh-CN/views/system.ts | 3 ++- ui/src/locales/lang/zh-Hant/views/system.ts | 3 ++- ui/src/router/modules/setting.ts | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ui/src/locales/lang/en-US/views/system.ts b/ui/src/locales/lang/en-US/views/system.ts index 303d1175dcf..6488ed6b5ae 100644 --- a/ui/src/locales/lang/en-US/views/system.ts +++ b/ui/src/locales/lang/en-US/views/system.ts @@ -1,5 +1,6 @@ export default { title: 'System', + subTitle: 'Setting', test: 'Test Connection', testSuccess: 'Successful', testFailed: 'Test connection failed', diff --git a/ui/src/locales/lang/zh-CN/views/system.ts b/ui/src/locales/lang/zh-CN/views/system.ts index 9ce23d90d86..1b72e1e01e1 100644 --- a/ui/src/locales/lang/zh-CN/views/system.ts +++ b/ui/src/locales/lang/zh-CN/views/system.ts @@ -1,5 +1,6 @@ export default { - title: '系统设置', + title: '系统管理', + subTitle: '系统设置', test: '测试连接', testSuccess: '测试连接成功', testFailed: '测试连接失败', diff --git a/ui/src/locales/lang/zh-Hant/views/system.ts b/ui/src/locales/lang/zh-Hant/views/system.ts index 10259390be1..e27e6c1228c 100644 --- a/ui/src/locales/lang/zh-Hant/views/system.ts +++ b/ui/src/locales/lang/zh-Hant/views/system.ts @@ -1,5 +1,6 @@ export default { - title: '系統設置', + title: '系統管理', + subTitle: '系統設置', test: '測試連線', testSuccess: '測試連線成功', testFailed: '測試連線失敗', diff --git a/ui/src/router/modules/setting.ts b/ui/src/router/modules/setting.ts index e97a658b02b..eaedb6a5f50 100644 --- a/ui/src/router/modules/setting.ts +++ b/ui/src/router/modules/setting.ts @@ -59,7 +59,7 @@ const settingRouter = { meta: { icon: 'app-setting', iconActive: 'app-setting-active', - title: 'common.setting', + title: 'views.system.subTitle', activeMenu: '/setting', parentPath: '/setting', parentName: 'setting', From 3b24373cd076e17842fbf374b06531f49cac4795 Mon Sep 17 00:00:00 2001 From: CaptainB Date: Mon, 14 Apr 2025 14:19:31 +0800 Subject: [PATCH 005/124] fix: handle line breaks in cell content for markdown table formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --bug=1054683 --user=刘瑞斌 【github#2831】知识库上传excel、应用编排文档内容提取节点中上传excel，单元格中有换行，导入后没有在一个单元格里显示 https://www.tapd.cn/57709429/s/1685274 --- apps/common/handle/impl/table/xls_parse_table_handle.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/common/handle/impl/table/xls_parse_table_handle.py b/apps/common/handle/impl/table/xls_parse_table_handle.py index 5609e3e8835..897e347e8a8 100644 --- a/apps/common/handle/impl/table/xls_parse_table_handle.py +++ b/apps/common/handle/impl/table/xls_parse_table_handle.py @@ -82,7 +82,10 @@ def get_content(self, file, save_image): for row in data: # 将每个单元格中的内容替换换行符为
以保留原始格式 md_table += '| ' + ' | '.join( - [str(cell).replace('\n', '
') if cell else '' for cell in row]) + ' |\n' + [str(cell) + .replace('\r\n', '
') + .replace('\n', '
') + if cell else '' for cell in row]) + ' |\n' md_tables += md_table + '\n\n' return md_tables From c781c11d268395f591f2046822a49987002cb3cf Mon Sep 17 00:00:00 2001 From: wangdan-fit2cloud <79562285+wangdan-fit2cloud@users.noreply.github.com> Date: Mon, 14 Apr 2025 15:11:55 +0800 Subject: [PATCH 006/124] fix: Application chat page style issue (#2866) --- ui/src/views/chat/base/index.vue | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/ui/src/views/chat/base/index.vue b/ui/src/views/chat/base/index.vue index 27be286f25a..7156f7d894a 100644 --- a/ui/src/views/chat/base/index.vue +++ b/ui/src/views/chat/base/index.vue @@ -42,7 +42,6 @@ -
diff --git a/ui/src/views/login/reset-password/index.vue b/ui/src/views/login/reset-password/index.vue index 2c2ff02576e..576e6f340a6 100644 --- a/ui/src/views/login/reset-password/index.vue +++ b/ui/src/views/login/reset-password/index.vue @@ -1,6 +1,10 @@