diff --git a/docSite/assets/imgs/image-39.png b/docSite/assets/imgs/image-39.png index 3c46ea9a3b30..0d522fd9b4a1 100644 Binary files a/docSite/assets/imgs/image-39.png and b/docSite/assets/imgs/image-39.png differ diff --git a/docSite/content/zh-cn/docs/guide/knowledge_base/lark_dataset.md b/docSite/content/zh-cn/docs/guide/knowledge_base/lark_dataset.md index 8631b0f0dbc8..91e3e3a7f3ac 100644 --- a/docSite/content/zh-cn/docs/guide/knowledge_base/lark_dataset.md +++ b/docSite/content/zh-cn/docs/guide/knowledge_base/lark_dataset.md @@ -11,9 +11,10 @@ weight: 405 | --- | --- | | ![alt text](/imgs/image-39.png) | ![alt text](/imgs/image-40.png) | -FastGPT v4.8.16 版本开始，商业版用户支持飞书知识库导入，用户可以通过配置飞书应用的 appId 和 appSecret，并选中一个**文档空间的顶层文件夹**来导入飞书知识库。目前处于测试阶段，部分交互有待优化。 + 目前，FastGPT用户支持飞书知识库导入，用户可以通过配置飞书应用的 appId 和 appSecret，并选中一个**文档空间的顶层文件夹**来导入飞书知识库。目前处于测试阶段，部分交互有待优化。 -由于飞书限制，无法直接获取所有文档内容，目前仅可以获取共享空间下文件目录的内容，无法获取个人空间和知识库里的内容。 +由于飞书限制，无法直接获取所有文档内容，目前仅可以获取共享空间下文件目录的内容，无法获取个人空间里的内容。 +如果需要获取知识库的内容，[点击这里](./lark_dataset.md) ## 1. 创建飞书应用 diff --git a/packages/global/common/system/types/index.d.ts b/packages/global/common/system/types/index.d.ts index f6f6c183d46d..c68c16af80b0 100644 --- a/packages/global/common/system/types/index.d.ts +++ b/packages/global/common/system/types/index.d.ts @@ -73,6 +73,9 @@ export type FastGPTFeConfigsType = { show_dataset_enhance?: boolean; show_batch_eval?: boolean; + feishu_auth_robot_client_id?: string; + feishu_auth_robot_client_secret?: string; + concatMd?: string; docUrl?: string; openAPIDocUrl?: string; diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts index 92dc32ed375e..75b96d5413b0 100644 --- a/packages/global/core/dataset/api.d.ts +++ b/packages/global/core/dataset/api.d.ts @@ -26,7 +26,9 @@ export type DatasetUpdateBody = { defaultPermission?: DatasetSchemaType['defaultPermission']; apiServer?: DatasetSchemaType['apiServer']; yuqueServer?: DatasetSchemaType['yuqueServer']; - feishuServer?: DatasetSchemaType['feishuServer']; + feishuShareServer?: DatasetSchemaType['feishuShareServer']; + feishuKnowledgeServer?: DatasetSchemaType['feishuKnowledgeServer']; + feishuPrivateServer?: DatasetSchemaType['feishuPrivateServer']; chunkSettings?: DatasetSchemaType['chunkSettings']; // sync schedule diff --git a/packages/global/core/dataset/apiDataset.d.ts b/packages/global/core/dataset/apiDataset.d.ts index 524b18674ef4..105ebc84bece 100644 --- a/packages/global/core/dataset/apiDataset.d.ts +++ b/packages/global/core/dataset/apiDataset.d.ts @@ -17,11 +17,26 @@ export type APIFileServer = { authorization?: string; basePath?: string; }; -export type FeishuServer = { - appId: string; - appSecret?: string; +export type FeishuShareServer = { + user_access_token: string; + refresh_token: string; + outdate_time: number; folderToken: string; }; +export type FeishuKnowledgeServer = { + user_access_token: string; + refresh_token: string; + outdate_time: number; + basePath?: string; +}; + +export type FeishuPrivateServer = { + user_access_token: string; + refresh_token: string; + outdate_time: number; + basePath?: string; +}; + export type YuqueServer = { userId: string; token?: string; diff --git a/packages/global/core/dataset/constants.ts b/packages/global/core/dataset/constants.ts index 25acca5632fc..537bc6be87b4 100644 --- a/packages/global/core/dataset/constants.ts +++ b/packages/global/core/dataset/constants.ts @@ -7,7 +7,9 @@ export enum DatasetTypeEnum { websiteDataset = 'websiteDataset', // depp link externalFile = 'externalFile', apiDataset = 'apiDataset', - feishu = 'feishu', + feishuShare = 'feishuShare', + feishuKnowledge = 'feishuKnowledge', + feishuPrivate = 'feishuPrivate', yuque = 'yuque' } export const DatasetTypeMap = { @@ -36,9 +38,19 @@ export const DatasetTypeMap = { label: i18nT('dataset:api_file'), collectionLabel: i18nT('common:File') }, - [DatasetTypeEnum.feishu]: { - icon: 'core/dataset/feishuDatasetOutline', - label: i18nT('dataset:feishu_dataset'), + [DatasetTypeEnum.feishuShare]: { + icon: 'core/dataset/feishuShareDatasetOutline', + label: i18nT('dataset:feishu_share_dataset'), + collectionLabel: i18nT('common:File') + }, + [DatasetTypeEnum.feishuKnowledge]: { + icon: 'core/dataset/feishuKnowledgeDatasetOutline', + label: i18nT('dataset:feishu_knowledge_dataset'), + collectionLabel: i18nT('common:File') + }, + [DatasetTypeEnum.feishuPrivate]: { + icon: 'core/dataset/feishuPrivateDatasetOutline', + label: i18nT('dataset:feishu_private_dataset'), collectionLabel: i18nT('common:File') }, [DatasetTypeEnum.yuque]: { diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index f3424eaa3fc7..e5392c415c96 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -13,7 +13,13 @@ import type { ChunkTriggerConfigTypeEnum } from './constants'; import type { DatasetPermission } from '../../support/permission/dataset/controller'; -import type { APIFileServer, FeishuServer, YuqueServer } from './apiDataset'; +import type { + APIFileServer, + FeishuShareServer, + YuqueServer, + FeishuKnowledgeServer, + FeishuPrivateServer +} from './apiDataset'; import type { SourceMemberType } from 'support/user/type'; import type { DatasetDataIndexTypeEnum } from './data/constants'; @@ -73,8 +79,10 @@ export type DatasetSchemaType = { inheritPermission: boolean; apiServer?: APIFileServer; - feishuServer?: FeishuServer; + feishuShareServer?: FeishuShareServer; yuqueServer?: YuqueServer; + feishuKnowledgeServer?: FeishuKnowledgeServer; + feishuPrivateServer?: FeishuPrivateServer; // abandon autoSync?: boolean; diff --git a/packages/service/common/api/type.d.ts b/packages/service/common/api/type.d.ts index 2a9af9e2ba8f..c7535dbefec6 100644 --- a/packages/service/common/api/type.d.ts +++ b/packages/service/common/api/type.d.ts @@ -1,5 +1,5 @@ import type { ApiDatasetDetailResponse } from '@fastgpt/global/core/dataset/apiDataset'; -import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset'; +import { FeishuShareServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset'; import type { DeepRagSearchProps, SearchDatasetDataResponse diff --git a/packages/service/core/dataset/apiDataset/feishuKnowledgeDataset/api.ts b/packages/service/core/dataset/apiDataset/feishuKnowledgeDataset/api.ts new file mode 100644 index 000000000000..e141e6f7632d --- /dev/null +++ b/packages/service/core/dataset/apiDataset/feishuKnowledgeDataset/api.ts @@ -0,0 +1,346 @@ +import type { + APIFileItem, + ApiFileReadContentResponse, + ApiDatasetDetailResponse, + FeishuKnowledgeServer +} from '@fastgpt/global/core/dataset/apiDataset'; +import { type ParentIdType } from '@fastgpt/global/common/parentFolder/type'; +import axios, { type Method } from 'axios'; +import { addLog } from '../../../../common/system/log'; + +type ResponseDataType = { + success: boolean; + message: string; + data: any; +}; + +/** + * Request + */ +type FeishuFileListResponse = { + items: { + title: string; + creator: string; + has_child: boolean; + parent_node_token: string; + owner_id: string; + space_id: string; + node_token: string; + node_type: string; + node_create_time: number; + obj_edit_time: number; + obj_create_time: number; + obj_token: string; + obj_type: string; + origin_node_token: string; + origin_space_id: string; + }[]; + has_more: boolean; + next_page_token: string; +}; + +type FeishuSpaceListResponse = { + code: number; + msg: string; + data: { + items: { + space_id: string; + name: string; + description: string; + open_sharing: boolean; + visibility: string; + space_type: string; + }[]; + has_more: boolean; + page_token: string; + }; +}; + +type FeishuNodeResponse = { + code: number; + msg: string; + data: { + node: { + title: string; + creator: string; + has_child: boolean; + parent_node_token: string; + owner_id: string; + space_id: string; + node_token: string; + node_type: string; + node_create_time: number; + obj_edit_time: number; + obj_create_time: number; + obj_token: string; + obj_type: string; + origin_node_token: string; + origin_space_id: string; + }; + }; +}; + +type FeishuSpaceResponse = { + code: number; + msg: string; + data: { + space: { + description: string; + name: string; + open_sharing: boolean; + space_id: string; + space_type: string; + visibility: string; + }; + }; +}; + +type FeishuDomainResponse = { + code: number; + msg: string; + data: { + tenant: { + domain: string; + name: string; + }; + }; +}; + +const feishuBaseUrl = process.env.FEISHU_BASE_URL || 'https://open.feishu.cn'; + +export const useFeishuKnowledgeDatasetRequest = ({ + feishuKnowledgeServer +}: { + feishuKnowledgeServer: FeishuKnowledgeServer; +}) => { + const instance = axios.create({ + baseURL: feishuBaseUrl, + timeout: 60000 + }); + + // 添加请求拦截器 + instance.interceptors.request.use(async (config) => { + if (!config.headers.Authorization) { + config.headers['Authorization'] = `Bearer ${feishuKnowledgeServer.user_access_token}`; + config.headers['Content-Type'] = 'application/json; charset=utf-8'; + } + return config; + }); + + /** + * 响应数据检查 + */ + const checkRes = (data: ResponseDataType) => { + if (data === undefined) { + addLog.info('yuque dataset data is empty'); + return Promise.reject('服务器异常'); + } + return data.data; + }; + const responseError = (err: any) => { + console.log('error->', '请求错误', err); + + if (!err) { + return Promise.reject({ message: '未知错误' }); + } + if (typeof err === 'string') { + return Promise.reject({ message: err }); + } + if (typeof err.message === 'string') { + return Promise.reject({ message: err.message }); + } + if (typeof err.data === 'string') { + return Promise.reject({ message: err.data }); + } + if (err?.response?.data) { + return Promise.reject(err?.response?.data); + } + return Promise.reject(err); + }; + + const request = (url: string, data: any, method: Method): Promise => { + /* 去空 */ + for (const key in data) { + if (data[key] === undefined) { + delete data[key]; + } + } + + return instance + .request({ + url, + method, + data: ['POST', 'PUT'].includes(method) ? data : undefined, + params: !['POST', 'PUT'].includes(method) ? data : undefined + }) + .then((res) => checkRes(res.data)) + .catch((err) => responseError(err)); + }; + + const listFiles = async ({ parentId }: { parentId?: ParentIdType }): Promise => { + const fetchSpaces = async ( + pageToken?: string + ): Promise => { + const response = await request( + `/open-apis/wiki/v2/spaces`, + { + lang: 'zh', + page_size: 50, + page_token: pageToken + }, + 'GET' + ); + + if (response.has_more) { + const nextFiles = await fetchSpaces(response.page_token); + return [...response.items, ...nextFiles]; + } + + return response.items; + }; + + if (!parentId) { + if (feishuKnowledgeServer.basePath) { + parentId = feishuKnowledgeServer.basePath; + } else { + const spaces = await fetchSpaces(); + return spaces.map((space) => ({ + id: space.space_id, + parentId: '', + name: space.name, + type: 'folder' as const, + hasChild: true, + updateTime: new Date(), + createTime: new Date() + })); + } + } + const spaceId = parentId.split('-')[0]; + const parent_node_token = parentId.split('-')[1]; + + const fetchFiles = async (pageToken?: string): Promise => { + const response = await request( + `/open-apis/wiki/v2/spaces/${spaceId}/nodes`, + { + page_size: 50, + page_token: pageToken, + parent_node_token: parent_node_token + }, + 'GET' + ); + + if (response.has_more) { + const nextFiles = await fetchFiles(response.next_page_token); + return [...response.items, ...nextFiles]; + } + + return response.items; + }; + const allFiles = await fetchFiles(); + const files = allFiles.filter((file) => ['folder', 'docx'].includes(file.obj_type)); + if (files.length === 0) { + return Promise.reject('There are no doc files in the current directory.'); + } + + return files.map((file) => ({ + id: spaceId + '-' + file.node_token, + parentId: file.parent_node_token ? spaceId + '-' + file.parent_node_token : spaceId, + name: file.title, + type: file.node_type === 'folder' ? ('folder' as const) : ('file' as const), + hasChild: file.has_child, + updateTime: new Date(file.obj_edit_time * 1000), + createTime: new Date(file.obj_create_time * 1000) + })); + }; + + const getNodeInfo = async (nodeToken: string) => + await request( + `/open-apis/wiki/v2/spaces/get_node`, + { + obj_type: 'wiki', + token: nodeToken + }, + 'GET' + ); + + const getFileContent = async ({ + apiFileId + }: { + apiFileId: string; + }): Promise => { + const nodeToken = apiFileId.split('-')[1]; + const node = await getNodeInfo(nodeToken); + const objToken = node.node.obj_token; + + const [{ content }, { document }] = await Promise.all([ + request<{ content: string }>( + `/open-apis/docx/v1/documents/${objToken}/raw_content`, + {}, + 'GET' + ), + request<{ document: { title: string } }>( + `/open-apis/docx/v1/documents/${objToken}`, + {}, + 'GET' + ) + ]); + + return { + title: document?.title, + rawText: content + }; + }; + + const getFilePreviewUrl = async ({ + apiFileId + }: { + apiFileId: string; + }): Promise => { + const nodeToken = apiFileId.split('-')[1]; + const response = await request( + `/open-apis/tenant/v2/tenant/query`, + {}, + 'GET' + ); + + return 'https://' + response.tenant.domain + '/wiki/' + nodeToken; + }; + + const getFileDetail = async ({ + apiFileId + }: { + apiFileId: string; + }): Promise => { + const spaceId = apiFileId.split('-')[0]; + const nodeToken = apiFileId.split('-')[1]; + const getSpace = await request( + `/open-apis/wiki/v2/spaces/${spaceId}`, + {}, + 'GET' + ); + if (nodeToken !== undefined) { + const getNode = await getNodeInfo(nodeToken); + + return { + name: getNode.node.title, + parentId: getNode.node.parent_node_token + ? spaceId + '-' + getNode.node.parent_node_token + : spaceId, + id: apiFileId + }; + } + + return { + name: getSpace.space.name, + parentId: null, + id: apiFileId + }; + }; + + return { + getFileContent, + listFiles, + getFilePreviewUrl, + getFileDetail + }; +}; diff --git a/packages/service/core/dataset/apiDataset/feishuPrivateDataset/api.ts b/packages/service/core/dataset/apiDataset/feishuPrivateDataset/api.ts new file mode 100644 index 000000000000..7ebec9117323 --- /dev/null +++ b/packages/service/core/dataset/apiDataset/feishuPrivateDataset/api.ts @@ -0,0 +1,234 @@ +import type { + APIFileItem, + ApiFileReadContentResponse, + ApiDatasetDetailResponse, + FeishuPrivateServer +} from '@fastgpt/global/core/dataset/apiDataset'; +import { type ParentIdType } from '@fastgpt/global/common/parentFolder/type'; +import axios, { type Method } from 'axios'; +import { addLog } from '../../../../common/system/log'; + +type ResponseDataType = { + success: boolean; + message: string; + data: any; +}; + +type FeishuFileListResponse = { + files: { + token: string; + parent_token: string; + name: string; + type: string; + modified_time: number; + created_time: number; + url: string; + owner_id: string; + shortcut_info?: { + target_token: string; + target_type: string; + }; + }[]; + has_more: boolean; + next_page_token: string; +}; + +type FeishuFileDetailResponse = { + code: number; + msg: string; + data: { + name: string; + parentId: string; + }; +}; + +const feishuBaseUrl = process.env.FEISHU_BASE_URL || 'https://open.feishu.cn'; + +export const useFeishuPrivateDatasetRequest = ({ + feishuPrivateServer +}: { + feishuPrivateServer: FeishuPrivateServer; +}) => { + const instance = axios.create({ + baseURL: feishuBaseUrl, + timeout: 60000 + }); + + instance.defaults.headers.common['Authorization'] = + `Bearer ${feishuPrivateServer.user_access_token}`; + instance.defaults.headers.common['Content-Type'] = 'application/json; charset=utf-8'; + + /** + * 响应数据检查 + */ + const checkRes = (data: ResponseDataType) => { + if (data === undefined) { + addLog.info('yuque dataset data is empty'); + return Promise.reject('服务器异常'); + } + return data.data; + }; + const responseError = (err: any) => { + console.log('error->', '请求错误', err); + + if (!err) { + return Promise.reject({ message: '未知错误' }); + } + if (typeof err === 'string') { + return Promise.reject({ message: err }); + } + if (typeof err.message === 'string') { + return Promise.reject({ message: err.message }); + } + if (typeof err.data === 'string') { + return Promise.reject({ message: err.data }); + } + if (err?.response?.data) { + return Promise.reject(err?.response?.data); + } + return Promise.reject(err); + }; + + const request = (url: string, data: any, method: Method): Promise => { + /* 去空 */ + for (const key in data) { + if (data[key] === undefined) { + delete data[key]; + } + } + + return instance + .request({ + url, + method, + data: ['POST', 'PUT'].includes(method) ? data : undefined, + params: !['POST', 'PUT'].includes(method) ? data : undefined + }) + .then((res) => checkRes(res.data)) + .catch((err) => responseError(err)); + }; + + const listFiles = async ({ parentId }: { parentId?: ParentIdType }): Promise => { + const fetchFiles = async ( + pageToken?: string, + parentId?: ParentIdType + ): Promise => { + const data = await request( + `/open-apis/drive/v1/files`, + { + page_size: 200, + page_token: pageToken, + folder_token: parentId ? parentId : undefined + }, + 'GET' + ); + if (data.has_more) { + const nextFiles = await fetchFiles(data.next_page_token); + return [...data.files, ...nextFiles]; + } + + return data.files; + }; + if (!parentId) { + parentId = feishuPrivateServer.basePath?.split('-').slice(-1)[0]; + } + const parent = parentId ? parentId.split('-').slice(-1)[0] : undefined; + + const allFiles = await fetchFiles(undefined, parent); + + return allFiles + .filter((file) => { + if (file.type === 'shortcut') { + return ( + file.shortcut_info?.target_type === 'docx' || + file.shortcut_info?.target_type === 'folder' + ); + } + return file.type === 'folder' || file.type === 'docx'; + }) + .map((file) => ({ + id: + file.type === 'shortcut' + ? parentId + '-' + file.shortcut_info!.target_token + : parentId + '-' + file.token, + parentId: parentId, + name: file.name, + type: file.type === 'folder' ? ('folder' as const) : ('file' as const), + hasChild: file.type === 'folder', + updateTime: new Date(file.modified_time * 1000), + createTime: new Date(file.created_time * 1000) + })); + }; + + const getFileContent = async ({ + apiFileId + }: { + apiFileId: string; + }): Promise => { + const fileId = apiFileId.split('-')[1]; + const [{ content }, { document }] = await Promise.all([ + request<{ content: string }>(`/open-apis/docx/v1/documents/${fileId}/raw_content`, {}, 'GET'), + request<{ document: { title: string } }>(`/open-apis/docx/v1/documents/${fileId}`, {}, 'GET') + ]); + + return { + title: document?.title, + rawText: content + }; + }; + + const getFilePreviewUrl = async ({ apiFileId }: { apiFileId: string }): Promise => { + const fileId = apiFileId.split('-')[1]; + const { metas } = await request<{ metas: { url: string }[] }>( + `/open-apis/drive/v1/metas/batch_query`, + { + request_docs: [ + { + doc_token: fileId, + doc_type: 'docx' + } + ], + with_url: true + }, + 'POST' + ); + + return metas[0].url; + }; + + const getFileDetail = async ({ + apiFileId + }: { + apiFileId: string; + }): Promise => { + const parentId = apiFileId.split('-').slice(0, -1).join('-'); + const fileId = apiFileId.split('-').slice(-1)[0]; + + const fileDetail = await request( + `/open-apis/drive/explorer/v2/folder/${fileId}/meta`, + {}, + 'GET' + ); + + if (!fileDetail) { + return { + name: '', + parentId: null, + id: apiFileId + }; + } + + return { + name: fileDetail?.name, + parentId: parentId !== 'null' ? parentId : null, + id: apiFileId + }; + }; + + return { + getFileContent, + listFiles, + getFilePreviewUrl, + getFileDetail + }; +}; diff --git a/packages/service/core/dataset/apiDataset/feishuPrivateDataset/refreshToken.ts b/packages/service/core/dataset/apiDataset/feishuPrivateDataset/refreshToken.ts new file mode 100644 index 000000000000..6383d2558b75 --- /dev/null +++ b/packages/service/core/dataset/apiDataset/feishuPrivateDataset/refreshToken.ts @@ -0,0 +1,80 @@ +import axios from 'axios'; +import { MongoDataset } from '../../schema'; +import { addLog } from '../../../../common/system/log'; + +/** + * refresh feishu token + * refresh token before 10 minutes + */ +export async function refreshFeishuToken() { + try { + const datasets = await MongoDataset.find({ + type: { $in: ['feishuPrivate', 'feishuShare', 'feishuKnowledge'] } + }); + + const appId = global.feConfigs?.feishu_auth_robot_client_id; + const appSecret = global.feConfigs?.feishu_auth_robot_client_secret; + + const refreshPromises = datasets + .filter( + (dataset) => + (dataset.type === 'feishuPrivate' && dataset.feishuPrivateServer) || + (dataset.type === 'feishuShare' && dataset.feishuShareServer) || + (dataset.type === 'feishuKnowledge' && dataset.feishuKnowledgeServer) + ) + .map(async (dataset) => { + try { + const serverKey = `${dataset.type}Server` as keyof typeof dataset; + const response = await axios.post<{ + access_token: string; + refresh_token: string; + expires_in: number; + }>( + 'https://open.feishu.cn/open-apis/authen/v2/oauth/token', + { + client_id: appId, + client_secret: appSecret, + grant_type: 'refresh_token', + refresh_token: dataset[serverKey]?.refresh_token + }, + { + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + ); + + if (!response?.data.access_token) { + addLog.error('Failed to refresh access token', { + datasetId: dataset._id + }); + return; + } + + // update dataset + await MongoDataset.findByIdAndUpdate(dataset._id, { + $set: { + [serverKey]: { + user_access_token: response.data.access_token, + refresh_token: response.data.refresh_token, + outdate_time: Date.now() + response.data.expires_in * 1000 + } + } + }); + + addLog.info('Feishu token refreshed successfully', { + datasetId: dataset._id + }); + } catch (error) { + addLog.error('Refresh Feishu token error', { + datasetId: dataset._id, + error: JSON.stringify(error) + }); + } + }); + + await Promise.all(refreshPromises); + } catch (error) { + addLog.error('Refresh Feishu tokens error', { error }); + } +} diff --git a/packages/service/core/dataset/feishuDataset/api.ts b/packages/service/core/dataset/apiDataset/feishuShareDataset/api.ts similarity index 79% rename from packages/service/core/dataset/feishuDataset/api.ts rename to packages/service/core/dataset/apiDataset/feishuShareDataset/api.ts index df0d58700aef..2d1188eb0622 100644 --- a/packages/service/core/dataset/feishuDataset/api.ts +++ b/packages/service/core/dataset/apiDataset/feishuShareDataset/api.ts @@ -2,11 +2,11 @@ import type { APIFileItem, ApiFileReadContentResponse, ApiDatasetDetailResponse, - FeishuServer + FeishuShareServer } from '@fastgpt/global/core/dataset/apiDataset'; import { type ParentIdType } from '@fastgpt/global/common/parentFolder/type'; import axios, { type Method } from 'axios'; -import { addLog } from '../../../common/system/log'; +import { addLog } from '../../../../common/system/log'; type ResponseDataType = { success: boolean; @@ -24,6 +24,10 @@ type FeishuFileListResponse = { created_time: number; url: string; owner_id: string; + shortcut_info?: { + target_token: string; + target_type: string; + }; }[]; has_more: boolean; next_page_token: string; @@ -31,7 +35,11 @@ type FeishuFileListResponse = { const feishuBaseUrl = process.env.FEISHU_BASE_URL || 'https://open.feishu.cn'; -export const useFeishuDatasetRequest = ({ feishuServer }: { feishuServer: FeishuServer }) => { +export const useFeishuShareDatasetRequest = ({ + feishuShareServer +}: { + feishuShareServer: FeishuShareServer; +}) => { const instance = axios.create({ baseURL: feishuBaseUrl, timeout: 60000 @@ -40,15 +48,7 @@ export const useFeishuDatasetRequest = ({ feishuServer }: { feishuServer: Feishu // 添加请求拦截器 instance.interceptors.request.use(async (config) => { if (!config.headers.Authorization) { - const { data } = await axios.post<{ tenant_access_token: string }>( - `${feishuBaseUrl}/open-apis/auth/v3/tenant_access_token/internal`, - { - app_id: feishuServer.appId, - app_secret: feishuServer.appSecret - } - ); - - config.headers['Authorization'] = `Bearer ${data.tenant_access_token}`; + config.headers['Authorization'] = `Bearer ${feishuShareServer.user_access_token}`; config.headers['Content-Type'] = 'application/json; charset=utf-8'; } return config; @@ -109,7 +109,7 @@ export const useFeishuDatasetRequest = ({ feishuServer }: { feishuServer: Feishu const data = await request( `/open-apis/drive/v1/files`, { - folder_token: parentId || feishuServer.folderToken, + folder_token: parentId || feishuShareServer.folderToken, page_size: 200, page_token: pageToken }, @@ -127,10 +127,18 @@ export const useFeishuDatasetRequest = ({ feishuServer }: { feishuServer: Feishu const allFiles = await fetchFiles(); return allFiles - .filter((file) => ['folder', 'docx'].includes(file.type)) + .filter((file) => { + if (file.type === 'shortcut') { + return ( + file.shortcut_info?.target_type === 'docx' || + file.shortcut_info?.target_type === 'folder' + ); + } + return file.type === 'folder' || file.type === 'docx'; + }) .map((file) => ({ - id: file.token, - parentId: file.parent_token, + id: file.type === 'shortcut' ? file.shortcut_info!.target_token : file.token, + parentId: parentId, name: file.name, type: file.type === 'folder' ? ('folder' as const) : ('file' as const), hasChild: file.type === 'folder', @@ -144,17 +152,10 @@ export const useFeishuDatasetRequest = ({ feishuServer }: { feishuServer: Feishu }: { apiFileId: string; }): Promise => { + const fileId = apiFileId.split('-')[1]; const [{ content }, { document }] = await Promise.all([ - request<{ content: string }>( - `/open-apis/docx/v1/documents/${apiFileId}/raw_content`, - {}, - 'GET' - ), - request<{ document: { title: string } }>( - `/open-apis/docx/v1/documents/${apiFileId}`, - {}, - 'GET' - ) + request<{ content: string }>(`/open-apis/docx/v1/documents/${fileId}/raw_content`, {}, 'GET'), + request<{ document: { title: string } }>(`/open-apis/docx/v1/documents/${fileId}`, {}, 'GET') ]); return { @@ -164,12 +165,13 @@ export const useFeishuDatasetRequest = ({ feishuServer }: { feishuServer: Feishu }; const getFilePreviewUrl = async ({ apiFileId }: { apiFileId: string }): Promise => { + const fileId = apiFileId.split('-')[1]; const { metas } = await request<{ metas: { url: string }[] }>( `/open-apis/drive/v1/metas/batch_query`, { request_docs: [ { - doc_token: apiFileId, + doc_token: fileId, doc_type: 'docx' } ], diff --git a/packages/service/core/dataset/apiDataset/index.ts b/packages/service/core/dataset/apiDataset/index.ts index 79af379c4490..c75b10aad066 100644 --- a/packages/service/core/dataset/apiDataset/index.ts +++ b/packages/service/core/dataset/apiDataset/index.ts @@ -1,18 +1,25 @@ import type { APIFileServer, YuqueServer, - FeishuServer + FeishuShareServer, + FeishuKnowledgeServer, + FeishuPrivateServer } from '@fastgpt/global/core/dataset/apiDataset'; import { useApiDatasetRequest } from './api'; -import { useYuqueDatasetRequest } from '../yuqueDataset/api'; -import { useFeishuDatasetRequest } from '../feishuDataset/api'; +import { useYuqueDatasetRequest } from './yuqueDataset/api'; +import { useFeishuShareDatasetRequest } from './feishuShareDataset/api'; +import { useFeishuKnowledgeDatasetRequest } from './feishuKnowledgeDataset/api'; +import { useFeishuPrivateDatasetRequest } from './feishuPrivateDataset/api'; export const getApiDatasetRequest = async (data: { apiServer?: APIFileServer; yuqueServer?: YuqueServer; - feishuServer?: FeishuServer; + feishuShareServer?: FeishuShareServer; + feishuKnowledgeServer?: FeishuKnowledgeServer; + feishuPrivateServer?: FeishuPrivateServer; }) => { - const { apiServer, yuqueServer, feishuServer } = data; + const { apiServer, yuqueServer, feishuShareServer, feishuKnowledgeServer, feishuPrivateServer } = + data; if (apiServer) { return useApiDatasetRequest({ apiServer }); @@ -20,8 +27,14 @@ export const getApiDatasetRequest = async (data: { if (yuqueServer) { return useYuqueDatasetRequest({ yuqueServer }); } - if (feishuServer) { - return useFeishuDatasetRequest({ feishuServer }); + if (feishuShareServer) { + return useFeishuShareDatasetRequest({ feishuShareServer }); + } + if (feishuKnowledgeServer) { + return useFeishuKnowledgeDatasetRequest({ feishuKnowledgeServer }); + } + if (feishuPrivateServer) { + return useFeishuPrivateDatasetRequest({ feishuPrivateServer }); } return Promise.reject('Can not find api dataset server'); }; diff --git a/packages/service/core/dataset/yuqueDataset/api.ts b/packages/service/core/dataset/apiDataset/yuqueDataset/api.ts similarity index 99% rename from packages/service/core/dataset/yuqueDataset/api.ts rename to packages/service/core/dataset/apiDataset/yuqueDataset/api.ts index cafbd9700d2e..6ffa50e327e9 100644 --- a/packages/service/core/dataset/yuqueDataset/api.ts +++ b/packages/service/core/dataset/apiDataset/yuqueDataset/api.ts @@ -5,7 +5,7 @@ import type { ApiDatasetDetailResponse } from '@fastgpt/global/core/dataset/apiDataset'; import axios, { type Method } from 'axios'; -import { addLog } from '../../../common/system/log'; +import { addLog } from '../../../../common/system/log'; import { type ParentIdType } from '@fastgpt/global/common/parentFolder/type'; type ResponseDataType = { diff --git a/packages/service/core/dataset/collection/utils.ts b/packages/service/core/dataset/collection/utils.ts index 96310ecb375e..47cfefa13775 100644 --- a/packages/service/core/dataset/collection/utils.ts +++ b/packages/service/core/dataset/collection/utils.ts @@ -160,7 +160,9 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => { type: DatasetSourceReadTypeEnum.apiFile, sourceId, apiServer: dataset.apiServer, - feishuServer: dataset.feishuServer, + feishuShareServer: dataset.feishuShareServer, + feishuKnowledgeServer: dataset.feishuKnowledgeServer, + feishuPrivateServer: dataset.feishuPrivateServer, yuqueServer: dataset.yuqueServer }; })(); diff --git a/packages/service/core/dataset/read.ts b/packages/service/core/dataset/read.ts index 647c057584b8..1d4b70a618a3 100644 --- a/packages/service/core/dataset/read.ts +++ b/packages/service/core/dataset/read.ts @@ -11,8 +11,10 @@ import { readRawContentByFileBuffer } from '../../common/file/read/utils'; import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools'; import { type APIFileServer, - type FeishuServer, - type YuqueServer + type FeishuShareServer, + type YuqueServer, + type FeishuKnowledgeServer, + type FeishuPrivateServer } from '@fastgpt/global/core/dataset/apiDataset'; import { getApiDatasetRequest } from './apiDataset'; import Papa from 'papaparse'; @@ -70,8 +72,10 @@ export const readDatasetSourceRawText = async ({ selector, externalFileId, apiServer, - feishuServer, + feishuShareServer, yuqueServer, + feishuKnowledgeServer, + feishuPrivateServer, customPdfParse, getFormatText }: { @@ -85,7 +89,9 @@ export const readDatasetSourceRawText = async ({ selector?: string; // link selector externalFileId?: string; // external file dataset apiServer?: APIFileServer; // api dataset - feishuServer?: FeishuServer; // feishu dataset + feishuShareServer?: FeishuShareServer; // feishu dataset + feishuKnowledgeServer?: FeishuKnowledgeServer; // feishu dataset + feishuPrivateServer?: FeishuPrivateServer; // feishu dataset yuqueServer?: YuqueServer; // yuque dataset }): Promise<{ title?: string; @@ -129,8 +135,10 @@ export const readDatasetSourceRawText = async ({ } else if (type === DatasetSourceReadTypeEnum.apiFile) { const { title, rawText } = await readApiServerFileContent({ apiServer, - feishuServer, + feishuShareServer, yuqueServer, + feishuKnowledgeServer, + feishuPrivateServer, apiFileId: sourceId, teamId, tmbId @@ -148,16 +156,20 @@ export const readDatasetSourceRawText = async ({ export const readApiServerFileContent = async ({ apiServer, - feishuServer, + feishuShareServer, yuqueServer, + feishuKnowledgeServer, + feishuPrivateServer, apiFileId, teamId, tmbId, customPdfParse }: { apiServer?: APIFileServer; - feishuServer?: FeishuServer; + feishuShareServer?: FeishuShareServer; yuqueServer?: YuqueServer; + feishuKnowledgeServer?: FeishuKnowledgeServer; + feishuPrivateServer?: FeishuPrivateServer; apiFileId: string; teamId: string; tmbId: string; @@ -170,7 +182,9 @@ export const readApiServerFileContent = async ({ await getApiDatasetRequest({ apiServer, yuqueServer, - feishuServer + feishuShareServer, + feishuKnowledgeServer, + feishuPrivateServer }) ).getFileContent({ teamId, diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 1573acb66f74..87bfabbfcc2a 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -128,8 +128,10 @@ const DatasetSchema = new Schema({ default: true }, apiServer: Object, - feishuServer: Object, + feishuShareServer: Object, yuqueServer: Object, + feishuKnowledgeServer: Object, + feishuPrivateServer: Object, // abandoned autoSync: Boolean, diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index 586420fe6a7e..df89b46fbba1 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -215,9 +215,18 @@ export const iconPaths = { import('./icons/core/dataset/externalDatasetColor.svg'), 'core/dataset/externalDatasetOutline': () => import('./icons/core/dataset/externalDatasetOutline.svg'), - 'core/dataset/feishuDatasetColor': () => import('./icons/core/dataset/feishuDatasetColor.svg'), - 'core/dataset/feishuDatasetOutline': () => - import('./icons/core/dataset/feishuDatasetOutline.svg'), + 'core/dataset/feishuShareDatasetColor': () => + import('./icons/core/dataset/feishuShareDatasetColor.svg'), + 'core/dataset/feishuKnowledgeDatasetColor': () => + import('./icons/core/dataset/feishuKnowledgeDatasetColor.svg'), + 'core/dataset/feishuPrivateDatasetColor': () => + import('./icons/core/dataset/feishuPrivateDatasetColor.svg'), + 'core/dataset/feishuPrivateDatasetOutline': () => + import('./icons/core/dataset/feishuPrivateDatasetOutline.svg'), + 'core/dataset/feishuKnowledgeDatasetOutline': () => + import('./icons/core/dataset/feishuKnowledgeDatasetOutline.svg'), + 'core/dataset/feishuShareDatasetOutline': () => + import('./icons/core/dataset/feishuShareDatasetOutline.svg'), 'core/dataset/fileCollection': () => import('./icons/core/dataset/fileCollection.svg'), 'core/dataset/fullTextRecall': () => import('./icons/core/dataset/fullTextRecall.svg'), 'core/dataset/manualCollection': () => import('./icons/core/dataset/manualCollection.svg'), diff --git a/packages/web/components/common/Icon/icons/core/dataset/feishuKnowledgeDatasetColor.svg b/packages/web/components/common/Icon/icons/core/dataset/feishuKnowledgeDatasetColor.svg new file mode 100644 index 000000000000..493b522f7790 --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/feishuKnowledgeDatasetColor.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/packages/web/components/common/Icon/icons/core/dataset/feishuKnowledgeDatasetOutline.svg b/packages/web/components/common/Icon/icons/core/dataset/feishuKnowledgeDatasetOutline.svg new file mode 100644 index 000000000000..ab86fd34a4bc --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/feishuKnowledgeDatasetOutline.svg @@ -0,0 +1,4 @@ + + + + diff --git a/packages/web/components/common/Icon/icons/core/dataset/feishuPrivateDatasetColor.svg b/packages/web/components/common/Icon/icons/core/dataset/feishuPrivateDatasetColor.svg new file mode 100644 index 000000000000..b38bd6228d7b --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/feishuPrivateDatasetColor.svg @@ -0,0 +1,4 @@ + + + + diff --git a/packages/web/components/common/Icon/icons/core/dataset/feishuPrivateDatasetOutline.svg b/packages/web/components/common/Icon/icons/core/dataset/feishuPrivateDatasetOutline.svg new file mode 100644 index 000000000000..70119e2246da --- /dev/null +++ b/packages/web/components/common/Icon/icons/core/dataset/feishuPrivateDatasetOutline.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/packages/web/components/common/Icon/icons/core/dataset/feishuDatasetColor.svg b/packages/web/components/common/Icon/icons/core/dataset/feishuShareDatasetColor.svg similarity index 100% rename from packages/web/components/common/Icon/icons/core/dataset/feishuDatasetColor.svg rename to packages/web/components/common/Icon/icons/core/dataset/feishuShareDatasetColor.svg diff --git a/packages/web/components/common/Icon/icons/core/dataset/feishuDatasetOutline.svg b/packages/web/components/common/Icon/icons/core/dataset/feishuShareDatasetOutline.svg similarity index 100% rename from packages/web/components/common/Icon/icons/core/dataset/feishuDatasetOutline.svg rename to packages/web/components/common/Icon/icons/core/dataset/feishuShareDatasetOutline.svg diff --git a/packages/web/i18n/en/dataset.json b/packages/web/i18n/en/dataset.json index 6a7b1e560adc..667e3d25c5cc 100644 --- a/packages/web/i18n/en/dataset.json +++ b/packages/web/i18n/en/dataset.json @@ -79,14 +79,24 @@ "external_url": "File Access URL", "failedToLoadRootDirectories": "Failed to load root directories", "failedToLoadSubDirectories": "Failed to load subdirectories", - "feishu_dataset": "Feishu Dataset", - "feishu_dataset_config": "Feishu Dataset Config", - "feishu_dataset_desc": "Can build a dataset using Feishu documents by configuring permissions, without secondary storage", + "feishu_auth_button": "Authorization", + "feishu_change_auth_button": "replace", + "feishu_knowledge_dataset": "Feishu Knowledge Base", + "feishu_knowledge_dataset_config": "Configure Feishu Knowledge Base", + "feishu_knowledge_dataset_desc": "You can use Feishu Knowledge Base to build a knowledge base, and the knowledge base documents will not be stored secondaryly.", + "feishu_private_dataset": "Feishu personal knowledge base", + "feishu_private_dataset_config": "Configure Feishu's personal knowledge base", + "feishu_private_dataset_desc": "You can use Feishu personal files to build a knowledge base, and the files will not be stored twice.", + "feishu_share_dataset": "Feishu Shared Knowledge Base", + "feishu_share_dataset_config": "Configure Feishu Shared Knowledge Base", + "feishu_share_dataset_desc": "You can use Feishu Shared Folder permissions to build a knowledge base, and the files will not be stored twice.", "file_list": "File list", "file_model_function_tip": "Enhances indexing and QA generation", "filename": "Filename", "folder_dataset": "Folder", "getDirectoryFailed": "Get directory failed", + "have_auth": "Authorized", + "have_not_auth": "Unauthorized", "image_auto_parse": "Automatic image indexing", "image_auto_parse_tips": "Call VLM to automatically label the pictures in the document and generate additional search indexes", "image_training_queue": "Queue of image processing", @@ -113,6 +123,7 @@ "noSelectedFolder": "No selected folder", "noSelectedId": "No selected ID", "noValidId": "No valid ID", + "not_set": "Not configured", "open_auto_sync": "After scheduled synchronization is turned on, the system will try to synchronize the collection from time to time every day. During the collection synchronization period, the collection data will not be searched.", "other_dataset": "Third-party knowledge base", "paragraph_max_deep": "Maximum paragraph depth", diff --git a/packages/web/i18n/zh-CN/dataset.json b/packages/web/i18n/zh-CN/dataset.json index 455343d17960..c477651b9ee4 100644 --- a/packages/web/i18n/zh-CN/dataset.json +++ b/packages/web/i18n/zh-CN/dataset.json @@ -79,14 +79,24 @@ "external_url": "文件访问 URL", "failedToLoadRootDirectories": "加载根目录失败", "failedToLoadSubDirectories": "加载子目录失败", - "feishu_dataset": "飞书知识库", - "feishu_dataset_config": "配置飞书知识库", - "feishu_dataset_desc": "可通过配置飞书文档权限，使用飞书文档构建知识库，文档不会进行二次存储", + "feishu_auth_button": "授权", + "feishu_change_auth_button": "更换", + "feishu_knowledge_dataset": "飞书知识库", + "feishu_knowledge_dataset_config": "配置飞书知识库", + "feishu_knowledge_dataset_desc": "使用飞书知识库构建知识库，知识库文档不会进行二次存储", + "feishu_private_dataset": "飞书个人知识库", + "feishu_private_dataset_config": "配置飞书个人知识库", + "feishu_private_dataset_desc": "使用飞书个人文件构建知识库，文件不会进行二次存储", + "feishu_share_dataset": "飞书共享知识库", + "feishu_share_dataset_config": "配置飞书共享知识库", + "feishu_share_dataset_desc": "使用飞书共享文件构建知识库，文件不会进行二次存储", "file_list": "文件列表", "file_model_function_tip": "用于增强索引和 QA 生成", "filename": "文件名", "folder_dataset": "文件夹", "getDirectoryFailed": "获取目录失败", + "have_auth": "已授权", + "have_not_auth": "未授权", "image_auto_parse": "图片自动索引", "image_auto_parse_tips": "调用 VLM 自动标注文档里的图片，并生成额外的检索索引", "image_training_queue": "图片处理排队", @@ -113,6 +123,7 @@ "noSelectedFolder": "没有选择文件夹", "noSelectedId": "没有选择 ID", "noValidId": "没有有效的 ID", + "not_set": "未配置", "open_auto_sync": "开启定时同步后，系统将会每天不定时尝试同步集合，集合同步期间，会出现无法搜索到该集合数据现象。", "other_dataset": "第三方知识库", "paragraph_max_deep": "最大段落深度", diff --git a/packages/web/i18n/zh-Hant/dataset.json b/packages/web/i18n/zh-Hant/dataset.json index 463d81a6e272..0b0d0f5e7210 100644 --- a/packages/web/i18n/zh-Hant/dataset.json +++ b/packages/web/i18n/zh-Hant/dataset.json @@ -77,14 +77,24 @@ "external_url": "檔案存取網址", "failedToLoadRootDirectories": "加載根目錄失敗", "failedToLoadSubDirectories": "加載子目錄失敗", - "feishu_dataset": "飛書知識庫", - "feishu_dataset_config": "設定飛書知識庫", - "feishu_dataset_desc": "可透過設定飛書文件權限，使用飛書文件建構知識庫，文件不會進行二次儲存", + "feishu_auth_button": "授權", + "feishu_change_auth_button": "更換", + "feishu_knowledge_dataset": "飛書知識庫", + "feishu_knowledge_dataset_config": "配置飛書知識庫", + "feishu_knowledge_dataset_desc": "使用飛書知識庫構建知識庫，知識庫文檔不會進行二次存儲", + "feishu_private_dataset": "飛書個人知識庫", + "feishu_private_dataset_config": "配置飛書個人知識庫", + "feishu_private_dataset_desc": "使用飛書個人文件構建知識庫，文件不會進行二次存儲", + "feishu_share_dataset": "飛書共享知識庫", + "feishu_share_dataset_config": "配置飛書共享知識庫", + "feishu_share_dataset_desc": "使用飛書共享文件構建知識庫，文件不會進行二次存儲", "file_list": "文件列表", "file_model_function_tip": "用於增強索引和問答生成", "filename": "檔案名稱", "folder_dataset": "資料夾", "getDirectoryFailed": "獲取目錄失敗", + "have_auth": "已授權", + "have_not_auth": "未授權", "image_auto_parse": "圖片自動索引", "image_auto_parse_tips": "呼叫 VLM 自動標註文件裡的圖片，並生成額外的檢索索引", "image_training_queue": "圖片處理排隊", @@ -111,6 +121,7 @@ "noSelectedFolder": "沒有選擇文件夾", "noSelectedId": "沒有選擇 ID", "noValidId": "沒有有效的 ID", + "not_set": "未配置", "open_auto_sync": "開啟定時同步後，系統將每天不定時嘗試同步集合，集合同步期間，會出現無法搜尋到該集合資料現象。", "other_dataset": "第三方知識庫", "paragraph_max_deep": "最大段落深度", diff --git a/projects/app/data/config.json b/projects/app/data/config.json index 1d0303370ce9..01cc514b3172 100644 --- a/projects/app/data/config.json +++ b/projects/app/data/config.json @@ -2,7 +2,9 @@ { "feConfigs": { "lafEnv": "https://laf.dev", // laf环境。 https://laf.run （杭州阿里云） ,或者私有化的laf环境。如果使用 Laf openapi 功能，需要最新版的 laf 。 - "mcpServerProxyEndpoint": "" // mcp server 代理地址，例如： http://localhost:3005 + "mcpServerProxyEndpoint": "", // mcp server 代理地址，例如： http://localhost:3005 + "feishu_auth_robot_client_id": "", // 飞书机器人客户端ID + "feishu_auth_robot_client_secret": "" // 飞书机器人客户端密钥 }, "systemEnv": { "vectorMaxProcess": 10, // 向量处理线程数量 diff --git a/projects/app/src/global/core/dataset/api.d.ts b/projects/app/src/global/core/dataset/api.d.ts index 7c9aa0dbb7a0..82e2910ccd2c 100644 --- a/projects/app/src/global/core/dataset/api.d.ts +++ b/projects/app/src/global/core/dataset/api.d.ts @@ -4,8 +4,10 @@ import type { } from '@fastgpt/global/core/dataset/api'; import type { APIFileServer, - FeishuServer, - YuqueServer + FeishuShareServer, + FeishuPrivateServer, + YuqueServer, + FeishuKnowledgeServer } from '@fastgpt/global/core/dataset/apiDataset'; import type { DatasetSearchModeEnum, @@ -32,8 +34,10 @@ export type CreateDatasetParams = { agentModel?: string; vlmModel?: string; apiServer?: APIFileServer; - feishuServer?: FeishuServer; + feishuShareServer?: FeishuShareServer; yuqueServer?: YuqueServer; + feishuKnowledgeServer?: FeishuKnowledgeServer; + feishuPrivateServer?: FeishuPrivateServer; }; export type RebuildEmbeddingProps = { diff --git a/projects/app/src/pageComponents/dataset/ApiDatasetForm.tsx b/projects/app/src/pageComponents/dataset/ApiDatasetForm.tsx index 3588b474232a..78c94253b97c 100644 --- a/projects/app/src/pageComponents/dataset/ApiDatasetForm.tsx +++ b/projects/app/src/pageComponents/dataset/ApiDatasetForm.tsx @@ -1,11 +1,13 @@ -import React, { useState, useMemo } from 'react'; +import React, { useState, useMemo, useEffect } from 'react'; import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { Flex, Input, Button, ModalBody, ModalFooter, Box } from '@chakra-ui/react'; import type { UseFormReturn } from 'react-hook-form'; import { useTranslation } from 'next-i18next'; import type { APIFileServer, - FeishuServer, + FeishuShareServer, + FeishuKnowledgeServer, + FeishuPrivateServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset'; import { getApiDatasetPaths, getApiDatasetCatalog } from '@/web/core/dataset/api'; @@ -18,6 +20,7 @@ import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import type { GetApiDatasetCataLogProps } from '@/pages/api/core/dataset/apiDataset/getCatalog'; import MyBox from '@fastgpt/web/components/common/MyBox'; import { useBoolean, useMemoizedFn, useMount } from 'ahooks'; +import { useSystemStore } from '@/web/common/system/useSystemStore'; import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel'; import MyModal from '@fastgpt/web/components/common/MyModal'; import MyIcon from '@fastgpt/web/components/common/Icon'; @@ -33,7 +36,9 @@ const ApiDatasetForm = ({ form: UseFormReturn< { apiServer?: APIFileServer; - feishuServer?: FeishuServer; + feishuShareServer?: FeishuShareServer; + feishuKnowledgeServer?: FeishuKnowledgeServer; + feishuPrivateServer?: FeishuPrivateServer; yuqueServer?: YuqueServer; }, any @@ -43,8 +48,10 @@ const ApiDatasetForm = ({ const { register, setValue, watch } = form; const yuqueServer = watch('yuqueServer'); - const feishuServer = watch('feishuServer'); + const feishuShareServer = watch('feishuShareServer'); const apiServer = watch('apiServer'); + const feishuKnowledgeServer = watch('feishuKnowledgeServer'); + const feishuPrivateServer = watch('feishuPrivateServer'); const [pathNames, setPathNames] = useState(t('dataset:rootdirectory')); const [ @@ -52,16 +59,85 @@ const ApiDatasetForm = ({ { setTrue: openBaseurlSeletModal, setFalse: closeBaseurlSelectModal } ] = useBoolean(); - const parentId = yuqueServer?.basePath || apiServer?.basePath; + const { feConfigs } = useSystemStore(); + + const appid = feConfigs?.feishu_auth_robot_client_id; + const urlParams = new URLSearchParams(window.location.search); + + const parentId = + yuqueServer?.basePath || + apiServer?.basePath || + feishuKnowledgeServer?.basePath || + feishuPrivateServer?.basePath; + + const renderFeishuAuth = ( + server: FeishuShareServer | FeishuKnowledgeServer | FeishuPrivateServer | undefined + ) => { + if (urlParams.get('datasetId')) { + return ( + <> + + + Feishu Auth + + + {!server?.user_access_token ? t('dataset:have_not_auth') : t('dataset:have_auth')} + + + + {type === DatasetTypeEnum.feishuShare ? ( + + + Folder Token + + + + ) : ( + <> + {renderBaseUrlSelector()} + {renderDirectoryModal()} + + )} + + ); + } + return <>; + }; const canSelectBaseUrl = useMemo(() => { switch (type) { case DatasetTypeEnum.yuque: return yuqueServer?.userId && yuqueServer?.token; - case DatasetTypeEnum.feishu: - return feishuServer?.appId && feishuServer?.appSecret; + case DatasetTypeEnum.feishuShare: + return feishuShareServer?.user_access_token; + case DatasetTypeEnum.feishuKnowledge: + return feishuKnowledgeServer?.user_access_token; case DatasetTypeEnum.apiDataset: return !!apiServer?.baseUrl; + case DatasetTypeEnum.feishuPrivate: + return feishuPrivateServer?.user_access_token; default: return false; } @@ -69,9 +145,10 @@ const ApiDatasetForm = ({ type, yuqueServer?.userId, yuqueServer?.token, - feishuServer?.appId, - feishuServer?.appSecret, - apiServer?.baseUrl + feishuShareServer?.user_access_token, + feishuKnowledgeServer?.user_access_token, + apiServer?.baseUrl, + feishuPrivateServer?.user_access_token ]); // Unified function to get the current path @@ -80,7 +157,9 @@ const ApiDatasetForm = ({ if ( !datasetId && ((yuqueServer && (!yuqueServer.userId || !yuqueServer?.token)) || - (apiServer && !apiServer?.baseUrl)) + (apiServer && !apiServer?.baseUrl) || + (feishuKnowledgeServer && !feishuKnowledgeServer?.user_access_token) || + (feishuPrivateServer && !feishuPrivateServer?.user_access_token)) ) { return setPathNames(t('dataset:input_required_field_to_select_baseurl')); } @@ -92,8 +171,10 @@ const ApiDatasetForm = ({ datasetId, parentId, yuqueServer, - feishuServer, - apiServer + feishuShareServer, + apiServer, + feishuKnowledgeServer, + feishuPrivateServer }); setPathNames(path); }, @@ -110,12 +191,18 @@ const ApiDatasetForm = ({ case DatasetTypeEnum.yuque: setValue('yuqueServer.basePath', value); break; - case DatasetTypeEnum.feishu: - setValue('feishuServer.folderToken', value); + case DatasetTypeEnum.feishuShare: + setValue('feishuShareServer.folderToken', value); break; case DatasetTypeEnum.apiDataset: setValue('apiServer.basePath', value); break; + case DatasetTypeEnum.feishuKnowledge: + setValue('feishuKnowledgeServer.basePath', value); + break; + case DatasetTypeEnum.feishuPrivate: + setValue('feishuPrivateServer.basePath', value); + break; } closeBaseurlSelectModal(); @@ -157,12 +244,28 @@ const ApiDatasetForm = ({ basePath: '' }; break; - // Currently, only Yuque is using it - case DatasetTypeEnum.feishu: - params.feishuServer = { - appId: feishuServer?.appId || '', - appSecret: feishuServer?.appSecret || '', - folderToken: feishuServer?.folderToken || '' + case DatasetTypeEnum.feishuShare: + params.feishuShareServer = { + user_access_token: feishuShareServer?.user_access_token || '', + refresh_token: feishuShareServer?.refresh_token || '', + outdate_time: feishuShareServer?.outdate_time || 0, + folderToken: feishuShareServer?.folderToken || '' + }; + break; + case DatasetTypeEnum.feishuKnowledge: + params.feishuKnowledgeServer = { + user_access_token: feishuKnowledgeServer?.user_access_token || '', + refresh_token: feishuKnowledgeServer?.refresh_token || '', + outdate_time: feishuKnowledgeServer?.outdate_time || 0, + basePath: '' + }; + break; + case DatasetTypeEnum.feishuPrivate: + params.feishuPrivateServer = { + user_access_token: feishuPrivateServer?.user_access_token || '', + refresh_token: feishuPrivateServer?.refresh_token || '', + outdate_time: feishuPrivateServer?.outdate_time || 0, + basePath: '' }; break; case DatasetTypeEnum.apiDataset: @@ -211,63 +314,9 @@ const ApiDatasetForm = ({ {renderDirectoryModal()} )} - {type === DatasetTypeEnum.feishu && ( - <> - - - App ID - - - - - - App Secret - - - - - - Folder Token - - - - {/* {renderBaseUrlSelector()} - {renderDirectoryModal()} */} - - )} + {type === DatasetTypeEnum.feishuShare && <>{renderFeishuAuth(feishuShareServer)}} + {type === DatasetTypeEnum.feishuKnowledge && <>{renderFeishuAuth(feishuKnowledgeServer)}} + {type === DatasetTypeEnum.feishuPrivate && <>{renderFeishuAuth(feishuPrivateServer)}} {type === DatasetTypeEnum.yuque && ( <> diff --git a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx index 6f5c857e0089..c4da516b52cd 100644 --- a/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx +++ b/projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx @@ -416,7 +416,9 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => { )} {/* apiDataset */} {(datasetDetail?.type === DatasetTypeEnum.apiDataset || - datasetDetail?.type === DatasetTypeEnum.feishu || + datasetDetail?.type === DatasetTypeEnum.feishuShare || + datasetDetail?.type === DatasetTypeEnum.feishuKnowledge || + datasetDetail?.type === DatasetTypeEnum.feishuPrivate || datasetDetail?.type === DatasetTypeEnum.yuque) && ( { reset(datasetDetail); }, [datasetDetail, datasetDetail._id, reset]); + useEffect(() => { + if (!datasetId) return; + + const serverConfig: any = { id: datasetDetail._id }; + + const needAuth = + (datasetDetail.type === DatasetTypeEnum.feishuPrivate && + !datasetDetail.feishuPrivateServer?.user_access_token) || + (datasetDetail.type === DatasetTypeEnum.feishuShare && + !datasetDetail.feishuShareServer?.user_access_token) || + (datasetDetail.type === DatasetTypeEnum.feishuKnowledge && + !datasetDetail.feishuKnowledgeServer?.user_access_token); + + let shouldSetDataset = false; + + if (needAuth) { + serverConfig.feishuPrivateServer = datasetDetail.feishuPrivateServer; + serverConfig.feishuShareServer = datasetDetail.feishuShareServer; + serverConfig.feishuKnowledgeServer = datasetDetail.feishuKnowledgeServer; + shouldSetDataset = true; + } + + if (typeof window !== 'undefined') { + const urlParams = new URLSearchParams(window.location.search); + const openConfigModal = urlParams.get('openConfigModal'); + if ( + openConfigModal === 'true' && + (datasetDetail.type === DatasetTypeEnum.feishuPrivate || + datasetDetail.type === DatasetTypeEnum.feishuShare || + datasetDetail.type === DatasetTypeEnum.feishuKnowledge) + ) { + switch (datasetDetail.type) { + case DatasetTypeEnum.feishuPrivate: + serverConfig.feishuPrivateServer = datasetDetail.feishuPrivateServer; + break; + case DatasetTypeEnum.feishuShare: + serverConfig.feishuShareServer = datasetDetail.feishuShareServer; + break; + case DatasetTypeEnum.feishuKnowledge: + serverConfig.feishuKnowledgeServer = datasetDetail.feishuKnowledgeServer; + break; + default: + break; + } + shouldSetDataset = true; + + const url = new URL(window.location.href); + url.searchParams.delete('openConfigModal'); + window.history.replaceState({}, document.title, url.toString()); + } + } + + if (shouldSetDataset) { + setEditedAPIDataset(serverConfig); + } + }, [ + datasetId, + datasetDetail._id, + datasetDetail.type, + datasetDetail.feishuPrivateServer, + datasetDetail.feishuPrivateServer?.user_access_token, + datasetDetail.feishuShareServer, + datasetDetail.feishuShareServer?.user_access_token, + datasetDetail.feishuKnowledgeServer, + datasetDetail.feishuKnowledgeServer?.user_access_token + ]); + const isTraining = rebuildingCount > 0 || trainingCount > 0; return ( @@ -345,13 +412,66 @@ const Info = ({ datasetId }: { datasetId: string }) => { )} + {datasetDetail.type === DatasetTypeEnum.feishuKnowledge && ( + <> + + + + {t('dataset:feishu_knowledge_dataset_config')} + + + setEditedAPIDataset({ + id: datasetDetail._id, + feishuKnowledgeServer: datasetDetail.feishuKnowledgeServer + }) + } + /> + + + {datasetDetail.feishuKnowledgeServer?.basePath || t('common:root_folder')} + + + + )} + + {datasetDetail.type === DatasetTypeEnum.feishuPrivate && ( + <> + + + + {t('dataset:feishu_private_dataset_config')} + + + setEditedAPIDataset({ + id: datasetDetail._id, + feishuPrivateServer: datasetDetail.feishuPrivateServer + }) + } + /> + + + {datasetDetail.feishuPrivateServer?.basePath || t('common:root_folder')} + + + + )} - {datasetDetail.type === DatasetTypeEnum.feishu && ( + {datasetDetail.type === DatasetTypeEnum.feishuShare && ( <> - {t('dataset:feishu_dataset_config')} + {t('dataset:feishu_share_dataset_config')} { onClick={() => setEditedAPIDataset({ id: datasetDetail._id, - feishuServer: datasetDetail.feishuServer + feishuShareServer: datasetDetail.feishuShareServer }) } /> - {datasetDetail.feishuServer?.folderToken} + + {datasetDetail.feishuShareServer?.folderToken || t('dataset:not_set')} + )} + + {datasetDetail.type === DatasetTypeEnum.feishuPrivate && <>} {datasetDetail.permission.hasManagePer && ( @@ -437,7 +561,9 @@ const Info = ({ datasetId }: { datasetId: string }) => { id: datasetId, apiServer: data.apiServer, yuqueServer: data.yuqueServer, - feishuServer: data.feishuServer + feishuShareServer: data.feishuShareServer, + feishuKnowledgeServer: data.feishuKnowledgeServer, + feishuPrivateServer: data.feishuPrivateServer }) } /> diff --git a/projects/app/src/pageComponents/dataset/list/CreateModal.tsx b/projects/app/src/pageComponents/dataset/list/CreateModal.tsx index 38e60863524d..eb4b1d148817 100644 --- a/projects/app/src/pageComponents/dataset/list/CreateModal.tsx +++ b/projects/app/src/pageComponents/dataset/list/CreateModal.tsx @@ -26,7 +26,9 @@ export type CreateDatasetType = | DatasetTypeEnum.dataset | DatasetTypeEnum.apiDataset | DatasetTypeEnum.websiteDataset - | DatasetTypeEnum.feishu + | DatasetTypeEnum.feishuShare + | DatasetTypeEnum.feishuKnowledge + | DatasetTypeEnum.feishuPrivate | DatasetTypeEnum.yuque; const CreateModal = ({ @@ -57,9 +59,17 @@ const CreateModal = ({ name: t('dataset:api_file'), icon: 'core/dataset/externalDatasetColor' }, - [DatasetTypeEnum.feishu]: { - name: t('dataset:feishu_dataset'), - icon: 'core/dataset/feishuDatasetColor' + [DatasetTypeEnum.feishuShare]: { + name: t('dataset:feishu_share_dataset'), + icon: 'core/dataset/feishuShareDatasetColor' + }, + [DatasetTypeEnum.feishuKnowledge]: { + name: t('dataset:feishu_knowledge_dataset'), + icon: 'core/dataset/feishuKnowledgeDatasetColor' + }, + [DatasetTypeEnum.feishuPrivate]: { + name: t('dataset:feishu_private_dataset'), + icon: 'core/dataset/feishuPrivateDatasetColor' }, [DatasetTypeEnum.yuque]: { name: t('dataset:yuque_dataset'), diff --git a/projects/app/src/pageComponents/dataset/list/SideTag.tsx b/projects/app/src/pageComponents/dataset/list/SideTag.tsx index 0cb84048a9e4..645147dfdd2a 100644 --- a/projects/app/src/pageComponents/dataset/list/SideTag.tsx +++ b/projects/app/src/pageComponents/dataset/list/SideTag.tsx @@ -25,9 +25,17 @@ const SideTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & FlexProps) icon: 'core/dataset/externalDatasetOutline', label: t('dataset:api_file') }, - [DatasetTypeEnum.feishu]: { - icon: 'core/dataset/feishuDatasetOutline', - label: t('dataset:feishu_dataset') + [DatasetTypeEnum.feishuShare]: { + icon: 'core/dataset/feishuShareDatasetOutline', + label: t('dataset:feishu_share_dataset') + }, + [DatasetTypeEnum.feishuPrivate]: { + icon: 'core/dataset/feishuPrivateDatasetOutline', + label: t('dataset:feishu_private_dataset') + }, + [DatasetTypeEnum.feishuKnowledge]: { + icon: 'core/dataset/feishuKnowledgeDatasetOutline', + label: t('dataset:feishu_knowledge_dataset') }, [DatasetTypeEnum.yuque]: { icon: 'core/dataset/yuqueDatasetOutline', diff --git a/projects/app/src/pages/api/core/dataset/apiDataset/getCatalog.ts b/projects/app/src/pages/api/core/dataset/apiDataset/getCatalog.ts index 906e465b5ede..2ca0606869d9 100644 --- a/projects/app/src/pages/api/core/dataset/apiDataset/getCatalog.ts +++ b/projects/app/src/pages/api/core/dataset/apiDataset/getCatalog.ts @@ -5,7 +5,9 @@ import type { APIFileItem, APIFileServer, YuqueServer, - FeishuServer + FeishuShareServer, + FeishuKnowledgeServer, + FeishuPrivateServer } from '@fastgpt/global/core/dataset/apiDataset'; import { type NextApiRequest } from 'next'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; @@ -13,22 +15,34 @@ import { authCert } from '@fastgpt/service/support/permission/auth/common'; export type GetApiDatasetCataLogProps = { parentId?: ParentIdType; yuqueServer?: YuqueServer; - feishuServer?: FeishuServer; + feishuShareServer?: FeishuShareServer; + feishuKnowledgeServer?: FeishuKnowledgeServer; + feishuPrivateServer?: FeishuPrivateServer; apiServer?: APIFileServer; }; export type GetApiDatasetCataLogResponse = APIFileItem[]; async function handler(req: NextApiRequest) { - let { searchKey = '', parentId = null, yuqueServer, feishuServer, apiServer } = req.body; + let { + searchKey = '', + parentId = null, + yuqueServer, + feishuShareServer, + apiServer, + feishuKnowledgeServer, + feishuPrivateServer + } = req.body; await authCert({ req, authToken: true }); const data = await ( await getApiDatasetRequest({ - feishuServer, + feishuShareServer, yuqueServer, - apiServer + apiServer, + feishuKnowledgeServer, + feishuPrivateServer }) ).listFiles({ parentId, searchKey }); diff --git a/projects/app/src/pages/api/core/dataset/apiDataset/getPathNames.ts b/projects/app/src/pages/api/core/dataset/apiDataset/getPathNames.ts index f069ea47b006..138e5786056c 100644 --- a/projects/app/src/pages/api/core/dataset/apiDataset/getPathNames.ts +++ b/projects/app/src/pages/api/core/dataset/apiDataset/getPathNames.ts @@ -4,8 +4,10 @@ import type { ParentIdType } from '@fastgpt/global/common/parentFolder/type'; import type { APIFileServer, YuqueServer, - FeishuServer, - ApiDatasetDetailResponse + FeishuShareServer, + ApiDatasetDetailResponse, + FeishuKnowledgeServer, + FeishuPrivateServer } from '@fastgpt/global/core/dataset/apiDataset'; import { getApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset'; import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; @@ -19,8 +21,10 @@ export type GetApiDatasetPathBody = { datasetId?: string; parentId?: ParentIdType; yuqueServer?: YuqueServer; - feishuServer?: FeishuServer; + feishuShareServer?: FeishuShareServer; apiServer?: APIFileServer; + feishuKnowledgeServer?: FeishuKnowledgeServer; + feishuPrivateServer?: FeishuPrivateServer; }; export type GetApiDatasetPathResponse = string; @@ -50,49 +54,69 @@ async function handler( const { datasetId, parentId } = req.body; if (!parentId) return ''; - const { yuqueServer, feishuServer, apiServer } = await (async () => { - if (datasetId) { - const { dataset } = await authDataset({ - req, - authToken: true, - authApiKey: true, - per: ManagePermissionVal, - datasetId - }); - - return { - yuqueServer: req.body.yuqueServer - ? { ...req.body.yuqueServer, token: dataset.yuqueServer?.token ?? '' } - : dataset.yuqueServer, - feishuServer: req.body.feishuServer - ? { ...req.body.feishuServer, appSecret: dataset.feishuServer?.appSecret ?? '' } - : dataset.feishuServer, - apiServer: req.body.apiServer - ? { - ...req.body.apiServer, - authorization: dataset.apiServer?.authorization ?? '' - } - : dataset.apiServer - }; - } else { - await authCert({ req, authToken: true }); - - return { - yuqueServer: req.body.yuqueServer, - feishuServer: req.body.feishuServer, - apiServer: req.body.apiServer - }; - } - })(); - - if (feishuServer) { + const { yuqueServer, feishuShareServer, apiServer, feishuKnowledgeServer, feishuPrivateServer } = + await (async () => { + if (datasetId) { + const { dataset } = await authDataset({ + req, + authToken: true, + authApiKey: true, + per: ManagePermissionVal, + datasetId + }); + + return { + yuqueServer: req.body.yuqueServer + ? { ...req.body.yuqueServer, token: dataset.yuqueServer?.token ?? '' } + : dataset.yuqueServer, + feishuShareServer: req.body.feishuShareServer + ? { + ...req.body.feishuShareServer, + user_access_token: dataset.feishuShareServer?.user_access_token ?? '' + } + : dataset.feishuShareServer, + apiServer: req.body.apiServer + ? { + ...req.body.apiServer, + authorization: dataset.apiServer?.authorization ?? '' + } + : dataset.apiServer, + feishuKnowledgeServer: req.body.feishuKnowledgeServer + ? { + ...req.body.feishuKnowledgeServer, + user_access_token: dataset.feishuKnowledgeServer?.user_access_token ?? '' + } + : dataset.feishuKnowledgeServer, + feishuPrivateServer: req.body.feishuPrivateServer + ? { + ...req.body.feishuPrivateServer, + user_access_token: dataset.feishuPrivateServer?.user_access_token ?? '' + } + : dataset.feishuPrivateServer + }; + } else { + await authCert({ req, authToken: true }); + + return { + yuqueServer: req.body.yuqueServer, + feishuShareServer: req.body.feishuShareServer, + apiServer: req.body.apiServer, + feishuKnowledgeServer: req.body.feishuKnowledgeServer, + feishuPrivateServer: req.body.feishuPrivateServer + }; + } + })(); + + if (feishuShareServer) { return ''; } - if (yuqueServer || apiServer) { + if (yuqueServer || apiServer || feishuKnowledgeServer || feishuPrivateServer) { const apiDataset = await getApiDatasetRequest({ yuqueServer, - apiServer + apiServer, + feishuKnowledgeServer, + feishuPrivateServer }); if (!apiDataset?.getFileDetail) { diff --git a/projects/app/src/pages/api/core/dataset/apiDataset/list.ts b/projects/app/src/pages/api/core/dataset/apiDataset/list.ts index c83aed5f28bb..ba87de55248b 100644 --- a/projects/app/src/pages/api/core/dataset/apiDataset/list.ts +++ b/projects/app/src/pages/api/core/dataset/apiDataset/list.ts @@ -24,14 +24,18 @@ async function handler(req: NextApiRequest) { }); const apiServer = dataset.apiServer; - const feishuServer = dataset.feishuServer; + const feishuShareServer = dataset.feishuShareServer; const yuqueServer = dataset.yuqueServer; + const feishuKnowledgeServer = dataset.feishuKnowledgeServer; + const feishuPrivateServer = dataset.feishuPrivateServer; return ( await getApiDatasetRequest({ apiServer, yuqueServer, - feishuServer + feishuShareServer, + feishuKnowledgeServer, + feishuPrivateServer }) ).listFiles({ searchKey, parentId }); } diff --git a/projects/app/src/pages/api/core/dataset/collection/create/apiCollection.ts b/projects/app/src/pages/api/core/dataset/collection/create/apiCollection.ts index 9bbebda2898f..327dbaf72b9b 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/apiCollection.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/apiCollection.ts @@ -24,8 +24,10 @@ async function handler(req: NextApiRequest): CreateCollectionResponse { }); const apiServer = dataset.apiServer; - const feishuServer = dataset.feishuServer; + const feishuShareServer = dataset.feishuShareServer; const yuqueServer = dataset.yuqueServer; + const feishuKnowledgeServer = dataset.feishuKnowledgeServer; + const feishuPrivateServer = dataset.feishuPrivateServer; // Auth same apiFileId const storeCol = await MongoDatasetCollection.findOne( @@ -43,8 +45,10 @@ async function handler(req: NextApiRequest): CreateCollectionResponse { const { title, rawText } = await readApiServerFileContent({ apiServer, - feishuServer, + feishuShareServer, + feishuKnowledgeServer, yuqueServer, + feishuPrivateServer, apiFileId, teamId, tmbId, diff --git a/projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts b/projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts index aa1fc3db062e..968e1b0c1d41 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts @@ -60,7 +60,9 @@ async function handler( type: DatasetSourceReadTypeEnum.apiFile, sourceId: collection.apiFileId, apiServer: collection.dataset.apiServer, - feishuServer: collection.dataset.feishuServer, + feishuShareServer: collection.dataset.feishuShareServer, + feishuKnowledgeServer: collection.dataset.feishuKnowledgeServer, + feishuPrivateServer: collection.dataset.feishuPrivateServer, yuqueServer: collection.dataset.yuqueServer }; } diff --git a/projects/app/src/pages/api/core/dataset/collection/read.ts b/projects/app/src/pages/api/core/dataset/collection/read.ts index f5d91eec1a28..4989b4228c17 100644 --- a/projects/app/src/pages/api/core/dataset/collection/read.ts +++ b/projects/app/src/pages/api/core/dataset/collection/read.ts @@ -95,14 +95,18 @@ async function handler( } if (collection.type === DatasetCollectionTypeEnum.apiFile && collection.apiFileId) { const apiServer = collection.dataset.apiServer; - const feishuServer = collection.dataset.feishuServer; + const feishuShareServer = collection.dataset.feishuShareServer; const yuqueServer = collection.dataset.yuqueServer; + const feishuKnowledgeServer = collection.dataset.feishuKnowledgeServer; + const feishuPrivateServer = collection.dataset.feishuPrivateServer; return ( await getApiDatasetRequest({ apiServer, - feishuServer, - yuqueServer + feishuShareServer, + feishuKnowledgeServer, + yuqueServer, + feishuPrivateServer }) ).getFilePreviewUrl({ apiFileId: collection.apiFileId diff --git a/projects/app/src/pages/api/core/dataset/create.ts b/projects/app/src/pages/api/core/dataset/create.ts index 2a57678207e9..0786910815a7 100644 --- a/projects/app/src/pages/api/core/dataset/create.ts +++ b/projects/app/src/pages/api/core/dataset/create.ts @@ -36,8 +36,10 @@ async function handler( agentModel = getDatasetModel()?.model, vlmModel, apiServer, - feishuServer, - yuqueServer + feishuShareServer, + yuqueServer, + feishuKnowledgeServer, + feishuPrivateServer } = req.body; // auth @@ -84,7 +86,9 @@ async function handler( avatar, type, apiServer, - feishuServer, + feishuShareServer, + feishuKnowledgeServer, + feishuPrivateServer, yuqueServer } ], diff --git a/projects/app/src/pages/api/core/dataset/detail.ts b/projects/app/src/pages/api/core/dataset/detail.ts index f6bef3c34ef3..84131a22c853 100644 --- a/projects/app/src/pages/api/core/dataset/detail.ts +++ b/projects/app/src/pages/api/core/dataset/detail.ts @@ -59,11 +59,28 @@ async function handler(req: ApiRequestProps): Promise { basePath: dataset.yuqueServer.basePath } : undefined, - feishuServer: dataset.feishuServer + feishuShareServer: dataset.feishuShareServer ? { - appId: dataset.feishuServer.appId, - appSecret: '', - folderToken: dataset.feishuServer.folderToken + user_access_token: dataset.feishuShareServer.user_access_token, + refresh_token: dataset.feishuShareServer.refresh_token, + outdate_time: dataset.feishuShareServer.outdate_time, + folderToken: dataset.feishuShareServer.folderToken + } + : undefined, + feishuKnowledgeServer: dataset.feishuKnowledgeServer + ? { + user_access_token: dataset.feishuKnowledgeServer.user_access_token, + refresh_token: dataset.feishuKnowledgeServer.refresh_token, + outdate_time: dataset.feishuKnowledgeServer.outdate_time, + basePath: dataset.feishuKnowledgeServer.basePath + } + : undefined, + feishuPrivateServer: dataset.feishuPrivateServer + ? { + user_access_token: dataset.feishuPrivateServer.user_access_token, + refresh_token: dataset.feishuPrivateServer.refresh_token, + outdate_time: dataset.feishuPrivateServer.outdate_time, + basePath: dataset.feishuPrivateServer.basePath } : undefined, permission, diff --git a/projects/app/src/pages/api/core/dataset/feishu/getUserAccessToken.ts b/projects/app/src/pages/api/core/dataset/feishu/getUserAccessToken.ts new file mode 100644 index 000000000000..75ddd15cfe53 --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/feishu/getUserAccessToken.ts @@ -0,0 +1,70 @@ +import axios from 'axios'; +import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; + +const appId = global.feConfigs.feishu_auth_robot_client_id; +const appSecret = global.feConfigs.feishu_auth_robot_client_secret; + +export async function getUserAccessToken(code: string, datasetId: string) { + if (!appId || !appSecret) { + throw new Error('FEISHU_APP_ID or FEISHU_APP_SECRET is not set'); + } + if (!code || !datasetId) { + throw new Error('code or datasetId is not set'); + } + const dataset = await MongoDataset.findById(datasetId); + if (!dataset) { + throw new Error('Dataset not found'); + } + + // 检查数据集类型 + if ( + dataset.type !== DatasetTypeEnum.feishuPrivate && + dataset.type !== DatasetTypeEnum.feishuShare && + dataset.type !== DatasetTypeEnum.feishuKnowledge + ) { + throw new Error('Dataset type is not feishu]'); + } + + try { + const response = await axios.post<{ + access_token: string; + refresh_token: string; + expires_in: number; + }>( + 'https://open.feishu.cn/open-apis/authen/v2/oauth/token', + { + client_id: appId, + client_secret: appSecret, + grant_type: 'authorization_code', + code, + redirect_uri: 'http://localhost:3000/api/core/dataset/feishu/oauth', + code_verifier: 'TxYmzM4PHLBlqm5NtnCmwxMH8mFlRWl_ipie3O0aVzo' + }, + { + headers: { + 'Content-Type': 'application/json' + } + } + ); + + if (!response?.data.access_token) { + throw new Error('Failed to get access token'); + } + + await MongoDataset.findByIdAndUpdate(datasetId, { + $set: { + [`${dataset.type}Server`]: { + user_access_token: response.data.access_token, + refresh_token: response.data.refresh_token, + outdate_time: Date.now() + response.data.expires_in * 1000 + } + } + }); + + return response.data.access_token; + } catch (error) { + console.error('Get Feishu Access Token Error:', error); + throw error; + } +} diff --git a/projects/app/src/pages/api/core/dataset/feishu/oauth.ts b/projects/app/src/pages/api/core/dataset/feishu/oauth.ts new file mode 100644 index 000000000000..0fdf3d3a2fdd --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/feishu/oauth.ts @@ -0,0 +1,44 @@ +import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next'; +import { getUserAccessToken } from './getUserAccessToken'; + +export type FeishuOauthQuery = { + code: string; + state: string; +}; + +export type FeishuOauthResponse = { + error?: string; +}; + +export default async function handler( + req: ApiRequestProps, + res: ApiResponseType +) { + try { + const { code, state } = req.query; + const stateData = JSON.parse(decodeURIComponent(state as string)); + const { returnUrl, datasetId } = stateData; + + if (!code || !state) { + return res.status(400).json({ error: 'Missing code or state' }); + } + + if (!datasetId) { + return res.status(400).json({ error: 'Missing datasetId in returnUrl' }); + } + + // get user_access_token + const user_access_token = await getUserAccessToken(code as string, datasetId); + + // redirect to original page + const redirectUrl = new URL(returnUrl, 'http://localhost:3000'); + + // add openConfigModal to redirectUrl + redirectUrl.searchParams.append('openConfigModal', 'true'); + + res.redirect(redirectUrl.toString()); + } catch (error) { + console.error('Feishu OAuth Error:', error); + res.status(500).json({ error: 'Internal Server Error' }); + } +} diff --git a/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts b/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts index b0e58238b184..17012a951fb6 100644 --- a/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts +++ b/projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts @@ -122,8 +122,10 @@ async function handler( sourceId, selector, apiServer: dataset.apiServer, - feishuServer: dataset.feishuServer, + feishuShareServer: dataset.feishuShareServer, yuqueServer: dataset.yuqueServer, + feishuKnowledgeServer: dataset.feishuKnowledgeServer, + feishuPrivateServer: dataset.feishuPrivateServer, externalFileId, customPdfParse }); diff --git a/projects/app/src/pages/api/core/dataset/update.ts b/projects/app/src/pages/api/core/dataset/update.ts index c19bbc764954..b9ca6c49d0c2 100644 --- a/projects/app/src/pages/api/core/dataset/update.ts +++ b/projects/app/src/pages/api/core/dataset/update.ts @@ -68,7 +68,9 @@ async function handler( externalReadUrl, apiServer, yuqueServer, - feishuServer, + feishuShareServer, + feishuKnowledgeServer, + feishuPrivateServer, autoSync, chunkSettings } = req.body; @@ -176,10 +178,14 @@ async function handler( ...(!!yuqueServer?.basePath !== undefined && { 'yuqueServer.basePath': yuqueServer?.basePath }), - ...(!!feishuServer?.appId && { 'feishuServer.appId': feishuServer.appId }), - ...(!!feishuServer?.appSecret && { 'feishuServer.appSecret': feishuServer.appSecret }), - ...(!!feishuServer?.folderToken && { - 'feishuServer.folderToken': feishuServer.folderToken + ...(!!feishuShareServer?.folderToken && { + 'feishuShareServer.folderToken': feishuShareServer.folderToken + }), + ...(!!feishuKnowledgeServer?.basePath !== undefined && { + 'feishuKnowledgeServer.basePath': feishuKnowledgeServer?.basePath + }), + ...(!!feishuPrivateServer?.basePath !== undefined && { + 'feishuPrivateServer.basePath': feishuPrivateServer?.basePath }), ...(isMove && { inheritPermission: true }), ...(typeof autoSync === 'boolean' && { autoSync }) diff --git a/projects/app/src/pages/dataset/detail/index.tsx b/projects/app/src/pages/dataset/detail/index.tsx index 0decfb61730a..b41461c18dfa 100644 --- a/projects/app/src/pages/dataset/detail/index.tsx +++ b/projects/app/src/pages/dataset/detail/index.tsx @@ -1,7 +1,19 @@ 'use client'; -import React from 'react'; +import React, { useEffect } from 'react'; import { useRouter } from 'next/router'; -import { Box, Flex, type FlexProps } from '@chakra-ui/react'; +import { + Box, + Flex, + type FlexProps, + useDisclosure, + Modal, + ModalOverlay, + ModalContent, + ModalHeader, + ModalBody, + Button, + ModalFooter +} from '@chakra-ui/react'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { getErrText } from '@fastgpt/global/common/error/utils'; import dynamic from 'next/dynamic'; @@ -50,6 +62,7 @@ const Detail = ({ datasetId, currentTab }: Props) => { const { toast } = useToast(); const router = useRouter(); const { isPc } = useSystem(); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); const loadDatasetDetail = useContextSelector(DatasetPageContext, (v) => v.loadDatasetDetail); diff --git a/projects/app/src/pages/dataset/list/index.tsx b/projects/app/src/pages/dataset/list/index.tsx index 8468ea96b4bd..f5b90169b14c 100644 --- a/projects/app/src/pages/dataset/list/index.tsx +++ b/projects/app/src/pages/dataset/list/index.tsx @@ -177,10 +177,30 @@ const Dataset = () => { ...(feConfigs?.show_dataset_feishu !== false ? [ { - icon: 'core/dataset/feishuDatasetColor', - label: t('dataset:feishu_dataset'), - description: t('dataset:feishu_dataset_desc'), - onClick: () => onSelectDatasetType(DatasetTypeEnum.feishu) + icon: 'core/dataset/feishuShareDatasetColor', + label: t('dataset:feishu_share_dataset'), + description: t('dataset:feishu_share_dataset_desc'), + onClick: () => onSelectDatasetType(DatasetTypeEnum.feishuShare) + } + ] + : []), + ...(feConfigs?.show_dataset_feishu !== false + ? [ + { + icon: 'core/dataset/feishuKnowledgeDatasetColor', + label: t('dataset:feishu_knowledge_dataset'), + description: t('dataset:feishu_knowledge_dataset_desc'), + onClick: () => onSelectDatasetType(DatasetTypeEnum.feishuKnowledge) + } + ] + : []), + ...(feConfigs?.show_dataset_feishu !== false + ? [ + { + icon: 'core/dataset/feishuPrivateDatasetColor', + label: t('dataset:feishu_private_dataset'), + description: t('dataset:feishu_private_dataset_desc'), + onClick: () => onSelectDatasetType(DatasetTypeEnum.feishuPrivate) } ] : []), diff --git a/projects/app/src/service/common/system/cron.ts b/projects/app/src/service/common/system/cron.ts index 2a08da17fdb9..0f627537c468 100644 --- a/projects/app/src/service/common/system/cron.ts +++ b/projects/app/src/service/common/system/cron.ts @@ -12,6 +12,7 @@ import { TimerIdEnum } from '@fastgpt/service/common/system/timerLock/constants' import { addHours } from 'date-fns'; import { getScheduleTriggerApp } from '@/service/core/app/utils'; import { clearExpiredRawTextBufferCron } from '@fastgpt/service/common/buffer/rawText/controller'; +import { refreshFeishuToken } from '@fastgpt/service/core/dataset/apiDataset/feishuPrivateDataset/refreshToken'; // Try to run train every minute const setTrainingQueueCron = () => { @@ -20,6 +21,13 @@ const setTrainingQueueCron = () => { }); }; +// Refresh feishu token every 110 minutes +const setRefreshFeishuTokenCron = () => { + setCron('*/110 * * * *', () => { + refreshFeishuToken(); + }); +}; + // Clear tmp upload files every ten minutes const setClearTmpUploadFilesCron = () => { // Clear tmp upload files every ten minutes @@ -85,4 +93,5 @@ export const startCron = () => { clearInvalidDataCron(); scheduleTriggerAppCron(); clearExpiredRawTextBufferCron(); + setRefreshFeishuTokenCron(); }; diff --git a/projects/app/src/web/core/dataset/api.ts b/projects/app/src/web/core/dataset/api.ts index adbc5f8ff875..98b7a10b7557 100644 --- a/projects/app/src/web/core/dataset/api.ts +++ b/projects/app/src/web/core/dataset/api.ts @@ -292,3 +292,6 @@ export const getApiDatasetCatalog = (data: GetApiDatasetCataLogProps) => export const getApiDatasetPaths = (data: GetApiDatasetPathBody) => POST('/core/dataset/apiDataset/getPathNames', data); + +export const getUserAccessToken = (data: { code: string; datasetId: string }) => + POST('/core/dataset/feishu/getUserAccessToken', data); diff --git a/projects/app/src/web/core/dataset/constants.ts b/projects/app/src/web/core/dataset/constants.ts index 93be215246c0..54f9738be64e 100644 --- a/projects/app/src/web/core/dataset/constants.ts +++ b/projects/app/src/web/core/dataset/constants.ts @@ -71,7 +71,9 @@ export const datasetTypeCourseMap: Record<`${DatasetTypeEnum}`, string> = { [DatasetTypeEnum.dataset]: '', [DatasetTypeEnum.apiDataset]: '/docs/guide/knowledge_base/api_dataset/', [DatasetTypeEnum.websiteDataset]: '/docs/guide/knowledge_base/websync/', - [DatasetTypeEnum.feishu]: '/docs/guide/knowledge_base/lark_dataset/', + [DatasetTypeEnum.feishuShare]: '/docs/guide/knowledge_base/lark_dataset/', + [DatasetTypeEnum.feishuKnowledge]: '/docs/guide/knowledge_base/lark_dataset/', + [DatasetTypeEnum.feishuPrivate]: '/docs/guide/knowledge_base/lark_dataset/', [DatasetTypeEnum.yuque]: '/docs/guide/knowledge_base/yuque_dataset/', [DatasetTypeEnum.externalFile]: '' }; diff --git a/projects/app/src/web/core/dataset/context/datasetPageContext.tsx b/projects/app/src/web/core/dataset/context/datasetPageContext.tsx index 947b094fe8fb..28676ea0688c 100644 --- a/projects/app/src/web/core/dataset/context/datasetPageContext.tsx +++ b/projects/app/src/web/core/dataset/context/datasetPageContext.tsx @@ -117,13 +117,30 @@ export const DatasetPageContextProvider = ({ basePath: data.yuqueServer.basePath } : state.yuqueServer, - feishuServer: data.feishuServer + feishuShareServer: data.feishuShareServer ? { - appId: data.feishuServer.appId, - appSecret: '', - folderToken: data.feishuServer.folderToken + user_access_token: data.feishuShareServer.user_access_token, + refresh_token: data.feishuShareServer.refresh_token, + outdate_time: data.feishuShareServer.outdate_time, + folderToken: data.feishuShareServer.folderToken } - : state.feishuServer + : state.feishuShareServer, + feishuKnowledgeServer: data.feishuKnowledgeServer + ? { + user_access_token: data.feishuKnowledgeServer.user_access_token, + refresh_token: data.feishuKnowledgeServer.refresh_token, + outdate_time: data.feishuKnowledgeServer.outdate_time, + basePath: data.feishuKnowledgeServer.basePath + } + : state.feishuKnowledgeServer, + feishuPrivateServer: data.feishuPrivateServer + ? { + user_access_token: data.feishuPrivateServer.user_access_token, + refresh_token: data.feishuPrivateServer.refresh_token, + outdate_time: data.feishuPrivateServer.outdate_time, + basePath: data.feishuPrivateServer.basePath + } + : state.feishuPrivateServer })); } };