Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 5ff4cc11 by Roy Committed by GitHub
Options

feat: export all chunks in collection (#6163) 

* feat: export all chunks in collection

* perf: export collection api

* doc

---------

Co-authored-by: archer <545436317@qq.com>
parent baf1a079
Showing with 154 additions and 60 deletions
document/content/docs/upgrading/4-14/4145.mdx
...@@ -9,6 +9,7 @@ description: 'FastGPT V4.14.5 更新说明' ...@@ -9,6 +9,7 @@ description: 'FastGPT V4.14.5 更新说明'
1. 对话记录使用侧改成软删除，增加从日志管理里删除对话记录。 1. 对话记录使用侧改成软删除，增加从日志管理里删除对话记录。
2. 更新Agent/工具时，会更新其上层所有目录的更新时间，以便其会排在列表前面。 2. 更新Agent/工具时，会更新其上层所有目录的更新时间，以便其会排在列表前面。
3. 门户页支持配置单个应用运行可见度配。 3. 门户页支持配置单个应用运行可见度配。
4. 导出单个知识库集合分块接口。
## ⚙️ 优化 ## ⚙️ 优化
......
document/data/doc-last-modified.json
...@@ -120,7 +120,7 @@ ...@@ -120,7 +120,7 @@
"document/content/docs/upgrading/4-14/4142.mdx": "2025-11-18T19:27:14+08:00", "document/content/docs/upgrading/4-14/4142.mdx": "2025-11-18T19:27:14+08:00",
"document/content/docs/upgrading/4-14/4143.mdx": "2025-11-26T20:52:05+08:00", "document/content/docs/upgrading/4-14/4143.mdx": "2025-11-26T20:52:05+08:00",
"document/content/docs/upgrading/4-14/4144.mdx": "2025-12-16T14:56:04+08:00", "document/content/docs/upgrading/4-14/4144.mdx": "2025-12-16T14:56:04+08:00",
"document/content/docs/upgrading/4-14/4145.mdx": "2025-12-21T23:28:19+08:00", "document/content/docs/upgrading/4-14/4145.mdx": "2025-12-24T14:28:42+08:00",
"document/content/docs/upgrading/4-8/40.mdx": "2025-08-02T19:38:37+08:00", "document/content/docs/upgrading/4-8/40.mdx": "2025-08-02T19:38:37+08:00",
"document/content/docs/upgrading/4-8/41.mdx": "2025-08-02T19:38:37+08:00", "document/content/docs/upgrading/4-8/41.mdx": "2025-08-02T19:38:37+08:00",
"document/content/docs/upgrading/4-8/42.mdx": "2025-08-02T19:38:37+08:00", "document/content/docs/upgrading/4-8/42.mdx": "2025-08-02T19:38:37+08:00",
......
packages/global/openapi/core/chat/controler/api.ts
import { OutLinkChatAuthSchema } from '../../../../support/permission/chat/type'; import { OutLinkChatAuthSchema } from '../../../../support/permission/chat';
import { ObjectIdSchema } from '../../../../common/type/mongo'; import { ObjectIdSchema } from '../../../../common/type/mongo';
import z from 'zod'; import z from 'zod';
......
packages/global/openapi/core/dataset/collection/api.ts 0 → 100644
import { ObjectIdSchema } from '../../../../common/type/mongo';
import { OutLinkChatAuthSchema } from '../../../../support/permission/chat';
import z from 'zod';
// Schema 1: Basic collection export with authentication
const BasicExportSchema = z
.object({
collectionId: ObjectIdSchema.describe('集合ID')
})
.meta({
description: '通过身份鉴权导出集合',
example: {
collectionId: '1234567890'
}
});
// Schema 2: Export from chat context with outlink authentication
const ChatExportSchema = OutLinkChatAuthSchema.extend({
collectionId: ObjectIdSchema.describe('集合ID'),
appId: ObjectIdSchema.describe('应用ID'),
chatId: ObjectIdSchema.describe('会话ID'),
chatItemDataId: z.string().describe('对话ID'),
chatTime: z.coerce.date().optional().describe('对话时间')
}).meta({
description: '对话中导出集合，可通过 chatId 等身份信息',
example: {
collectionId: '1234567890',
appId: '1234567890',
chatId: '1234567890',
chatItemDataId: '1234567890',
chatTime: '2025-12-30T00:00:00.000Z',
shareId: '1234567890',
outLinkUid: '1234567890'
}
});
export const ExportCollectionBodySchema = z.union([BasicExportSchema, ChatExportSchema]);
export type ExportCollectionBodyType = z.infer<typeof ExportCollectionBodySchema>;
packages/global/openapi/core/dataset/collection/index.ts 0 → 100644
import type { OpenAPIPath } from '../../../type';
import { TagsMap } from '../../../tag';
import { ExportCollectionBodySchema } from './api';
export const DatasetCollectionPath: OpenAPIPath = {
'/core/dataset/collection/export': {
post: {
summary: '下载集合的所有数据块',
description: '下载集合的所有数据块',
tags: [TagsMap.datasetCollection],
requestBody: {
content: {
'application/json': {
schema: ExportCollectionBodySchema
}
}
},
responses: {
200: {
description: '成功导出并下载集合的所有数据块内容'
}
}
}
}
};
packages/global/openapi/core/dataset/data/index.ts 0 → 100644
import type { OpenAPIPath } from '../../../type';
import { TagsMap } from '../../../tag';
export const DatasetDataPath: OpenAPIPath = {};
packages/global/openapi/core/dataset/index.ts 0 → 100644
import type { OpenAPIPath } from '../../type';
import { DatasetDataPath } from './data';
import { DatasetCollectionPath } from './collection';
export const DatasetPath: OpenAPIPath = {
...DatasetDataPath,
...DatasetCollectionPath
};
packages/global/openapi/index.ts
...@@ -4,6 +4,7 @@ import { TagsMap } from './tag'; ...@@ -4,6 +4,7 @@ import { TagsMap } from './tag';
import { PluginPath } from './core/plugin'; import { PluginPath } from './core/plugin';
import { AppPath } from './core/app'; import { AppPath } from './core/app';
import { SupportPath } from './support'; import { SupportPath } from './support';
import { DatasetPath } from './core/dataset';
export const openAPIDocument = createDocument({ export const openAPIDocument = createDocument({
openapi: '3.1.0', openapi: '3.1.0',
...@@ -15,6 +16,7 @@ export const openAPIDocument = createDocument({ ...@@ -15,6 +16,7 @@ export const openAPIDocument = createDocument({
paths: { paths: {
...AppPath, ...AppPath,
...ChatPath, ...ChatPath,
...DatasetPath,
...PluginPath, ...PluginPath,
...SupportPath ...SupportPath
}, },
...@@ -29,6 +31,10 @@ export const openAPIDocument = createDocument({ ...@@ -29,6 +31,10 @@ export const openAPIDocument = createDocument({
tags: [TagsMap.chatHistory, TagsMap.chatPage, TagsMap.chatFeedback, TagsMap.chatSetting] tags: [TagsMap.chatHistory, TagsMap.chatPage, TagsMap.chatFeedback, TagsMap.chatSetting]
}, },
{ {
name: '知识库',
tags: [TagsMap.datasetCollection]
},
{
name: '插件系统', name: '插件系统',
tags: [TagsMap.pluginToolTag, TagsMap.pluginTeam] tags: [TagsMap.pluginToolTag, TagsMap.pluginTeam]
}, },
......
packages/global/openapi/tag.ts
...@@ -10,6 +10,10 @@ export const TagsMap = { ...@@ -10,6 +10,10 @@ export const TagsMap = {
chatSetting: '门户页配置', chatSetting: '门户页配置',
chatFeedback: '对话反馈', chatFeedback: '对话反馈',
// Dataset
datasetCollection: '集合',
datasetData: '数据',
// Plugin // Plugin
pluginToolTag: '工具标签', pluginToolTag: '工具标签',
pluginTeam: '团队插件管理', pluginTeam: '团队插件管理',
......
packages/global/support/permission/chat/type.ts deleted 100644 → 0
import { z } from 'zod';
export const OutLinkChatAuthSchema = z.union([
z
.object({
shareId: z.string().optional(),
outLinkUid: z.string().optional()
})
.meta({
description: '分享链接鉴权',
example: {
shareId: '1234567890',
outLinkUid: '1234567890'
}
}),
z
.object({
teamId: z.string().optional(),
teamToken: z.string().optional()
})
.meta({
description: '团队鉴权',
example: {
teamId: '1234567890',
teamToken: '1234567890'
}
})
]);
export type OutLinkChatAuthType = z.infer<typeof OutLinkChatAuthSchema>;
packages/web/i18n/en/dataset.json
...@@ -26,6 +26,8 @@ ...@@ -26,6 +26,8 @@
"close_auto_sync": "Are you sure you want to turn off automatic sync?", "close_auto_sync": "Are you sure you want to turn off automatic sync?",
"collection.Create update time": "Creation/Update Time", "collection.Create update time": "Creation/Update Time",
"collection.Training type": "Training", "collection.Training type": "Training",
"collection.export_all_chunks": "Export chunks",
"collection.not_found": "Collection does not exist",
"collection.sync.submit": "The synchronization task has been submitted", "collection.sync.submit": "The synchronization task has been submitted",
"collection.training_type": "Chunk type", "collection.training_type": "Chunk type",
"collection_data_count": "Data amount", "collection_data_count": "Data amount",
......
packages/web/i18n/zh-CN/dataset.json
...@@ -25,7 +25,9 @@ ...@@ -25,7 +25,9 @@
"chunk_trigger_tips": "当满足一定条件时才触发分块存储，否则会直接完整存储原文", "chunk_trigger_tips": "当满足一定条件时才触发分块存储，否则会直接完整存储原文",
"close_auto_sync": "确认关闭自动同步功能？", "close_auto_sync": "确认关闭自动同步功能？",
"collection.Create update time": "创建/更新时间", "collection.Create update time": "创建/更新时间",
"collection.not_found": "集合不存在",
"collection.Training type": "训练模式", "collection.Training type": "训练模式",
"collection.export_all_chunks": "导出分块",
"collection.sync.submit": "已提交同步任务", "collection.sync.submit": "已提交同步任务",
"collection.training_type": "处理模式", "collection.training_type": "处理模式",
"collection_data_count": "数据量", "collection_data_count": "数据量",
......
packages/web/i18n/zh-Hant/dataset.json
...@@ -26,6 +26,8 @@ ...@@ -26,6 +26,8 @@
"close_auto_sync": "確認關閉自動同步功能？", "close_auto_sync": "確認關閉自動同步功能？",
"collection.Create update time": "建立／更新時間", "collection.Create update time": "建立／更新時間",
"collection.Training type": "分段模式", "collection.Training type": "分段模式",
"collection.export_all_chunks": "導出分塊",
"collection.not_found": "集合不存在",
"collection.sync.submit": "已提交同步任務", "collection.sync.submit": "已提交同步任務",
"collection.training_type": "處理模式", "collection.training_type": "處理模式",
"collection_data_count": "資料量", "collection_data_count": "資料量",
......
projects/app/src/pageComponents/dataset/detail/DataCard.tsx
...@@ -38,6 +38,7 @@ import PopoverConfirm from '@fastgpt/web/components/common/MyPopover/PopoverConf ...@@ -38,6 +38,7 @@ import PopoverConfirm from '@fastgpt/web/components/common/MyPopover/PopoverConf
import { formatFileSize } from '@fastgpt/global/common/file/tools'; import { formatFileSize } from '@fastgpt/global/common/file/tools';
import MyImage from '@fastgpt/web/components/common/Image/MyImage'; import MyImage from '@fastgpt/web/components/common/Image/MyImage';
import dynamic from 'next/dynamic'; import dynamic from 'next/dynamic';
import { downloadFetch } from '@/web/common/system/utils';
const InsertImagesModal = dynamic(() => import('./data/InsertImageModal'), { const InsertImagesModal = dynamic(() => import('./data/InsertImageModal'), {
ssr: false ssr: false
...@@ -128,6 +129,21 @@ const DataCard = () => { ...@@ -128,6 +129,21 @@ const DataCard = () => {
} }
}); });
const { runAsync: onExportAllChunks, loading: isExportChunksLoading } = useRequest2(
async (collectionId: string) => {
await downloadFetch({
url: '/api/core/dataset/collection/export',
filename: `${collection?.name}.csv`,
body: {
collectionId
}
});
},
{
manual: true
}
);
return ( return (
<MyBox py={[1, 0]} h={'100%'}> <MyBox py={[1, 0]} h={'100%'}>
<Flex flexDirection={'column'} h={'100%'}> <Flex flexDirection={'column'} h={'100%'}>
...@@ -155,6 +171,19 @@ const DataCard = () => { ...@@ -155,6 +171,19 @@ const DataCard = () => {
<TagsPopOver currentCollection={collection} /> <TagsPopOver currentCollection={collection} />
)} )}
</Box> </Box>
<Button
variant={'whitePrimary'}
size={['sm', 'md']}
isDisabled={!collection}
isLoading={isExportChunksLoading}
onClick={() => {
onExportAllChunks(collection?._id!);
}}
>
{t('dataset:collection.export_all_chunks')}
</Button>
{datasetDetail.type !== 'websiteDataset' && {datasetDetail.type !== 'websiteDataset' &&
!!collection?.chunkSize && !!collection?.chunkSize &&
collection.permission?.hasWritePer && ( collection.permission?.hasWritePer && (
...@@ -382,6 +411,7 @@ const DataCard = () => { ...@@ -382,6 +411,7 @@ const DataCard = () => {
</> </>
)} )}
</Flex> </Flex>
{canWrite && ( {canWrite && (
<PopoverConfirm <PopoverConfirm
Trigger={ Trigger={
......
projects/app/src/pages/api/core/dataset/collection/export.ts
import { NextAPI } from '@/service/middleware/entry'; import { NextAPI } from '@/service/middleware/entry';
import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat'; import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { type OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { useIPFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit'; import { useIPFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
...@@ -13,40 +12,35 @@ import { authDatasetCollection } from '@fastgpt/service/support/permission/datas ...@@ -13,40 +12,35 @@ import { authDatasetCollection } from '@fastgpt/service/support/permission/datas
import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { type NextApiResponse } from 'next'; import { type NextApiResponse } from 'next';
import { sanitizeCsvField } from '@fastgpt/service/common/file/csv'; import { sanitizeCsvField } from '@fastgpt/service/common/file/csv';
import { replaceS3KeyToPreviewUrl } from '@fastgpt/service/core/dataset/utils';
import { addDays } from 'date-fns';
import { ExportCollectionBodySchema } from '@fastgpt/global/openapi/core/dataset/collection/api';
export type ExportCollectionBody = { async function handler(req: ApiRequestProps, res: NextApiResponse) {
collectionId: string; const parseBody = ExportCollectionBodySchema.parse(req.body);
const collectionId = parseBody.collectionId;
appId?: string;
chatId?: string;
chatItemDataId?: string;
chatTime: Date;
} & OutLinkChatAuthProps;
async function handler(req: ApiRequestProps<ExportCollectionBody, {}>, res: NextApiResponse) {
const { const {
collectionId, collection,
appId, teamId: userTeamId,
chatId,
chatItemDataId,
shareId,
outLinkUid,
teamId,
teamToken,
chatTime chatTime
} = req.body; } = await (async () => {
if (!('chatItemDataId' in parseBody)) {
const { collection, teamId: userTeamId } = await (async () => { const result = await authDatasetCollection({
if (!appId || !chatId || !chatItemDataId) {
return authDatasetCollection({
req, req,
authToken: true, authToken: true,
authApiKey: true, authApiKey: true,
collectionId: req.body.collectionId, collectionId,
per: ReadPermissionVal per: ReadPermissionVal
}); });
return {
...result,
chatTime: undefined
};
} }
const { appId, chatId, chatItemDataId, shareId, outLinkUid, teamId, teamToken, chatTime } =
parseBody;
/* /*
1. auth chat read permission 1. auth chat read permission
2. auth collection quote in chat 2. auth collection quote in chat
...@@ -73,7 +67,8 @@ async function handler(req: ApiRequestProps<ExportCollectionBody, {}>, res: Next ...@@ -73,7 +67,8 @@ async function handler(req: ApiRequestProps<ExportCollectionBody, {}>, res: Next
return { return {
...authRes, ...authRes,
collection collection,
chatTime
}; };
})(); })();
...@@ -107,11 +102,17 @@ async function handler(req: ApiRequestProps<ExportCollectionBody, {}>, res: Next ...@@ -107,11 +102,17 @@ async function handler(req: ApiRequestProps<ExportCollectionBody, {}>, res: Next
readStream: cursor readStream: cursor
}); });
write(`\uFEFFindex,content`); write(`\uFEFFq,a`);
cursor.on('data', (doc) => { cursor.on('data', (doc) => {
const sanitizedQ = sanitizeCsvField(doc.q || ''); const sanitizedQ = replaceS3KeyToPreviewUrl(
const sanitizedA = sanitizeCsvField(doc.a || ''); sanitizeCsvField(doc.q || ''),
addDays(new Date(), 90)
);
const sanitizedA = replaceS3KeyToPreviewUrl(
sanitizeCsvField(doc.a || ''),
addDays(new Date(), 90)
);
write(`\n${sanitizedQ},${sanitizedA}`); write(`\n${sanitizedQ},${sanitizedA}`);
}); });
......
projects/app/src/web/core/dataset/api.ts
...@@ -83,7 +83,6 @@ import type { ...@@ -83,7 +83,6 @@ import type {
DatasetCreateWithFilesBody, DatasetCreateWithFilesBody,
DatasetCreateWithFilesResponse DatasetCreateWithFilesResponse
} from '@/pages/api/core/dataset/createWithFiles'; } from '@/pages/api/core/dataset/createWithFiles';
import type { PresignDatasetFileGetUrlParams } from '@fastgpt/global/core/dataset/v2/api';
/* ======================== dataset ======================= */ /* ======================== dataset ======================= */
export const getDatasets = (data: GetDatasetListBody) => export const getDatasets = (data: GetDatasetListBody) =>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment