Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 58000324 by Archer Committed by GitHub
Options

feature: V4.14.3 (#5970) 

* feat(marketplace): update plugin/ download count statistic (#5957)

* feat: download count

* feat: update ui

* fix: ui

* chore: update sdk verison

* chore: update .env.template

* chore: adjust

* chore: remove console.log

* chore: adjust

* Update projects/marketplace/src/pages/index.tsx

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update projects/marketplace/src/pages/index.tsx

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update projects/app/src/pages/config/tool/marketplace.tsx

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* fix: update refresh; feat: marketplace download count per hour

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* download

* marketplace code

* fix: ui (#5963)

* feat: support dataset and files as global variables (#5961)

* json & dataset

* file

* fix file var

* fix

* fix init

* remove

* perf: file vars

* fix: file uploading errors (#5969)

* fix: file uploading errors

* fix build

* perf: fileselector ux

* feat: integrate S3 for dataset with compatibility (#5941)

* fix: text split

* remove test

* feat: integrate S3 for dataset with compatibility

* fix: delay s3 files delete timing

* fix: remove imageKeys

* fix: remove parsed images' TTL

* fix: improve codes by pr comments

---------

Co-authored-by: archer <545436317@qq.com>

* remove log

* perf: request limit

* chore: s3 migration script (#5971)

* test

* perf: s3 code

* fix: migration script (#5972)

* perf: s3 move object

* wip: fix s3 bugs (#5976)

* fix: incorrect replace origin logic (#5978)

* fix: add downloadURL (#5980)

* perf: file variable ttl & quick create dataset with temp s3 bucket (#5973)

* perf: file variable ttl & quick create dataset with temp s3 bucket

* fix

* plugin & form input variables (#5979)

* plugin & form input variables

* fix

* docs: 4143.mdx (#5981)

* doc: update 4143.mdx (#5982)

* fix form input file ttl (#5983)

* trans file type (#5986)

* trans file type

* fix

* fix: S3 script early return (#5985)

* fix: S3 script typeof

* fix: truncate large filename to fit S3 name

* perf(permission): add a schema verification for resource permission, tmbId, groupId, orgId should be set at least one of them (#5987)

* fix: version & typo (#5988)

* fix-v4.14.3 (#5991)

* fix: empty alt make replace JWT failed & incorrect image dataset preview url (#5989)

* fix: empty alt make replace JWT failed & incorrect image dataset preview url

* fix: s3 files recovery script

* fix: incorrect chat external url parsing (#5993)

---------

Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: Roy <whoeverimf5@gmail.com>
parent 91156f13
Showing with 2624 additions and 563 deletions
document/content/docs/upgrading/4-14/4143.mdx
---
title: 'V4.14.3(进行中)'
description: 'FastGPT V4.14.2 更新说明'
title: 'V4.14.3（进行中）'
description: 'FastGPT V4.14.3 更新说明'
---
## 更新指南
### 1. 更新镜像：
- 更新 FastGPT 镜像tag: v4.14.3
- 更新 FastGPT 商业版镜像tag: v4.14.3
- 更新 fastgpt-plugin 镜像 tag: v0.3.3
- mcp_server 无需更新
- Sandbox 无需更新
- AIProxy 无需更新
### 2. 执行升级脚本
从任意终端，发起 1 个 HTTP 请求。其中 `{{rootkey}}` 替换成环境变量里的 `rootkey`；`{{host}}` 替换成**FastGPT 域名**。
```bash
curl --location --request POST 'https://{{host}}/api/admin/initv4143' \
--header 'rootkey: {{rootkey}}' \
--header 'Content-Type: application/json'
```
会将原系统 MongoDB 的 GridFS 中的所有知识库文件迁移到 S3 中，包含文本数据集和图片数据集，但不包括文档（如 .docx）里解析出来的图片。
## 🚀 新增内容
1. 知识库文件迁移至 S3(全部使用文件的地方均已迁移)。
2. 全局变量支持文件上传。
3. 表单输入节点支持密码、开关、时间点、时间范围、文件上传、对话模型选择。
4. 插件输入支持多选、时间点、时间范围、内部变量。
5. 系统插件，插件市场中会提示是否有新版本，并提供更新按键。
6. 工作流运行 QPM 限制。
## ⚙️ 优化
1. 工作流工具，文件上传输入 UX 优化。
## 🐛 修复
## 插件
1. 工作流调试预览窗口，重新渲染导致输入丢失。
2. S3 服务与主服务相同 Origin 的域名会导致对 S3 的文件请求 URL 被错误替换，产生 404 报错。
document/data/doc-last-modified.json
......@@ -101,7 +101,7 @@
"document/content/docs/protocol/terms.en.mdx": "2025-08-03T22:37:45+08:00",
"document/content/docs/protocol/terms.mdx": "2025-08-03T22:37:45+08:00",
"document/content/docs/toc.en.mdx": "2025-08-04T13:42:36+08:00",
"document/content/docs/toc.mdx": "2025-11-14T13:21:17+08:00",
"document/content/docs/toc.mdx": "2025-11-18T19:27:14+08:00",
"document/content/docs/upgrading/4-10/4100.mdx": "2025-08-02T19:38:37+08:00",
"document/content/docs/upgrading/4-10/4101.mdx": "2025-09-08T20:07:20+08:00",
"document/content/docs/upgrading/4-11/4110.mdx": "2025-08-05T23:20:39+08:00",
......@@ -115,8 +115,9 @@
"document/content/docs/upgrading/4-13/4131.mdx": "2025-09-30T15:47:06+08:00",
"document/content/docs/upgrading/4-13/4132.mdx": "2025-10-21T11:46:53+08:00",
"document/content/docs/upgrading/4-14/4140.mdx": "2025-11-06T15:43:00+08:00",
"document/content/docs/upgrading/4-14/4141.mdx": "2025-11-12T12:19:02+08:00",
"document/content/docs/upgrading/4-14/4142.mdx": "2025-11-18T18:16:09+08:00",
"document/content/docs/upgrading/4-14/4141.mdx": "2025-11-19T10:15:27+08:00",
"document/content/docs/upgrading/4-14/4142.mdx": "2025-11-18T19:27:14+08:00",
"document/content/docs/upgrading/4-14/4143.mdx": "2025-11-25T17:00:39+08:00",
"document/content/docs/upgrading/4-8/40.mdx": "2025-08-02T19:38:37+08:00",
"document/content/docs/upgrading/4-8/41.mdx": "2025-08-02T19:38:37+08:00",
"document/content/docs/upgrading/4-8/42.mdx": "2025-08-02T19:38:37+08:00",
......
packages/global/common/file/image/constants.ts
......@@ -4,3 +4,4 @@ export const FolderIcon = 'file/fill/folder';
export const FolderImgUrl = '/imgs/files/folder.svg';
export const HttpPluginImgUrl = '/imgs/app/httpPluginFill.svg';
export const HttpImgUrl = '/imgs/workflow/http.png';
export const TempFileURL = '/api/file/temp';
packages/global/core/dataset/training/type.d.ts
......@@ -9,7 +9,6 @@ export type PushDataToTrainingQueueProps = {
data: PushDatasetDataChunkProps[];
mode?: TrainingModeEnum;
data: PushDatasetDataChunkProps[];
agentModel: string;
vectorModel: string;
......
packages/global/core/dataset/type.d.ts
......@@ -118,6 +118,7 @@ export type DatasetCollectionSchemaType = ChunkSettingsType & {
rawTextLength?: number;
hashRawText?: string;
metadata?: {
webPageSelector?: string;
relatedImgId?: string; // The id of the associated image collections
......@@ -250,7 +251,10 @@ export type TagUsageType = {
export type DatasetCollectionItemType = CollectionWithDatasetType & {
sourceName: string;
sourceId?: string;
file?: DatasetFileSchema;
file?: {
filename?: string;
contentLength?: number;
};
permission: DatasetPermission;
indexAmount: number;
errorCount?: number;
......
packages/global/core/dataset/v2/api.ts 0 → 100644
import { ObjectIdSchema } from '../../../common/type/mongo';
import z from 'zod';
export const PresignDatasetFileGetUrlSchema = z.union([
z.object({
key: z
.string()
.nonempty()
.refine((key) => key.startsWith('dataset/'), {
message: 'Invalid key format: must start with "dataset/"'
})
.transform((k) => decodeURIComponent(k)),
preview: z.boolean().optional()
}),
z.object({
collectionId: ObjectIdSchema
// datasetId: ObjectIdSchema
})
]);
export type PresignDatasetFileGetUrlParams = z.infer<typeof PresignDatasetFileGetUrlSchema>;
export const PresignDatasetFilePostUrlSchema = z.object({
filename: z.string().min(1),
datasetId: ObjectIdSchema
});
export type PresignDatasetFilePostUrlParams = z.infer<typeof PresignDatasetFilePostUrlSchema>;
export const ShortPreviewLinkSchema = z.object({
k: z
.string()
.nonempty()
.transform((k) => `chat:temp_file:${decodeURIComponent(k)}`)
});
export type ShortPreviewLinkParams = z.infer<typeof ShortPreviewLinkSchema>;
packages/global/core/workflow/constants.ts
......@@ -319,7 +319,6 @@ export enum VariableInputEnum {
input = 'input',
textarea = 'textarea',
numberInput = 'numberInput',
JSONEditor = 'JSONEditor',
select = 'select',
multipleSelect = 'multipleSelect',
timePointSelect = 'timePointSelect',
......@@ -363,12 +362,6 @@ export const variableConfigs: VariableConfigType[][] = [
value: VariableInputEnum.numberInput,
defaultValueType: WorkflowIOValueTypeEnum.number
},
// {
// icon: 'core/workflow/inputType/jsonEditor',
// label: i18nT('common:core.workflow.inputType.jsonEditor'),
// value: VariableInputEnum.JSONEditor,
// defaultValueType: WorkflowIOValueTypeEnum.object
// },
{
icon: 'core/workflow/inputType/option',
label: i18nT('common:core.workflow.inputType.select'),
......@@ -398,34 +391,22 @@ export const variableConfigs: VariableConfigType[][] = [
label: i18nT('common:core.workflow.inputType.timeRangeSelect'),
value: VariableInputEnum.timeRangeSelect,
defaultValueType: WorkflowIOValueTypeEnum.arrayString
},
}
],
[
{
icon: 'core/workflow/inputType/model',
label: i18nT('common:core.workflow.inputType.modelSelect'),
value: VariableInputEnum.llmSelect,
defaultValueType: WorkflowIOValueTypeEnum.string
},
{
icon: 'core/workflow/inputType/file',
label: i18nT('common:core.workflow.inputType.file'),
value: VariableInputEnum.file,
defaultValueType: WorkflowIOValueTypeEnum.arrayString
}
// {
// icon: 'core/workflow/inputType/file',
// label: i18nT('common:core.workflow.inputType.file'),
// value: VariableInputEnum.file,
// defaultValueType: WorkflowIOValueTypeEnum.arrayString
// }
],
// [
// {
// icon: 'core/workflow/inputType/model',
// label: i18nT('common:core.workflow.inputType.modelSelect'),
// value: VariableInputEnum.llmSelect,
// defaultValueType: WorkflowIOValueTypeEnum.string
// },
// {
// icon: 'core/workflow/inputType/dataset',
// label: i18nT('common:core.workflow.inputType.datasetSelect'),
// value: VariableInputEnum.datasetSelect,
// defaultValueType: WorkflowIOValueTypeEnum.selectDataset
// }
// ],
[
{
icon: 'core/workflow/inputType/external',
......
packages/global/core/workflow/node/constant.ts
......@@ -49,7 +49,7 @@ export const FlowNodeInputMap: Record<
icon: 'core/workflow/inputType/option'
},
[FlowNodeInputTypeEnum.multipleSelect]: {
icon: 'core/workflow/inputType/option'
icon: 'core/workflow/inputType/multipleSelect'
},
[FlowNodeInputTypeEnum.switch]: {
icon: 'core/workflow/inputType/switch'
......@@ -79,7 +79,7 @@ export const FlowNodeInputMap: Record<
icon: 'core/workflow/inputType/selectDataset'
},
[FlowNodeInputTypeEnum.hidden]: {
icon: 'core/workflow/inputType/select'
icon: 'core/workflow/inputType/internal'
},
[FlowNodeInputTypeEnum.customVariable]: {
icon: 'core/workflow/inputType/customVariable'
......
packages/global/core/workflow/runtime/type.d.ts
......@@ -67,6 +67,7 @@ export type ChatDispatchProps = {
responseChatItemId?: string;
histories: ChatItemType[];
variables: Record<string, any>; // global variable
cloneVariables: Record<string, any>;
query: UserChatItemValueItemType[]; // trigger query
chatConfig: AppSchema['chatConfig'];
lastInteractive?: WorkflowInteractiveResponseType; // last interactive response
......@@ -247,7 +248,7 @@ export type DispatchNodeResponseType = {
loopOutputValue?: any;
// form input
formInputResult?: string;
formInputResult?: Record<string, any>;
// tool params
toolParamsResult?: Record<string, any>;
......
packages/global/core/workflow/template/system/interactive/type.d.ts
import type { NodeOutputItemType } from '../../../../chat/type';
import type { FlowNodeOutputItemType } from '../../../type/io';
import type { FlowNodeInputTypeEnum } from 'core/workflow/node/constant';
import type { WorkflowIOValueTypeEnum } from 'core/workflow/constants';
import type { FlowNodeInputTypeEnum } from '../../../node/constant';
import type { WorkflowIOValueTypeEnum } from '../../../constants';
import type { ChatCompletionMessageParam } from '../../../../ai/type';
type InteractiveBasicType = {
......@@ -71,6 +71,10 @@ export type UserInputFormItemType = {
// input & textarea
maxLength?: number;
// password
minLength?: number;
// numberInput
max?: number;
min?: number;
......
packages/global/core/workflow/type/io.d.ts
......@@ -54,6 +54,9 @@ export type InputComponentPropsType = {
timeRangeStart?: string;
timeRangeEnd?: string;
// dataset select
datasetOptions?: SelectedDatasetType[];
// dynamic input
customInputConfig?: CustomFieldConfigType;
......
packages/global/core/workflow/utils.ts
......@@ -254,7 +254,6 @@ export const appData2FlowNodeIO = ({
[VariableInputEnum.numberInput]: [FlowNodeInputTypeEnum.numberInput],
[VariableInputEnum.select]: [FlowNodeInputTypeEnum.select],
[VariableInputEnum.multipleSelect]: [FlowNodeInputTypeEnum.multipleSelect],
[VariableInputEnum.JSONEditor]: [FlowNodeInputTypeEnum.JSONEditor],
[VariableInputEnum.timePointSelect]: [FlowNodeInputTypeEnum.timePointSelect],
[VariableInputEnum.timeRangeSelect]: [FlowNodeInputTypeEnum.timeRangeSelect],
[VariableInputEnum.switch]: [FlowNodeInputTypeEnum.switch],
......
packages/global/openapi/core/plugin/marketplace/api.ts
......@@ -7,19 +7,16 @@ const formatToolDetailSchema = z.object({});
const formatToolSimpleSchema = z.object({});
// Create intersection types for extended schemas
export const MarketplaceToolListItemSchema = formatToolSimpleSchema.extend({
downloadUrl: z.string()
});
export const MarketplaceToolListItemSchema = formatToolSimpleSchema;
export type MarketplaceToolListItemType = ToolSimpleType & {
downloadUrl: string;
downloadCount: number;
};
export const MarketplaceToolDetailItemSchema = formatToolDetailSchema.extend({
readme: z.string().optional()
});
export const MarketplaceToolDetailSchema = z.object({
tools: z.array(MarketplaceToolDetailItemSchema),
downloadUrl: z.string()
tools: z.array(MarketplaceToolDetailItemSchema)
});
// List
......@@ -57,7 +54,12 @@ export type GetSystemInstalledPluginsQueryType = z.infer<
typeof GetSystemInstalledPluginsQuerySchema
>;
export const GetSystemInstalledPluginsResponseSchema = z.object({
ids: z.array(z.string())
list: z.array(
z.object({
id: z.string(),
version: z.string()
})
)
});
export type GetSystemInstalledPluginsResponseType = z.infer<
typeof GetSystemInstalledPluginsResponseSchema
......
packages/global/package.json
......@@ -2,7 +2,7 @@
"name": "@fastgpt/global",
"version": "1.0.0",
"dependencies": {
"@fastgpt-sdk/plugin": "0.2.15",
"@fastgpt-sdk/plugin": "0.2.16",
"@apidevtools/swagger-parser": "^10.1.0",
"@bany/curl-to-json": "^1.2.8",
"axios": "^1.12.1",
......
packages/service/common/api/frequencyLimit.ts
/* 基于 Team 的限流 */
import { getGlobalRedisConnection } from '../../common/redis';
import { jsonRes } from '../../common/response';
import type { NextApiResponse } from 'next';
export enum LimitTypeEnum {
chat = 'chat'
}
const limitMap = {
[LimitTypeEnum.chat]: {
seconds: 60,
limit: Number(process.env.CHAT_MAX_QPM || 5000)
}
};
type FrequencyLimitOption = {
teamId: string;
seconds: number;
limit: number;
keyPrefix: string;
type: LimitTypeEnum;
res: NextApiResponse;
};
export const teamFrequencyLimit = async ({
teamId,
seconds,
limit,
keyPrefix,
res
}: FrequencyLimitOption) => {
export const teamFrequencyLimit = async ({ teamId, type, res }: FrequencyLimitOption) => {
const { seconds, limit } = limitMap[type];
const redis = getGlobalRedisConnection();
const key = `${keyPrefix}:${teamId}`;
const key = `frequency:${type}:${teamId}`;
const result = await redis
.multi()
......
packages/service/common/file/gridfs/controller.ts
......@@ -4,16 +4,13 @@ import fsp from 'fs/promises';
import fs from 'fs';
import { type DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
import { MongoChatFileSchema, MongoDatasetFileSchema } from './schema';
import { detectFileEncoding, detectFileEncodingByPath } from '@fastgpt/global/common/file/tools';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { readRawContentByFileBuffer } from '../read/utils';
import { computeGridFsChunSize, gridFsStream2Buffer, stream2Encoding } from './utils';
import { detectFileEncodingByPath } from '@fastgpt/global/common/file/tools';
import { computeGridFsChunSize, stream2Encoding } from './utils';
import { addLog } from '../../system/log';
import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
import { Readable } from 'stream';
import { addRawTextBuffer, getRawTextBuffer } from '../../buffer/rawText/controller';
import { addMinutes } from 'date-fns';
import { retryFn } from '@fastgpt/global/common/system/utils';
import { getS3DatasetSource } from '../../s3/sources/dataset';
import { isS3ObjectKey } from '../../s3/utils';
export function getGFSCollection(bucket: `${BucketNameEnum}`) {
MongoDatasetFileSchema;
......@@ -85,59 +82,6 @@ export async function uploadFile({
return String(stream.id);
}
export async function uploadFileFromBase64Img({
bucketName,
teamId,
tmbId,
base64,
filename,
metadata = {}
}: {
bucketName: `${BucketNameEnum}`;
teamId: string;
tmbId: string;
base64: string;
filename: string;
metadata?: Record<string, any>;
}) {
if (!base64) return Promise.reject(`filePath is empty`);
if (!filename) return Promise.reject(`filename is empty`);
const base64Data = base64.split(',')[1];
const contentType = base64.split(',')?.[0]?.split?.(':')?.[1];
const buffer = Buffer.from(base64Data, 'base64');
const readableStream = new Readable({
read() {
this.push(buffer);
this.push(null);
}
});
const { stream: readStream, encoding } = await stream2Encoding(readableStream);
// Add default metadata
metadata.teamId = teamId;
metadata.tmbId = tmbId;
metadata.encoding = encoding;
// create a gridfs bucket
const bucket = getGridBucket(bucketName);
const stream = bucket.openUploadStream(filename, {
metadata,
contentType
});
// save to gridfs
await new Promise((resolve, reject) => {
readStream
.pipe(stream as any)
.on('finish', resolve)
.on('error', reject);
});
return String(stream.id);
}
export async function getFileById({
bucketName,
......@@ -166,7 +110,11 @@ export async function delFileByFileIdList({
for await (const fileId of fileIdList) {
try {
await bucket.delete(new Types.ObjectId(String(fileId)));
if (isS3ObjectKey(fileId, 'dataset')) {
await getS3DatasetSource().deleteDatasetFileByKey(fileId);
} else {
await bucket.delete(new Types.ObjectId(String(fileId)));
}
} catch (error: any) {
if (typeof error?.message === 'string' && error.message.includes('File not found')) {
addLog.warn('File not found', { fileId });
......@@ -189,78 +137,3 @@ export async function getDownloadStream({
return bucket.openDownloadStream(new Types.ObjectId(fileId));
}
export const readFileContentFromMongo = async ({
teamId,
tmbId,
bucketName,
fileId,
customPdfParse = false,
getFormatText,
usageId
}: {
teamId: string;
tmbId: string;
bucketName: `${BucketNameEnum}`;
fileId: string;
customPdfParse?: boolean;
getFormatText?: boolean; // 数据类型都尽可能转化成 markdown 格式
usageId?: string;
}): Promise<{
rawText: string;
filename: string;
}> => {
const bufferId = `${String(fileId)}-${customPdfParse}`;
// read buffer
const fileBuffer = await getRawTextBuffer(bufferId);
if (fileBuffer) {
return {
rawText: fileBuffer.text,
filename: fileBuffer?.sourceName
};
}
const [file, fileStream] = await Promise.all([
getFileById({ bucketName, fileId }),
getDownloadStream({ bucketName, fileId })
]);
if (!file) {
return Promise.reject(CommonErrEnum.fileNotFound);
}
const extension = parseFileExtensionFromUrl(file?.filename);
const start = Date.now();
const fileBuffers = await gridFsStream2Buffer(fileStream);
addLog.debug('get file buffer', { time: Date.now() - start });
const encoding = file?.metadata?.encoding || detectFileEncoding(fileBuffers);
// Get raw text
const { rawText } = await readRawContentByFileBuffer({
customPdfParse,
usageId,
getFormatText,
extension,
teamId,
tmbId,
buffer: fileBuffers,
encoding,
metadata: {
relatedId: fileId
}
});
// Add buffer
addRawTextBuffer({
sourceId: bufferId,
sourceName: file.filename,
text: rawText,
expiredTime: addMinutes(new Date(), 20)
});
return {
rawText,
filename: file.filename
};
};
packages/service/common/file/image/controller.ts
......@@ -10,6 +10,7 @@ import { retryFn } from '@fastgpt/global/common/system/utils';
import { UserError } from '@fastgpt/global/common/error/utils';
import { S3Sources } from '../../s3/type';
import { getS3AvatarSource } from '../../s3/sources/avatar';
import { isS3ObjectKey } from '../../s3/utils';
import path from 'path';
import { getNanoid } from '@fastgpt/global/common/string/tools';
......@@ -64,23 +65,28 @@ export async function uploadMongoImg({
export const copyAvatarImage = async ({
teamId,
imageUrl,
ttl,
temporary,
session
}: {
teamId: string;
imageUrl: string;
ttl: boolean;
temporary: boolean;
session?: ClientSession;
}) => {
if (!imageUrl) return;
// S3
if (imageUrl.startsWith(`${imageBaseUrl}/${S3Sources.avatar}`)) {
const extendName = path.extname(imageUrl);
const avatarSource = getS3AvatarSource();
if (isS3ObjectKey(imageUrl?.slice(avatarSource.prefix.length), 'avatar')) {
const filename = (() => {
const last = imageUrl.split('/').pop()?.split('-')[1];
if (!last) return getNanoid(6).concat(path.extname(imageUrl));
return `${getNanoid(6)}-${last}`;
})();
const key = await getS3AvatarSource().copyAvatar({
sourceKey: imageUrl.slice(imageBaseUrl.length),
targetKey: `${S3Sources.avatar}/${teamId}/${getNanoid(6)}${extendName}`,
ttl
key: imageUrl,
teamId,
filename,
temporary
});
return key;
}
......@@ -130,9 +136,13 @@ export const removeImageByPath = (path?: string, session?: ClientSession) => {
if (!name) return;
const id = name.split('.')[0];
if (!id || !Types.ObjectId.isValid(id)) return;
if (!id) return;
return MongoImage.deleteOne({ _id: id }, { session });
if (Types.ObjectId.isValid(id)) {
return MongoImage.deleteOne({ _id: id }, { session });
} else if (isS3ObjectKey(path?.slice(getS3AvatarSource().prefix.length), 'avatar')) {
return getS3AvatarSource().deleteAvatar(path, session);
}
};
export async function readMongoImg({ id }: { id: string }) {
......
packages/service/common/file/read/utils.ts
import { uploadMongoImg } from '../image/controller';
import FormData from 'form-data';
import fs from 'fs';
import type { ReadFileResponse } from '../../../worker/readFile/type';
......@@ -9,6 +8,8 @@ import { matchMdImg } from '@fastgpt/global/common/string/markdown';
import { createPdfParseUsage } from '../../../support/wallet/usage/controller';
import { useDoc2xServer } from '../../../thirdProvider/doc2x';
import { readRawContentFromBuffer } from '../../../worker/function';
import { uploadImage2S3Bucket } from '../../s3/utils';
import { Mimes } from '../../s3/constants';
export type readRawTextByLocalFileParams = {
teamId: string;
......@@ -17,6 +18,7 @@ export type readRawTextByLocalFileParams = {
encoding: string;
customPdfParse?: boolean;
getFormatText?: boolean;
fileParsedPrefix?: string;
metadata?: Record<string, any>;
};
export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParams) => {
......@@ -26,7 +28,7 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam
const buffer = await fs.promises.readFile(path);
return readRawContentByFileBuffer({
return readS3FileContentByBuffer({
extension,
customPdfParse: params.customPdfParse,
getFormatText: params.getFormatText,
......@@ -34,21 +36,25 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam
tmbId: params.tmbId,
encoding: params.encoding,
buffer,
metadata: params.metadata
imageKeyOptions: params.fileParsedPrefix
? {
prefix: params.fileParsedPrefix
}
: undefined
});
};
export const readRawContentByFileBuffer = async ({
export const readS3FileContentByBuffer = async ({
teamId,
tmbId,
extension,
buffer,
encoding,
metadata,
customPdfParse = false,
usageId,
getFormatText = true
getFormatText = true,
imageKeyOptions
}: {
teamId: string;
tmbId: string;
......@@ -56,11 +62,14 @@ export const readRawContentByFileBuffer = async ({
extension: string;
buffer: Buffer;
encoding: string;
metadata?: Record<string, any>;
customPdfParse?: boolean;
usageId?: string;
getFormatText?: boolean;
imageKeyOptions?: {
prefix: string;
expiredTime?: Date;
};
}): Promise<{
rawText: string;
}> => {
......@@ -158,31 +167,36 @@ export const readRawContentByFileBuffer = async ({
addLog.debug(`Parse file success, time: ${Date.now() - start}ms. `);
// markdown data format
if (imageList) {
if (imageList && imageList.length > 0) {
addLog.debug(`Processing ${imageList.length} images from parsed document`);
await batchRun(imageList, async (item) => {
const src = await (async () => {
if (!imageKeyOptions) return '';
try {
return await uploadMongoImg({
const { prefix, expiredTime } = imageKeyOptions;
const ext = `.${item.mime.split('/')[1].replace('x-', '')}`;
return await uploadImage2S3Bucket('private', {
base64Img: `data:${item.mime};base64,${item.base64}`,
teamId,
metadata: {
...metadata,
mime: item.mime
}
uploadKey: `${prefix}/${item.uuid}${ext}`,
mimetype: Mimes[ext as keyof typeof Mimes],
filename: `${item.uuid}${ext}`,
expiredTime
});
} catch (error) {
addLog.warn('Upload file image error', { error });
return 'Upload load image error';
return `[Image Upload Failed: ${item.uuid}]`;
}
})();
rawText = rawText.replace(item.uuid, src);
// rawText = rawText.replace(item.uuid, jwtSignS3ObjectKey(src, addDays(new Date(), 90)));
if (formatText) {
formatText = formatText.replace(item.uuid, src);
}
});
}
addLog.debug(`Upload file success, time: ${Date.now() - start}ms`);
return { rawText: getFormatText ? formatText || rawText : rawText };
return {
rawText: getFormatText ? formatText || rawText : rawText
};
};
packages/service/common/s3/buckets/base.ts
import { Client, type RemoveOptions, type CopyConditions, InvalidObjectNameError } from 'minio';
import { Client, type RemoveOptions, type CopyConditions, S3Error } from 'minio';
import {
type CreatePostPresignedUrlOptions,
type CreatePostPresignedUrlParams,
......@@ -11,9 +11,10 @@ import { defaultS3Options, getSystemMaxFileSize, Mimes } from '../constants';
import path from 'node:path';
import { MongoS3TTL } from '../schema';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { addHours } from 'date-fns';
import { addHours, addMinutes } from 'date-fns';
import { addLog } from '../../system/log';
import { addS3DelJob } from '../mq';
import { type Readable } from 'node:stream';
export class S3BaseBucket {
private _client: Client;
......@@ -73,31 +74,48 @@ export class S3BaseBucket {
return this._externalClient ?? this._client;
}
move(src: string, dst: string, options?: CopyConditions): Promise<void> {
const bucket = this.name;
this.client.copyObject(bucket, dst, `/${bucket}/${src}`, options);
return this.delete(src);
// TODO: 加到 MQ 里保障幂等
async move({
from,
to,
options
}: {
from: string;
to: string;
options?: CopyConditions;
}): Promise<void> {
await this.copy({
from,
to,
options: {
copyConditions: options,
temporary: false
}
});
await this.delete(from);
}
async copy({
src,
dst,
ttl,
from,
to,
options
}: {
src: string;
dst: string;
ttl: boolean;
options?: CopyConditions;
from: string;
to: string;
options?: {
temporary?: boolean;
copyConditions?: CopyConditions;
};
}): ReturnType<Client['copyObject']> {
if (ttl) {
const bucket = this.name;
if (options?.temporary) {
await MongoS3TTL.create({
minioKey: dst,
minioKey: to,
bucketName: this.name,
expiredTime: addHours(new Date(), 24)
});
}
return this.client.copyObject(this.name, src, dst, options);
return this.client.copyObject(bucket, to, `${bucket}/${from}`, options?.copyConditions);
}
exist(): Promise<boolean> {
......@@ -107,26 +125,53 @@ export class S3BaseBucket {
async delete(objectKey: string, options?: RemoveOptions): Promise<void> {
try {
if (!objectKey) return Promise.resolve();
// 把连带的 parsed 数据一起删除
const fileParsedPrefix = `${path.dirname(objectKey)}/${path.basename(objectKey, path.extname(objectKey))}-parsed`;
await this.addDeleteJob({ prefix: fileParsedPrefix });
return await this.client.removeObject(this.name, objectKey, options);
} catch (error) {
if (error instanceof InvalidObjectNameError) {
addLog.warn(`${this.name} delete object not found: ${objectKey}`, error);
return Promise.resolve();
if (error instanceof S3Error) {
if (error.code === 'InvalidObjectName') {
addLog.warn(`${this.name} delete object not found: ${objectKey}`, error);
return Promise.resolve();
}
}
return Promise.reject(error);
}
}
addDeleteJob({ prefix, key }: { prefix?: string; key?: string }): Promise<void> {
return addS3DelJob({ prefix, key, bucketName: this.name });
}
listObjectsV2(
...params: Parameters<Client['listObjectsV2']> extends [string, ...infer R] ? R : never
) {
return this.client.listObjectsV2(this.name, ...params);
}
putObject(...params: Parameters<Client['putObject']> extends [string, ...infer R] ? R : never) {
return this.client.putObject(this.name, ...params);
}
getObject(...params: Parameters<Client['getObject']> extends [string, ...infer R] ? R : never) {
return this.client.getObject(this.name, ...params);
}
statObject(...params: Parameters<Client['statObject']> extends [string, ...infer R] ? R : never) {
return this.client.statObject(this.name, ...params);
}
async fileStreamToBuffer(stream: Readable): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(chunk);
}
return Buffer.concat(chunks);
}
addDeleteJob(params: Omit<Parameters<typeof addS3DelJob>[0], 'bucketName'>) {
return addS3DelJob({ ...params, bucketName: this.name });
}
async createPostPresignedUrl(
params: CreatePostPresignedUrlParams,
options: CreatePostPresignedUrlOptions = {}
......@@ -140,8 +185,7 @@ export class S3BaseBucket {
const key = (() => {
if ('rawKey' in params) return params.rawKey;
return `${params.source}/${params.teamId}/${getNanoid(6)}-${filename}`;
return [params.source, params.teamId, `${getNanoid(6)}-${filename}`].join('/');
})();
const policy = this.externalClient.newPostPolicy();
......@@ -151,11 +195,12 @@ export class S3BaseBucket {
if (formatMaxFileSize) {
policy.setContentLengthRange(1, formatMaxFileSize);
}
policy.setExpires(new Date(Date.now() + 10 * 60 * 1000));
policy.setExpires(addMinutes(new Date(), 10));
policy.setUserMetaData({
'content-disposition': `attachment; filename="${encodeURIComponent(filename)}"`,
'origin-filename': encodeURIComponent(filename),
'upload-time': new Date().toISOString()
'upload-time': new Date().toISOString(),
...params.metadata
});
const { formData, postURL } = await this.externalClient.presignedPostPolicy(policy);
......@@ -179,20 +224,25 @@ export class S3BaseBucket {
}
}
async createExtenalUrl(params: createPreviewUrlParams) {
async createExternalUrl(params: createPreviewUrlParams) {
const parsed = CreateGetPresignedUrlParamsSchema.parse(params);
const { key, expiredHours } = parsed;
const expires = expiredHours ? expiredHours * 60 * 60 : 30 * 60; // expires 的单位是秒 默认 30 分钟
return await this.externalClient.presignedGetObject(this.name, key, expires);
return await this.externalClient.presignedGetObject(this.name, key, expires, {
'Content-Disposition': `attachment; filename="${path.basename(key)}"`
});
}
async createPreviewlUrl(params: createPreviewUrlParams) {
async createPreviewUrl(params: createPreviewUrlParams) {
const parsed = CreateGetPresignedUrlParamsSchema.parse(params);
const { key, expiredHours } = parsed;
const expires = expiredHours ? expiredHours * 60 * 60 : 30 * 60; // expires 的单位是秒 默认 30 分钟
return await this.client.presignedGetObject(this.name, key, expires);
return await this.client.presignedGetObject(this.name, key, expires, {
'Content-Disposition': `attachment; filename="${path.basename(key)}"`
});
}
}
packages/service/common/s3/controller.ts
......@@ -3,6 +3,7 @@ import { addLog } from '../system/log';
import { setCron } from '../system/cron';
import { checkTimerLock } from '../system/timerLock/utils';
import { TimerIdEnum } from '../system/timerLock/constants';
import path from 'node:path';
export async function clearExpiredMinioFiles() {
try {
......@@ -26,6 +27,25 @@ export async function clearExpiredMinioFiles() {
if (bucket) {
await bucket.delete(file.minioKey);
if (!file.minioKey.includes('-parsed/')) {
try {
const dir = path.dirname(file.minioKey);
const basename = path.basename(file.minioKey);
const ext = path.extname(basename);
if (ext) {
const nameWithoutExt = path.basename(basename, ext);
const parsedPrefix = `${dir}/${nameWithoutExt}-parsed`;
await bucket.addDeleteJob({ prefix: parsedPrefix });
addLog.info(`Scheduled deletion of parsed images: ${parsedPrefix}`);
}
} catch (error) {
addLog.debug(`Failed to schedule parsed images deletion for ${file.minioKey}`);
}
}
await MongoS3TTL.deleteOne({ _id: file._id });
success++;
......
packages/service/common/s3/mq.ts
......@@ -4,6 +4,7 @@ import { retryFn } from '@fastgpt/global/common/system/utils';
export type S3MQJobData = {
key?: string;
keys?: string[];
prefix?: string;
bucketName: string;
};
......@@ -29,9 +30,8 @@ export const startS3DelWorker = async () => {
return getWorker<S3MQJobData>(
QueueNames.s3FileDelete,
async (job) => {
const { prefix, bucketName, key } = job.data;
const { prefix, bucketName, key, keys } = job.data;
const limit = pLimit(10);
const tasks: Promise<void>[] = [];
const bucket = s3BucketMap[bucketName];
if (!bucket) {
return Promise.reject(`Bucket not found: ${bucketName}`);
......@@ -40,13 +40,25 @@ export const startS3DelWorker = async () => {
if (key) {
await bucket.delete(key);
}
if (keys) {
const tasks: Promise<void>[] = [];
for (const key of keys) {
const p = limit(() => retryFn(() => bucket.delete(key)));
tasks.push(p);
}
await Promise.all(tasks);
}
if (prefix) {
const tasks: Promise<void>[] = [];
return new Promise<void>(async (resolve, reject) => {
const stream = bucket.listObjectsV2(prefix, true);
stream.on('data', async (file) => {
if (!file.name) return;
const p = limit(() => retryFn(() => bucket.delete(file.name)));
const p = limit(() =>
// 因为封装的 delete 方法里，包含前缀删除，这里不能再使用，避免循环。
retryFn(() => bucket.client.removeObject(bucket.name, file.name))
);
tasks.push(p);
});
......
packages/service/common/s3/sources/avatar.ts
......@@ -68,20 +68,20 @@ class S3AvatarSource {
}
async copyAvatar({
sourceKey,
targetKey,
ttl
key,
teamId,
filename,
temporary = false
}: {
sourceKey: string;
targetKey: string;
ttl: boolean;
key: string;
teamId: string;
filename: string;
temporary: boolean;
}) {
await this.bucket.copy({
src: sourceKey,
dst: targetKey,
ttl
});
return targetKey;
const from = key.slice(this.prefix.length);
const to = `${S3Sources.avatar}/${teamId}/${filename}`;
await this.bucket.copy({ from, to, options: { temporary } });
return this.prefix.concat(to);
}
}
......
packages/service/common/s3/sources/chat/index.ts
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
import { S3PrivateBucket } from '../../buckets/private';
import { S3Sources } from '../../type';
import {
......@@ -7,10 +7,12 @@ import {
ChatFileUploadSchema,
DelChatFileByPrefixSchema
} from './type';
import { MongoS3TTL } from '../../schema';
import { addHours } from 'date-fns';
import { differenceInHours } from 'date-fns';
import { S3Buckets } from '../../constants';
import path from 'path';
import { getFileS3Key } from '../../utils';
class S3ChatSource {
export class S3ChatSource {
private bucket: S3PrivateBucket;
private static instance: S3ChatSource;
......@@ -22,28 +24,79 @@ class S3ChatSource {
return (this.instance ??= new S3ChatSource());
}
static isChatFileKey(key?: string): key is `${typeof S3Sources.chat}/${string}` {
return key?.startsWith(`${S3Sources.chat}/`) ?? false;
static parseChatUrl(url: string | URL) {
try {
const parseUrl = new URL(url);
const pathname = decodeURIComponent(parseUrl.pathname);
// 非 S3 key
if (!pathname.startsWith(`/${S3Buckets.private}/${S3Sources.chat}/`)) {
return {
filename: '',
extension: '',
imageParsePrefix: ''
};
}
const filename = pathname.split('/').pop() || 'file';
const extension = path.extname(filename);
return {
filename,
extension: extension.replace('.', ''),
imageParsePrefix: `${pathname.replace(`/${S3Buckets.private}/`, '').replace(extension, '')}-parsed`
};
} catch (error) {
return {
filename: '',
extension: '',
imageParsePrefix: ''
};
}
}
// 获取文件流
getChatFileStream(key: string) {
return this.bucket.getObject(key);
}
// 获取文件状态
getChatFileStat(key: string) {
return this.bucket.statObject(key);
}
// 获取文件元数据
async getFileMetadata(key: string) {
const stat = await this.getChatFileStat(key);
if (!stat) return { filename: '', extension: '', contentLength: 0, contentType: '' };
const contentLength = stat.size;
const filename: string = decodeURIComponent(stat.metaData['origin-filename']);
const extension = parseFileExtensionFromUrl(filename);
const contentType: string = stat.metaData['content-type'];
return {
filename,
extension,
contentType,
contentLength
};
}
async createGetChatFileURL(params: { key: string; expiredHours?: number; external: boolean }) {
const { key, expiredHours = 1, external = false } = params; // 默认一个小时
if (external) {
return await this.bucket.createExtenalUrl({ key, expiredHours });
return await this.bucket.createExternalUrl({ key, expiredHours });
}
return await this.bucket.createPreviewlUrl({ key, expiredHours });
return await this.bucket.createPreviewUrl({ key, expiredHours });
}
async createUploadChatFileURL(params: CheckChatFileKeys) {
const { appId, chatId, uId, filename } = ChatFileUploadSchema.parse(params);
const rawKey = [S3Sources.chat, appId, uId, chatId, `${getNanoid(6)}-${filename}`].join('/');
await MongoS3TTL.create({
minioKey: rawKey,
bucketName: this.bucket.name,
expiredTime: addHours(new Date(), 24)
});
return await this.bucket.createPostPresignedUrl({ rawKey, filename });
const { appId, chatId, uId, filename, expiredTime } = ChatFileUploadSchema.parse(params);
const { fileKey } = getFileS3Key.chat({ appId, chatId, uId, filename });
return await this.bucket.createPostPresignedUrl(
{ rawKey: fileKey, filename },
{ expiredHours: expiredTime ? differenceInHours(new Date(), expiredTime) : 24 }
);
}
deleteChatFilesByPrefix(params: DelChatFileByPrefixParams) {
......
packages/service/common/s3/sources/chat/type.ts
......@@ -5,7 +5,8 @@ export const ChatFileUploadSchema = z.object({
appId: ObjectIdSchema,
chatId: z.string().nonempty(),
uId: z.string().nonempty(),
filename: z.string().nonempty()
filename: z.string().nonempty(),
expiredTime: z.date().optional()
});
export type CheckChatFileKeys = z.infer<typeof ChatFileUploadSchema>;
......
packages/service/common/s3/sources/dataset/index.ts 0 → 100644
import { S3Sources } from '../../type';
import { S3PrivateBucket } from '../../buckets/private';
import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
import {
type CreateGetDatasetFileURLParams,
CreateGetDatasetFileURLParamsSchema,
type CreateUploadDatasetFileParams,
CreateUploadDatasetFileParamsSchema,
type DeleteDatasetFilesByPrefixParams,
DeleteDatasetFilesByPrefixParamsSchema,
type GetDatasetFileContentParams,
GetDatasetFileContentParamsSchema,
type UploadDatasetFileByBufferParams,
UploadDatasetFileByBufferParamsSchema
} from './type';
import { MongoS3TTL } from '../../schema';
import { addHours, addMinutes } from 'date-fns';
import { addLog } from '../../../system/log';
import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
import { readS3FileContentByBuffer } from '../../../file/read/utils';
import { addRawTextBuffer, getRawTextBuffer } from '../../../buffer/rawText/controller';
import path from 'node:path';
import { Mimes } from '../../constants';
import { getFileS3Key, truncateFilename } from '../../utils';
export class S3DatasetSource {
public bucket: S3PrivateBucket;
private static instance: S3DatasetSource;
constructor() {
this.bucket = new S3PrivateBucket();
}
static getInstance() {
return (this.instance ??= new S3DatasetSource());
}
// 下载链接
async createGetDatasetFileURL(params: CreateGetDatasetFileURLParams) {
const { key, expiredHours, external } = CreateGetDatasetFileURLParamsSchema.parse(params);
if (external) {
return await this.bucket.createExternalUrl({ key, expiredHours });
}
return await this.bucket.createPreviewUrl({ key, expiredHours });
}
// 上传链接
async createUploadDatasetFileURL(params: CreateUploadDatasetFileParams) {
const { filename, datasetId } = CreateUploadDatasetFileParamsSchema.parse(params);
const { fileKey } = getFileS3Key.dataset({ datasetId, filename });
return await this.bucket.createPostPresignedUrl(
{ rawKey: fileKey, filename },
{ expiredHours: 3 }
);
}
/**
* 可以根据 datasetId 或者 prefix 删除文件
* 如果存在 rawPrefix 则优先使用 rawPrefix 去删除文件，否则使用 datasetId 拼接前缀去删除文件
* 比如根据被解析的文档前缀去删除解析出来的图片
**/
deleteDatasetFilesByPrefix(params: DeleteDatasetFilesByPrefixParams) {
const { datasetId, rawPrefix } = DeleteDatasetFilesByPrefixParamsSchema.parse(params);
const prefix = rawPrefix || [S3Sources.dataset, datasetId].filter(Boolean).join('/');
return this.bucket.addDeleteJob({ prefix });
}
// 单个键删除
deleteDatasetFileByKey(key?: string) {
return this.bucket.addDeleteJob({ key });
}
// 多个键删除
deleteDatasetFilesByKeys(keys: string[]) {
return this.bucket.addDeleteJob({ keys });
}
// 获取文件流
getDatasetFileStream(key: string) {
return this.bucket.getObject(key);
}
// 获取文件状态
getDatasetFileStat(key: string) {
return this.bucket.statObject(key);
}
// 获取文件元数据
async getFileMetadata(key: string) {
const stat = await this.getDatasetFileStat(key);
if (!stat) return { filename: '', extension: '', contentLength: 0, contentType: '' };
const contentLength = stat.size;
const filename: string = decodeURIComponent(stat.metaData['origin-filename']);
const extension = parseFileExtensionFromUrl(filename);
const contentType: string = stat.metaData['content-type'];
return {
filename,
extension,
contentType,
contentLength
};
}
async getDatasetBase64Image(key: string): Promise<string> {
const [stream, metadata] = await Promise.all([
this.getDatasetFileStream(key),
this.getFileMetadata(key)
]);
const buffer = await this.bucket.fileStreamToBuffer(stream);
const base64 = buffer.toString('base64');
return `data:${metadata.contentType || 'image/jpeg'};base64,${base64}`;
}
async getDatasetFileRawText(params: GetDatasetFileContentParams) {
const { fileId, teamId, tmbId, customPdfParse, getFormatText, usageId } =
GetDatasetFileContentParamsSchema.parse(params);
const bufferId = `${fileId}-${customPdfParse}`;
const fileBuffer = await getRawTextBuffer(bufferId);
if (fileBuffer) {
return {
rawText: fileBuffer.text,
filename: fileBuffer.sourceName
};
}
const [metadata, stream] = await Promise.all([
this.getFileMetadata(fileId),
this.getDatasetFileStream(fileId)
]);
const extension = metadata.extension;
const filename: string = decodeURIComponent(metadata.filename);
const start = Date.now();
const buffer = await this.bucket.fileStreamToBuffer(stream);
addLog.debug('get dataset file buffer', { time: Date.now() - start });
const encoding = detectFileEncoding(buffer);
const { fileParsedPrefix } = getFileS3Key.s3Key(fileId);
const { rawText } = await readS3FileContentByBuffer({
teamId,
tmbId,
extension,
buffer,
encoding,
customPdfParse,
usageId,
getFormatText,
imageKeyOptions: {
prefix: fileParsedPrefix
}
});
addRawTextBuffer({
sourceId: bufferId,
sourceName: filename,
text: rawText,
expiredTime: addMinutes(new Date(), 20)
});
return {
rawText,
filename
};
}
// 根据文件 Buffer 上传文件
async uploadDatasetFileByBuffer(params: UploadDatasetFileByBufferParams): Promise<string> {
const { datasetId, buffer, filename } = UploadDatasetFileByBufferParamsSchema.parse(params);
// 截断文件名以避免S3 key过长的问题
const truncatedFilename = truncateFilename(filename);
const { fileKey: key } = getFileS3Key.dataset({ datasetId, filename: truncatedFilename });
await this.bucket.putObject(key, buffer, buffer.length, {
'content-type': Mimes[path.extname(truncatedFilename) as keyof typeof Mimes],
'upload-time': new Date().toISOString(),
'origin-filename': encodeURIComponent(filename) // 保持原始文件名在元数据中
});
await MongoS3TTL.create({
minioKey: key,
bucketName: this.bucket.name,
expiredTime: addHours(new Date(), 3)
});
return key;
}
}
export function getS3DatasetSource() {
return S3DatasetSource.getInstance();
}
packages/service/common/s3/sources/dataset/type.ts 0 → 100644
import { ObjectIdSchema } from '@fastgpt/global/common/type/mongo';
import { z } from 'zod';
export const CreateUploadDatasetFileParamsSchema = z.object({
filename: z.string().nonempty(),
datasetId: ObjectIdSchema
});
export type CreateUploadDatasetFileParams = z.infer<typeof CreateUploadDatasetFileParamsSchema>;
export const CreateGetDatasetFileURLParamsSchema = z.object({
key: z.string().nonempty(),
expiredHours: z.number().positive().optional(),
external: z.boolean().optional()
});
export type CreateGetDatasetFileURLParams = z.infer<typeof CreateGetDatasetFileURLParamsSchema>;
export const DeleteDatasetFilesByPrefixParamsSchema = z.object({
datasetId: ObjectIdSchema.optional(),
rawPrefix: z.string().nonempty().optional()
});
export type DeleteDatasetFilesByPrefixParams = z.infer<
typeof DeleteDatasetFilesByPrefixParamsSchema
>;
export const GetDatasetFileContentParamsSchema = z.object({
teamId: ObjectIdSchema,
tmbId: ObjectIdSchema,
fileId: z.string().nonempty(), // 这是 ObjectKey
customPdfParse: z.boolean().optional(),
getFormatText: z.boolean().optional(), // 数据类型都尽可能转化成 markdown 格式
datasetId: ObjectIdSchema,
usageId: ObjectIdSchema.optional()
});
export type GetDatasetFileContentParams = z.infer<typeof GetDatasetFileContentParamsSchema>;
export const UploadParsedDatasetImagesParamsSchema = z.object({
key: z.string().nonempty()
});
export type UploadParsedDatasetImagesParams = z.infer<typeof UploadParsedDatasetImagesParamsSchema>;
export const ParsedFileContentS3KeyParamsSchema = z.object({
datasetId: ObjectIdSchema,
filename: z.string()
});
export type ParsedFileContentS3KeyParams = z.infer<typeof ParsedFileContentS3KeyParamsSchema>;
export const UploadDatasetFileByBufferParamsSchema = z.object({
datasetId: ObjectIdSchema,
buffer: z.instanceof(Buffer),
filename: z.string().nonempty()
});
export type UploadDatasetFileByBufferParams = z.infer<typeof UploadDatasetFileByBufferParamsSchema>;
packages/service/common/s3/type.ts
......@@ -17,7 +17,7 @@ export type ExtensionType = keyof typeof Mimes;
export type S3OptionsType = typeof defaultS3Options;
export const S3SourcesSchema = z.enum(['avatar', 'chat']);
export const S3SourcesSchema = z.enum(['avatar', 'chat', 'dataset', 'temp']);
export const S3Sources = S3SourcesSchema.enum;
export type S3SourceType = z.infer<typeof S3SourcesSchema>;
......@@ -25,13 +25,15 @@ export const CreatePostPresignedUrlParamsSchema = z.union([
// Option 1: Only rawKey
z.object({
filename: z.string().min(1),
rawKey: z.string().min(1)
rawKey: z.string().min(1),
metadata: z.record(z.string(), z.string()).optional()
}),
// Option 2: filename with optional source and teamId
z.object({
filename: z.string().min(1),
source: S3SourcesSchema.optional(),
teamId: z.string().length(16).optional()
teamId: z.string().length(16).optional(),
metadata: z.record(z.string(), z.string()).optional()
})
]);
export type CreatePostPresignedUrlParams = z.infer<typeof CreatePostPresignedUrlParamsSchema>;
......@@ -55,6 +57,15 @@ export const CreateGetPresignedUrlParamsSchema = z.object({
});
export type createPreviewUrlParams = z.infer<typeof CreateGetPresignedUrlParamsSchema>;
export const UploadImage2S3BucketParamsSchema = z.object({
base64Img: z.string().nonempty(),
uploadKey: z.string().nonempty(),
mimetype: z.string().nonempty(),
filename: z.string().nonempty(),
expiredTime: z.date().optional()
});
export type UploadImage2S3BucketParams = z.infer<typeof UploadImage2S3BucketParamsSchema>;
declare global {
var s3BucketMap: {
[key: string]: S3BaseBucket;
......
packages/service/common/s3/utils.ts 0 → 100644
import jwt from 'jsonwebtoken';
import { isAfter, differenceInSeconds } from 'date-fns';
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import type { ClientSession } from 'mongoose';
import { MongoS3TTL } from './schema';
import { S3Buckets } from './constants';
import { S3PrivateBucket } from './buckets/private';
import { S3Sources, type UploadImage2S3BucketParams } from './type';
import { S3PublicBucket } from './buckets/public';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import path from 'node:path';
import type { ParsedFileContentS3KeyParams } from './sources/dataset/type';
import { EndpointUrl } from '@fastgpt/global/common/file/constants';
// S3文件名最大长度配置
export const S3_FILENAME_MAX_LENGTH = 50;
/**
* 截断文件名，确保不超过最大长度，同时保留扩展名
* @param filename 原始文件名
* @param maxLength 最大长度限制
* @returns 截断后的文件名
*/
export function truncateFilename(
filename: string,
maxLength: number = S3_FILENAME_MAX_LENGTH
): string {
if (!filename) return filename;
// 如果文件名长度已经符合要求，直接返回
if (filename.length <= maxLength) {
return filename;
}
const extension = path.extname(filename); // 包含点的扩展名，如 ".pdf"
const nameWithoutExt = path.basename(filename, extension); // 不包含扩展名的文件名
// 计算名称部分的最大长度（总长度减去扩展名长度）
const maxNameLength = maxLength - extension.length;
// 如果扩展名本身就很长导致没有空间放名称，则截断扩展名
if (maxNameLength <= 0) {
// 保留扩展名的开头部分，至少保留一个点
const truncatedExt = extension.substring(0, Math.min(maxLength, extension.length));
return truncatedExt;
}
// 截断文件名部分
const truncatedName = nameWithoutExt.substring(0, maxNameLength);
return truncatedName + extension;
}
/**
*
* @param objectKey
* @param expiredTime
* @returns
*/
export function jwtSignS3ObjectKey(objectKey: string, expiredTime: Date) {
const secret = process.env.FILE_TOKEN_KEY as string;
const expiresIn = differenceInSeconds(expiredTime, new Date());
const token = jwt.sign({ objectKey }, secret, { expiresIn });
return `${EndpointUrl}/api/system/file/${token}`;
}
export function jwtVerifyS3ObjectKey(token: string) {
const secret = process.env.FILE_TOKEN_KEY as string;
return new Promise<{ objectKey: string }>((resolve, reject) => {
jwt.verify(token, secret, (err, payload) => {
if (err || !payload || !(payload as jwt.JwtPayload).objectKey) {
reject(ERROR_ENUM.unAuthFile);
}
resolve(payload as { objectKey: string });
});
});
}
export function removeS3TTL({
key,
bucketName,
session
}: {
key: string[] | string;
bucketName: keyof typeof S3Buckets;
session?: ClientSession;
}) {
if (!key) return;
if (Array.isArray(key)) {
return MongoS3TTL.deleteMany(
{
minioKey: { $in: key },
bucketName: S3Buckets[bucketName]
},
{ session }
);
}
if (typeof key === 'string') {
return MongoS3TTL.deleteOne(
{
minioKey: key,
bucketName: S3Buckets[bucketName]
},
{ session }
);
}
}
export async function uploadImage2S3Bucket(
bucketName: keyof typeof S3Buckets,
params: UploadImage2S3BucketParams
) {
const { base64Img, filename, mimetype, uploadKey, expiredTime } = params;
const bucket = bucketName === 'private' ? new S3PrivateBucket() : new S3PublicBucket();
const base64Data = base64Img.split(',')[1] || base64Img;
const buffer = Buffer.from(base64Data, 'base64');
await bucket.putObject(uploadKey, buffer, buffer.length, {
'content-type': mimetype,
'upload-time': new Date().toISOString(),
'origin-filename': encodeURIComponent(filename)
});
const now = new Date();
if (expiredTime && isAfter(expiredTime, now)) {
await MongoS3TTL.create({
minioKey: uploadKey,
bucketName: bucket.name,
expiredTime: expiredTime
});
}
return uploadKey;
}
const getFormatedFilename = (filename?: string) => {
if (!filename) {
return {
formatedFilename: getNanoid(12),
extension: ''
};
}
const id = getNanoid(6);
// 先截断文件名，再进行格式化
const truncatedFilename = truncateFilename(filename);
const extension = path.extname(truncatedFilename); // 带.
const name = path.basename(truncatedFilename, extension);
return {
formatedFilename: `${id}-${name}`,
extension: extension.replace('.', '')
};
};
export const getFileS3Key = {
// 临时的文件路径（比如 evaluation)
temp: ({ teamId, filename }: { teamId: string; filename?: string }) => {
const { formatedFilename, extension } = getFormatedFilename(filename);
return {
fileKey: [
S3Sources.temp,
teamId,
`${formatedFilename}${extension ? `.${extension}` : ''}`
].join('/'),
fileParsedPrefix: [S3Sources.temp, teamId, `${formatedFilename}-parsed`].join('/')
};
},
// 对话中上传的文件的解析结果的图片的 Key
chat: ({
appId,
chatId,
uId,
filename
}: {
chatId: string;
uId: string;
appId: string;
filename: string;
}) => {
const { formatedFilename, extension } = getFormatedFilename(filename);
const basePrefix = [S3Sources.chat, appId, uId, chatId].filter(Boolean).join('/');
return {
fileKey: [basePrefix, `${formatedFilename}${extension ? `.${extension}` : ''}`].join('/'),
fileParsedPrefix: [basePrefix, `${formatedFilename}-parsed`].join('/')
};
},
// 上传数据集的文件的解析结果的图片的 Key
dataset: (params: ParsedFileContentS3KeyParams) => {
const { datasetId, filename } = params;
const { formatedFilename, extension } = getFormatedFilename(filename);
return {
fileKey: [
S3Sources.dataset,
datasetId,
`${formatedFilename}${extension ? `.${extension}` : ''}`
].join('/'),
fileParsedPrefix: [S3Sources.dataset, datasetId, `${formatedFilename}-parsed`].join('/')
};
},
s3Key: (key: string) => {
const prefix = `${path.dirname(key)}/${path.basename(key, path.extname(key))}-parsed`;
return {
fileKey: key,
fileParsedPrefix: prefix
};
}
};
/**
* Check if a key is a valid S3 object key
* @param key - The key to check
* @param source - The source of the key
* @returns True if the key is a valid S3 object key
*/
export function isS3ObjectKey<T extends keyof typeof S3Sources>(
key: string | undefined | null,
source: T
): key is `${T}/${string}` {
return typeof key === 'string' && key.startsWith(`${S3Sources[source]}/`);
}
packages/service/core/ai/llm/request.ts
......@@ -86,7 +86,7 @@ export const createLLMResponse = async <T extends CompletionsBodyType>(
messages: rewriteMessages
});
// console.dir(requestBody, { depth: null });
// console.log(JSON.stringify(requestBody, null, 2));
const { response, isStreamResponse, getEmptyResponseTip } = await createChatCompletion({
body: requestBody,
userKey,
......
packages/service/core/ai/llm/utils.ts
......@@ -127,7 +127,7 @@ export const loadRequestMessages = async ({
const result: ChatCompletionContentPart[] = [];
// 提取所有HTTPS图片URL并添加到result开头
const httpsImages = [...new Set(Array.from(input.matchAll(imageRegex), (m) => m[0]))];
const httpsImages = Array.from(new Set(input.matchAll(imageRegex)), (m) => m[0]);
httpsImages.forEach((url) => {
result.push({
type: 'image_url',
......@@ -379,7 +379,10 @@ export const loadRequestMessages = async ({
return {
...item,
content: formatContent
content:
typeof formatContent === 'string'
? formatContent
: (formatContent as ChatCompletionContentPartText[])
};
} else if (item.role === ChatCompletionRequestMessageRoleEnum.Assistant) {
if (item.tool_calls || item.function_call) {
......
packages/service/core/chat/saveChat.ts
......@@ -8,17 +8,23 @@ import { addLog } from '../../common/system/log';
import { mongoSessionRun } from '../../common/mongo/sessionRun';
import { type StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
import { getAppChatConfig, getGuideModule } from '@fastgpt/global/core/workflow/utils';
import { type AppChatConfigType } from '@fastgpt/global/core/app/type';
import { type AppChatConfigType, type VariableItemType } from '@fastgpt/global/core/app/type';
import { mergeChatResponseData } from '@fastgpt/global/core/chat/utils';
import { pushChatLog } from './pushChatLog';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import {
FlowNodeTypeEnum,
FlowNodeInputTypeEnum
} from '@fastgpt/global/core/workflow/node/constant';
import { extractDeepestInteractive } from '@fastgpt/global/core/workflow/runtime/utils';
import { MongoAppChatLog } from '../app/logs/chatLogsSchema';
import { writePrimary } from '../../common/mongo/utils';
import { MongoChatItemResponse } from './chatItemResponseSchema';
import { chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
import { MongoS3TTL } from '../../common/s3/schema';
import type { ClientSession } from '../../common/mongo';
import { removeS3TTL } from '../../common/s3/utils';
import { VariableInputEnum } from '@fastgpt/global/core/workflow/constants';
import { encryptSecretValue, anyValueDecrypt } from '../../common/secret/utils';
import type { SecretValueType } from '@fastgpt/global/common/secret/type';
type Props = {
chatId: string;
......@@ -51,27 +57,86 @@ const beforProcess = (props: Props) => {
};
const afterProcess = async ({
contents,
variables,
variableList,
session
}: {
contents: (UserChatItemType | AIChatItemType)[];
variables?: Record<string, any>;
variableList?: VariableItemType[];
session: ClientSession;
}) => {
// Remove ttl
const fileKeys = contents
const contentFileKeys = contents
.map((item) => {
if (item.value && Array.isArray(item.value)) {
return item.value.map((valueItem) => {
return item.value.flatMap((valueItem) => {
const keys: string[] = [];
// 1. chat file
if (valueItem.type === ChatItemValueTypeEnum.file && valueItem.file?.key) {
return valueItem.file.key;
keys.push(valueItem.file.key);
}
// 2. plugin input
if (valueItem.type === 'text' && valueItem.text?.content) {
try {
const parsed = JSON.parse(valueItem.text.content);
// 2.1 plugin input - 数组格式
if (Array.isArray(parsed)) {
parsed.forEach((field) => {
if (field.value && Array.isArray(field.value)) {
field.value.forEach((file: { key: string }) => {
if (file.key && typeof file.key === 'string') {
keys.push(file.key);
}
});
}
});
}
// 2.2 form input - 对象格式 { "字段名": [{ key, url, ... }] }
else if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
Object.values(parsed).forEach((fieldValue) => {
if (Array.isArray(fieldValue)) {
fieldValue.forEach((file: any) => {
if (
file &&
typeof file === 'object' &&
file.key &&
typeof file.key === 'string'
) {
keys.push(file.key);
}
});
}
});
}
} catch (err) {}
}
return keys;
});
}
return [];
})
.flat()
.filter(Boolean) as string[];
if (fileKeys.length > 0) {
await MongoS3TTL.deleteMany({ minioKey: { $in: fileKeys } }, { session });
const variableFileKeys: string[] = [];
if (variables && variableList) {
variableList.forEach((varItem) => {
if (varItem.type === VariableInputEnum.file) {
const varValue = variables[varItem.key];
if (Array.isArray(varValue)) {
variableFileKeys.push(...varValue.map((item) => item.key));
}
}
});
}
const allFileKeys = [...new Set([...contentFileKeys, ...variableFileKeys])];
if (allFileKeys.length > 0) {
await removeS3TTL({ key: allFileKeys, bucketName: 'private', session });
}
};
......@@ -254,7 +319,12 @@ export async function saveChat(props: Props) {
}
);
await afterProcess({ contents: processedContent, session });
await afterProcess({
contents: processedContent,
variables,
variableList,
session
});
pushChatLog({
chatId,
......@@ -336,8 +406,18 @@ export async function saveChat(props: Props) {
export const updateInteractiveChat = async (props: Props) => {
beforProcess(props);
const { teamId, chatId, appId, userContent, aiContent, variables, durationSeconds, errorMsg } =
props;
const {
teamId,
chatId,
appId,
nodes,
appChatConfig,
userContent,
aiContent,
variables,
durationSeconds,
errorMsg
} = props;
if (!chatId) return;
const chatItem = await MongoChatItem.findOne({ appId, chatId, obj: ChatRoleEnum.AI }).sort({
......@@ -371,6 +451,12 @@ export const updateInteractiveChat = async (props: Props) => {
errorMsg
});
const { variables: variableList } = getAppChatConfig({
chatConfig: appChatConfig,
systemConfigNode: getGuideModule(nodes),
isPublicFetch: false
});
let finalInteractive = extractDeepestInteractive(interactiveValue.interactive);
if (finalInteractive.type === 'userSelect') {
......@@ -380,13 +466,31 @@ export const updateInteractiveChat = async (props: Props) => {
typeof parsedUserInteractiveVal === 'object'
) {
finalInteractive.params.inputForm = finalInteractive.params.inputForm.map((item) => {
const itemValue = parsedUserInteractiveVal[item.label];
return itemValue !== undefined
? {
const itemValue = parsedUserInteractiveVal[item.key];
if (itemValue === undefined) return item;
// 如果是密码类型，加密后存储
if (item.type === FlowNodeInputTypeEnum.password) {
const decryptedVal = anyValueDecrypt(itemValue);
if (typeof decryptedVal === 'string') {
return {
...item,
value: itemValue
}
: item;
value: encryptSecretValue({
value: decryptedVal,
secret: ''
} as SecretValueType)
};
}
return {
...item,
value: itemValue
};
}
return {
...item,
value: itemValue
};
});
finalInteractive.params.submitted = true;
} else if (finalInteractive.type === 'paymentPause') {
......@@ -460,7 +564,12 @@ export const updateInteractiveChat = async (props: Props) => {
);
}
await afterProcess({ contents: [userContent, aiContent], session });
await afterProcess({
contents: [userContent, aiContent],
variables,
variableList,
session
});
});
// Push chat data logs
......
packages/service/core/chat/utils.ts
......@@ -3,6 +3,9 @@ import type { ChatItemType } from '@fastgpt/global/core/chat/type';
import { getS3ChatSource } from '../../common/s3/sources/chat';
import type { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io';
import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import type { VariableItemType } from '@fastgpt/global/core/app/type';
import { VariableInputEnum } from '@fastgpt/global/core/workflow/constants';
import { clone, cloneDeep } from 'lodash';
export const addPreviewUrlToChatItems = async (
histories: ChatItemType[],
......@@ -61,3 +64,42 @@ export const addPreviewUrlToChatItems = async (
}
}
};
// Presign variables file urls
export const presignVariablesFileUrls = async ({
variables,
variableConfig
}: {
variables?: Record<string, any>;
variableConfig?: VariableItemType[];
}) => {
if (!variables || !variableConfig) return variables;
const cloneVars = cloneDeep(variables);
await Promise.all(
variableConfig.map(async (item) => {
if (item.type === VariableInputEnum.file) {
const val = cloneVars[item.key];
if (Array.isArray(val)) {
cloneVars[item.key] = await Promise.all(
val.map(async (item) => {
if (!item.key) return item;
const url = await getS3ChatSource().createGetChatFileURL({
key: item.key,
external: true
});
return {
...item,
url
};
})
).then((urls) => urls.filter(Boolean));
}
}
})
);
return cloneVars;
};
packages/service/core/dataset/apiDataset/custom/api.ts
......@@ -126,12 +126,14 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
teamId,
tmbId,
apiFileId,
customPdfParse
customPdfParse,
datasetId
}: {
teamId: string;
tmbId: string;
apiFileId: string;
customPdfParse?: boolean;
datasetId: string;
}): Promise<ApiFileReadContentResponse> => {
const data = await request<
{
......@@ -161,11 +163,12 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
};
}
const rawText = await readFileRawTextByUrl({
const { rawText } = await readFileRawTextByUrl({
teamId,
tmbId,
url: previewUrl,
relatedId: apiFileId,
datasetId,
customPdfParse,
getFormatText: true
});
......
packages/service/core/dataset/collection/controller.ts
import { DatasetCollectionDataProcessModeEnum } from '@fastgpt/global/core/dataset/constants';
import {
DatasetCollectionDataProcessModeEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { MongoDatasetCollection } from './schema';
import type {
......@@ -30,7 +33,10 @@ import {
getLLMMaxChunkSize
} from '@fastgpt/global/core/dataset/training/utils';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
import { clearCollectionImages, removeDatasetImageExpiredTime } from '../image/utils';
import { clearCollectionImages } from '../image/utils';
import { getS3DatasetSource, S3DatasetSource } from '../../../common/s3/sources/dataset';
import path from 'node:path';
import { removeS3TTL, isS3ObjectKey } from '../../../common/s3/utils';
export const createCollectionAndInsertData = async ({
dataset,
......@@ -232,13 +238,6 @@ export const createCollectionAndInsertData = async ({
}
})();
// 6. Remove images ttl index
await removeDatasetImageExpiredTime({
ids: imageIds,
collectionId,
session
});
return {
collectionId: String(collectionId),
insertResults
......@@ -300,6 +299,10 @@ export async function createOneCollection({ session, ...props }: CreateOneCollec
{ session, ordered: true }
);
if (isS3ObjectKey(fileId, 'dataset')) {
await removeS3TTL({ key: fileId, bucketName: 'private', session });
}
return collection;
}
......@@ -363,9 +366,25 @@ export async function delCollection({
if (!teamId) return Promise.reject('teamId is not exist');
const s3DatasetSource = getS3DatasetSource();
const datasetIds = Array.from(new Set(collections.map((item) => String(item.datasetId))));
const collectionIds = collections.map((item) => String(item._id));
const imageCollectionIds = collections
.filter((item) => item.type === DatasetCollectionTypeEnum.images)
.map((item) => String(item._id));
const imageDatas = await MongoDatasetData.find(
{
teamId,
datasetId: { $in: datasetIds },
collectionId: { $in: imageCollectionIds }
},
{ imageId: 1 }
).lean();
const imageIds = imageDatas
.map((item) => item.imageId)
.filter((key) => isS3ObjectKey(key, 'dataset'));
await retryFn(async () => {
await Promise.all([
// Delete training data
......@@ -419,6 +438,9 @@ export async function delCollection({
_id: { $in: collectionIds }
},
{ session }
);
).lean();
// delete s3 images which are uploaded by users
await s3DatasetSource.deleteDatasetFilesByKeys(imageIds);
});
}
packages/service/core/dataset/collection/schema.ts
......@@ -58,10 +58,8 @@ const DatasetCollectionSchema = new Schema({
// Metadata
// local file collection
fileId: {
type: Schema.Types.ObjectId,
ref: 'dataset.files'
},
// Support both GridFS ObjectId (string) and S3 key (string)
fileId: String,
// web link collection
rawLink: String,
// Api collection
......@@ -72,6 +70,7 @@ const DatasetCollectionSchema = new Schema({
rawTextLength: Number,
hashRawText: String,
metadata: {
type: Object,
default: {}
......
packages/service/core/dataset/collection/utils.ts
......@@ -164,6 +164,7 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
const { title, rawText } = await readDatasetSourceRawText({
teamId: collection.teamId,
tmbId: collection.tmbId,
datasetId: collection.datasetId,
...sourceReadType
});
......@@ -208,7 +209,7 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
return DatasetCollectionSyncResultEnum.sameRaw;
};
/*
/*
QA: 独立进程
Chunk: Image Index -> Auto index -> chunk index
*/
......
packages/service/core/dataset/controller.ts
......@@ -15,6 +15,7 @@ import { removeDatasetSyncJobScheduler } from './datasetSync';
import { mongoSessionRun } from '../../common/mongo/sessionRun';
import { removeImageByPath } from '../../common/file/image/controller';
import { UserError } from '@fastgpt/global/common/error/utils';
import { getS3DatasetSource } from '../../common/s3/sources/dataset';
/* ============= dataset ========== */
/* find all datasetId by top datasetId */
......@@ -122,6 +123,10 @@ export async function delDatasetRelevantData({
teamId,
datasetId: { $in: datasetIds }
}).session(session);
for (const datasetId of datasetIds) {
await getS3DatasetSource().deleteDatasetFilesByPrefix({ datasetId });
}
}
export const deleteDatasets = async ({
......
packages/service/core/dataset/data/controller.ts
import { replaceDatasetQuoteTextWithJWT } from '../../../core/dataset/utils';
import { addEndpointToImageUrl } from '../../../common/file/image/utils';
import { getDatasetImagePreviewUrl } from '../image/utils';
import type { DatasetCiteItemType, DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type';
import type { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type';
import { addDays } from 'date-fns';
import { isS3ObjectKey, jwtSignS3ObjectKey } from '../../../common/s3/utils';
export const formatDatasetDataValue = ({
teamId,
......@@ -51,27 +53,24 @@ export const formatDatasetDataValue = ({
if (!imageId) {
return {
q,
a
q: replaceDatasetQuoteTextWithJWT(q, addDays(new Date(), 90)),
a: a ? replaceDatasetQuoteTextWithJWT(a, addDays(new Date(), 90)) : undefined
};
}
const previewUrl = getDatasetImagePreviewUrl({
imageId,
teamId,
datasetId,
expiredMinutes: 60 * 24 * 7 // 7 days
});
const imagePreivewUrl = isS3ObjectKey(imageId, 'dataset')
? jwtSignS3ObjectKey(imageId, addDays(new Date(), 90))
: imageId;
return {
q: `![${q.replaceAll('\n', '')}](${previewUrl})`,
q: `![${q.replaceAll('\n', '')}](${imagePreivewUrl})`,
a,
imagePreivewUrl: previewUrl
imagePreivewUrl
};
};
export const getFormatDatasetCiteList = (list: DatasetDataSchemaType[]) => {
return list.map<DatasetCiteItemType>((item) => ({
return list.map((item) => ({
_id: item._id,
...formatDatasetDataValue({
teamId: item.teamId,
......
packages/service/core/dataset/image/controller.ts
......@@ -10,6 +10,8 @@ import { checkTimerLock } from '../../../common/system/timerLock/utils';
import { TimerIdEnum } from '../../../common/system/timerLock/constants';
import { addLog } from '../../../common/system/log';
import { UserError } from '@fastgpt/global/common/error/utils';
import { getS3DatasetSource, S3DatasetSource } from '../../../common/s3/sources/dataset';
import { isS3ObjectKey } from '../../../common/s3/utils';
const getGridBucket = () => {
return new connectionMongo.mongo.GridFSBucket(connectionMongo.connection.db!, {
......@@ -116,7 +118,11 @@ export const deleteDatasetImage = async (imageId: string) => {
const gridBucket = getGridBucket();
try {
await gridBucket.delete(new Types.ObjectId(imageId));
if (isS3ObjectKey(imageId, 'dataset')) {
await getS3DatasetSource().deleteDatasetFileByKey(imageId);
} else {
await gridBucket.delete(new Types.ObjectId(imageId));
}
} catch (error: any) {
const msg = error?.message;
if (msg.includes('File not found')) {
......
packages/service/core/dataset/migration/schema.ts 0 → 100644
import { connectionMongo, getMongoModel } from '../../../common/mongo';
const { Schema } = connectionMongo;
export const DatasetMigrationLogCollectionName = 'dataset_migration_logs';
export type DatasetMigrationLogSchemaType = {
_id: string;
// 迁移批次信息
batchId: string; // 同一次运行的迁移使用相同的 batchId
migrationVersion: string; // 如 'v4.14.3'
// 资源类型和标识
resourceType: 'collection' | 'image' | 'chat_file'; // 支持不同类型的文件迁移
resourceId: string; // collection._id 或 image._id
teamId: string;
datasetId?: string; // collection 有，image 可能没有
// 迁移前后的存储信息
sourceStorage: {
type: 'gridfs';
fileId: string; // GridFS 的 ObjectId
bucketName: string; // 'dataset' or 'chat'
fileSize?: number; // 文件大小（字节）
checksum?: string; // MD5/SHA256
};
targetStorage?: {
type: 's3';
key: string; // S3 key
bucket?: string; // S3 bucket名称
fileSize?: number;
checksum?: string;
};
// 迁移状态
status: 'pending' | 'processing' | 'completed' | 'failed' | 'rollback' | 'verified';
// 时间戳
createdAt: Date; // 创建时间
startedAt?: Date; // 开始迁移时间
completedAt?: Date; // 完成时间
rolledBackAt?: Date; // 回滚时间
// 重试信息
attemptCount: number; // 尝试次数
maxAttempts: number; // 最大重试次数
lastAttemptAt?: Date; // 最后一次尝试时间
// 错误信息
error?: {
message: string;
stack?: string;
code?: string; // 错误代码，便于分类统计
phase: 'download' | 'upload' | 'verify' | 'update_db'; // 错误发生在哪个阶段
};
// 校验信息
verified: boolean; // 是否已验证数据一致性
verifiedAt?: Date;
// 操作日志（记录详细步骤）
operations: Array<{
action: string; // 'start_download', 'upload_to_s3', 'update_collection', 'rollback' 等
timestamp: Date;
success: boolean;
duration?: number; // 耗时（毫秒）
details?: any; // 额外信息
}>;
// 元数据（用于调试和审计）
metadata: {
fileName?: string; // 原文件名
fileType?: string; // 文件类型
originalUpdateTime?: Date; // collection 的原始更新时间
executorIp?: string; // 执行迁移的服务器 IP
nodeEnv?: string; // 'production' or 'development'
};
// 回滚信息
rollbackInfo?: {
reason: string; // 回滚原因
rolledBackBy?: string; // 操作人员或系统
s3FileDeleted: boolean; // S3 文件是否已删除
dbRestored: boolean; // 数据库是否已恢复
};
};
const DatasetMigrationLogSchema = new Schema({
// 批次信息
batchId: {
type: String,
required: true,
index: true
},
migrationVersion: {
type: String,
required: true
},
// 资源类型和标识
resourceType: {
type: String,
enum: ['collection', 'image', 'chat_file'],
required: true
},
resourceId: {
type: Schema.Types.ObjectId,
required: true,
index: true
},
teamId: {
type: Schema.Types.ObjectId,
required: true,
index: true
},
datasetId: {
type: Schema.Types.ObjectId,
index: true
},
// 存储信息
sourceStorage: {
type: {
type: String,
default: 'gridfs'
},
fileId: {
type: String,
required: true
},
bucketName: String,
fileSize: Number,
checksum: String
},
targetStorage: {
type: {
type: String,
default: 's3'
},
key: String,
bucket: String,
fileSize: Number,
checksum: String
},
// 状态
status: {
type: String,
enum: ['pending', 'processing', 'completed', 'failed', 'rollback', 'verified'],
default: 'pending',
required: true,
index: true
},
// 时间戳
createdAt: {
type: Date,
default: () => new Date(),
index: true
},
startedAt: Date,
completedAt: Date,
rolledBackAt: Date,
// 重试信息
attemptCount: {
type: Number,
default: 0
},
maxAttempts: {
type: Number,
default: 3
},
lastAttemptAt: Date,
// 错误信息
error: {
message: String,
stack: String,
code: String,
phase: {
type: String,
enum: ['download', 'upload', 'verify', 'update_db']
}
},
// 校验信息
verified: {
type: Boolean,
default: false
},
verifiedAt: Date,
// 操作日志
operations: [
{
action: String,
timestamp: {
type: Date,
default: () => new Date()
},
success: Boolean,
duration: Number,
details: Schema.Types.Mixed
}
],
// 元数据
metadata: {
fileName: String,
fileType: String,
originalUpdateTime: Date,
executorIp: String,
nodeEnv: String
},
// 回滚信息
rollbackInfo: {
reason: String,
rolledBackBy: String,
s3FileDeleted: Boolean,
dbRestored: Boolean
}
});
// 索引优化
try {
// 查询某个批次的迁移状态
DatasetMigrationLogSchema.index({ batchId: 1, status: 1 });
// 查询某个资源的迁移历史
DatasetMigrationLogSchema.index({ resourceType: 1, resourceId: 1 });
// 查询失败的迁移（需要重试）
DatasetMigrationLogSchema.index({
status: 1,
attemptCount: 1,
lastAttemptAt: 1
});
// 查询某个团队的迁移情况
DatasetMigrationLogSchema.index({ teamId: 1, status: 1 });
// 唯一索引：同一个资源在同一个批次只能有一条记录
DatasetMigrationLogSchema.index({ batchId: 1, resourceType: 1, resourceId: 1 }, { unique: true });
} catch (error) {
console.log(error);
}
export const MongoDatasetMigrationLog = getMongoModel<DatasetMigrationLogSchemaType>(
DatasetMigrationLogCollectionName,
DatasetMigrationLogSchema
);
packages/service/core/dataset/read.ts
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import {
ChunkTriggerConfigTypeEnum,
DatasetSourceReadTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { readFileContentFromMongo } from '../../common/file/gridfs/controller';
import { urlsFetch } from '../../common/string/cheerio';
import { type TextSplitProps } from '@fastgpt/global/common/string/textSplitter';
import axios from 'axios';
import { readRawContentByFileBuffer } from '../../common/file/read/utils';
import { readS3FileContentByBuffer } from '../../common/file/read/utils';
import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
import { getApiDatasetRequest } from './apiDataset';
import Papa from 'papaparse';
......@@ -17,6 +15,8 @@ import { addLog } from '../../common/system/log';
import { retryFn } from '@fastgpt/global/common/system/utils';
import { getFileMaxSize } from '../../common/file/utils';
import { UserError } from '@fastgpt/global/common/error/utils';
import { getS3DatasetSource, S3DatasetSource } from '../../common/s3/sources/dataset';
import { getFileS3Key, isS3ObjectKey } from '../../common/s3/utils';
export const readFileRawTextByUrl = async ({
teamId,
......@@ -25,6 +25,7 @@ export const readFileRawTextByUrl = async ({
customPdfParse,
getFormatText,
relatedId,
datasetId,
maxFileSize = getFileMaxSize()
}: {
teamId: string;
......@@ -33,6 +34,7 @@ export const readFileRawTextByUrl = async ({
customPdfParse?: boolean;
getFormatText?: boolean;
relatedId: string; // externalFileId / apiFileId
datasetId: string;
maxFileSize?: number;
}) => {
const extension = parseFileExtensionFromUrl(url);
......@@ -64,7 +66,7 @@ export const readFileRawTextByUrl = async ({
const chunks: Buffer[] = [];
let totalLength = 0;
return new Promise<string>((resolve, reject) => {
return new Promise<{ rawText: string }>((resolve, reject) => {
let isAborted = false;
const cleanup = () => {
......@@ -107,8 +109,12 @@ export const readFileRawTextByUrl = async ({
// 立即清理 chunks 数组释放内存
chunks.length = 0;
const { rawText } = await retryFn(() =>
readRawContentByFileBuffer({
const { rawText } = await retryFn(() => {
const { fileParsedPrefix } = getFileS3Key.dataset({
datasetId,
filename: 'file'
});
return readS3FileContentByBuffer({
customPdfParse,
getFormatText,
extension,
......@@ -116,13 +122,14 @@ export const readFileRawTextByUrl = async ({
tmbId,
buffer,
encoding: 'utf-8',
metadata: {
relatedId
imageKeyOptions: {
// TODO: 链接解析出来的图片不过期，删除知识库时候也需要一起删
prefix: fileParsedPrefix
}
})
);
});
});
resolve(rawText);
resolve({ rawText });
} catch (error) {
cleanup();
reject(error);
......@@ -142,7 +149,7 @@ export const readFileRawTextByUrl = async ({
});
};
/*
/*
fileId - local file, read from mongo
link - request
externalFile/apiFile = request read
......@@ -157,7 +164,8 @@ export const readDatasetSourceRawText = async ({
apiDatasetServer,
customPdfParse,
getFormatText,
usageId
usageId,
datasetId
}: {
teamId: string;
tmbId: string;
......@@ -170,20 +178,26 @@ export const readDatasetSourceRawText = async ({
externalFileId?: string; // external file dataset
apiDatasetServer?: ApiDatasetServerType; // api dataset
usageId?: string;
datasetId: string; // For S3 image upload
}): Promise<{
title?: string;
rawText: string;
}> => {
if (type === DatasetSourceReadTypeEnum.fileLocal) {
const { filename, rawText } = await readFileContentFromMongo({
if (!datasetId || !isS3ObjectKey(sourceId, 'dataset')) {
return Promise.reject('datasetId is required for S3 files');
}
const { filename, rawText } = await getS3DatasetSource().getDatasetFileRawText({
teamId,
tmbId,
bucketName: BucketNameEnum.dataset,
fileId: sourceId,
getFormatText,
customPdfParse,
usageId
usageId,
datasetId
});
return {
title: filename,
rawText
......@@ -205,11 +219,12 @@ export const readDatasetSourceRawText = async ({
};
} else if (type === DatasetSourceReadTypeEnum.externalFile) {
if (!externalFileId) return Promise.reject(new UserError('FileId not found'));
const rawText = await readFileRawTextByUrl({
const { rawText } = await readFileRawTextByUrl({
teamId,
tmbId,
url: sourceId,
relatedId: externalFileId,
datasetId,
customPdfParse
});
return {
......@@ -221,7 +236,8 @@ export const readDatasetSourceRawText = async ({
apiFileId: sourceId,
teamId,
tmbId,
customPdfParse
customPdfParse,
datasetId
});
return {
title,
......@@ -239,13 +255,15 @@ export const readApiServerFileContent = async ({
apiFileId,
teamId,
tmbId,
customPdfParse
customPdfParse,
datasetId
}: {
apiDatasetServer?: ApiDatasetServerType;
apiFileId: string;
teamId: string;
tmbId: string;
customPdfParse?: boolean;
datasetId: string;
}): Promise<{
title?: string;
rawText: string;
......@@ -254,7 +272,8 @@ export const readApiServerFileContent = async ({
teamId,
tmbId,
apiFileId,
customPdfParse
customPdfParse,
datasetId
});
};
......@@ -285,7 +304,6 @@ export const rawText2Chunks = async ({
> => {
const parseDatasetBackup2Chunks = (rawText: string) => {
const csvArr = Papa.parse(rawText).data as string[][];
const chunks = csvArr
.slice(1)
.map((item) => ({
......
packages/service/core/dataset/search/controller.ts
......@@ -33,6 +33,8 @@ import { datasetSearchQueryExtension } from './utils';
import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.d';
import { formatDatasetDataValue } from '../data/controller';
import { pushTrack } from '../../../common/middle/tracks/utils';
import { replaceDatasetQuoteTextWithJWT } from '../../../core/dataset/utils';
import { addDays, addHours } from 'date-fns';
export type SearchDatasetDataProps = {
histories: ChatItemType[];
......@@ -53,7 +55,7 @@ export type SearchDatasetDataProps = {
[NodeInputKeyEnum.datasetSearchRerankModel]?: RerankModelItemType;
[NodeInputKeyEnum.datasetSearchRerankWeight]?: number;
/*
/*
{
tags: {
$and: ["str1","str2"],
......@@ -230,7 +232,7 @@ export async function searchDatasetData(
};
};
/*
/*
Collection metadata filter
标签过滤：
1. and 先生效
......@@ -903,10 +905,15 @@ export async function searchDatasetData(
// token filter
const filterMaxTokensResult = await filterDatasetDataByMaxTokens(scoreFilter, maxTokens);
const finalResult = filterMaxTokensResult.map((item) => {
item.q = replaceDatasetQuoteTextWithJWT(item.q, addDays(new Date(), 90));
return item;
});
pushTrack.datasetSearch({ datasetIds, teamId });
return {
searchRes: filterMaxTokensResult,
searchRes: finalResult,
embeddingTokens,
reRankInputTokens,
searchMode,
......
packages/service/core/dataset/utils.ts
import { authDatasetByTmbId } from '../../support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { S3Sources } from '../../common/s3/type';
import { getS3DatasetSource, S3DatasetSource } from '../../common/s3/sources/dataset';
import { getS3ChatSource } from '../../common/s3/sources/chat';
import { jwtSignS3ObjectKey, isS3ObjectKey } from '../../common/s3/utils';
// TODO: 需要优化成批量获取权限
export const filterDatasetsByTmbId = async ({
......@@ -28,3 +32,45 @@ export const filterDatasetsByTmbId = async ({
// Then filter datasetIds based on permissions
return datasetIds.filter((_, index) => permissions[index]);
};
/**
* 替换数据集引用 markdown 文本中的图片链接格式的 S3 对象键为 JWT 签名后的 URL
*
* @param documentQuoteText 数据集引用文本
* @param expiredTime 过期时间
* @returns 替换后的文本
*
* @example
*
* ```typescript
* const datasetQuoteText = '![image.png](dataset/68fee42e1d416bb5ddc85b19/6901c3071ba2bea567e8d8db/aZos7D-214afce5-4d42-4356-9e05-8164d51c59ae.png)';
* const replacedText = await replaceDatasetQuoteTextWithJWT(datasetQuoteText, addDays(new Date(), 90))
* console.log(replacedText)
* // '![image.png](http://localhost:3000/api/system/file/eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJvYmplY3RLZXkiOiJjaGF0LzY5MWFlMjlkNDA0ZDA0Njg3MTdkZDc0Ny82OGFkODVhNzQ2MzAwNmM5NjM3OTlhMDcvalhmWHk4eWZHQUZzOVdKcGNXUmJBaFYyL3BhcnNlZC85YTBmNGZlZC00ZWRmLTQ2MTMtYThkNi01MzNhZjVhZTUxZGMucG5nIiwiaWF0IjoxNzYzMzcwOTYwLCJleHAiOjk1MzkzNzA5NjB9.tMDWg0-ZWRnWPNp9Hakd0w1hhaO8jj2oD98SU0wAQYQ)'
* ```
*/
export function replaceDatasetQuoteTextWithJWT(documentQuoteText: string, expiredTime: Date) {
if (!documentQuoteText || typeof documentQuoteText !== 'string')
return documentQuoteText as string;
const prefixPattern = Object.values(S3Sources)
.map((pattern) => `${pattern}\\/[^\\s)]+`)
.join('|');
const regex = new RegExp(String.raw`(!?)\[([^\]]*)\]\((?!https?:\/\/)(${prefixPattern})\)`, 'g');
const matches = Array.from(documentQuoteText.matchAll(regex));
let content = documentQuoteText;
for (const match of matches.slice().reverse()) {
const [full, bang, alt, objectKey] = match;
if (isS3ObjectKey(objectKey, 'dataset') || isS3ObjectKey(objectKey, 'chat')) {
const url = jwtSignS3ObjectKey(objectKey, expiredTime);
const replacement = `${bang}[${alt}](${url})`;
content =
content.slice(0, match.index) + replacement + content.slice(match.index + full.length);
}
}
return content;
}
packages/service/core/workflow/dispatch/ai/chat.ts
......@@ -41,6 +41,9 @@ import { i18nT } from '../../../../../web/i18n/utils';
import { postTextCensor } from '../../../chat/postTextCensor';
import { createLLMResponse } from '../../../ai/llm/request';
import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';
import { replaceDatasetQuoteTextWithJWT } from '../../../dataset/utils';
import { getFileS3Key } from '../../../../common/s3/utils';
import { addDays } from 'date-fns';
export type ChatProps = ModuleDispatchProps<
AIChatNodeProps & {
......@@ -98,6 +101,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
stringQuoteText //abandon
}
} = props;
const { files: inputFiles } = chatValue2RuntimePrompt(query); // Chat box input files
const modelConstantsData = getLLMModel(model);
......@@ -303,6 +307,7 @@ async function filterDatasetQuote({
: '';
return {
// datasetQuoteText: replaceDatasetQuoteTextWithJWT(datasetQuoteText, addDays(new Date(), 90))
datasetQuoteText
};
}
......
packages/service/core/workflow/dispatch/ai/tool/index.ts
......@@ -31,6 +31,7 @@ import { postTextCensor } from '../../../../chat/postTextCensor';
import type { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io';
import type { McpToolDataType } from '@fastgpt/global/core/app/tool/mcpTool/type';
import type { JSONSchemaInputType } from '@fastgpt/global/core/app/jsonschema';
import { getFileS3Key } from '../../../../../common/s3/utils';
type Response = DispatchNodeResultType<{
[NodeOutputKeyEnum.answerText]: string;
......@@ -119,7 +120,10 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
fileLinks,
inputFiles: globalFiles,
hasReadFilesTool,
usageId
usageId,
appId: props.runningAppInfo.id,
chatId: props.chatId,
uId: props.uid
});
const concatenateSystemPrompt = [
......@@ -277,7 +281,10 @@ const getMultiInput = async ({
customPdfParse,
inputFiles,
hasReadFilesTool,
usageId
usageId,
appId,
chatId,
uId
}: {
runningUserInfo: ChatDispatchProps['runningUserInfo'];
histories: ChatItemType[];
......@@ -288,6 +295,9 @@ const getMultiInput = async ({
inputFiles: UserChatItemValueItemType['file'][];
hasReadFilesTool: boolean;
usageId?: string;
appId: string;
chatId?: string;
uId: string;
}) => {
// Not file quote
if (!fileLinks || hasReadFilesTool) {
......
packages/service/core/workflow/dispatch/index.ts
......@@ -53,12 +53,16 @@ import type { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/const
import { createChatUsageRecord, pushChatItemUsage } from '../../../support/wallet/usage/controller';
import type { RequireOnlyOne } from '@fastgpt/global/common/type/utils';
import { getS3ChatSource } from '../../../common/s3/sources/chat';
import { addPreviewUrlToChatItems } from '../../chat/utils';
import { addPreviewUrlToChatItems, presignVariablesFileUrls } from '../../chat/utils';
import type { MCPClient } from '../../app/mcp';
import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
import { i18nT } from '../../../../web/i18n/utils';
import { clone } from 'lodash';
type Props = Omit<ChatDispatchProps, 'workflowDispatchDeep' | 'timezone' | 'externalProvider'> & {
type Props = Omit<
ChatDispatchProps,
'workflowDispatchDeep' | 'timezone' | 'externalProvider' | 'cloneVariables'
> & {
runtimeNodes: RuntimeNodeItemType[];
runtimeEdges: RuntimeEdgeItemType[];
defaultSkipNodeQueue?: WorkflowDebugResponse['skipNodeQueue'];
......@@ -144,7 +148,7 @@ export async function dispatchWorkFlow({
}
// Get default variables
const cloneVariables = clone(data.variables);
const defaultVariables = {
...externalProvider.externalWorkflowVariables,
...(await getSystemVariables({
......@@ -168,7 +172,8 @@ export async function dispatchWorkFlow({
workflowDispatchDeep: 0,
usageId: newUsageId,
concatUsage,
mcpClientMemory
mcpClientMemory,
cloneVariables
}).finally(() => {
if (streamCheckTimer) {
clearInterval(streamCheckTimer);
......@@ -203,7 +208,8 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
usageId,
concatUsage,
runningUserInfo: { teamId },
mcpClientMemory
mcpClientMemory,
cloneVariables
} = data;
// Over max depth
......@@ -225,6 +231,7 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
[DispatchNodeResponseKeyEnum.toolResponses]: null,
[DispatchNodeResponseKeyEnum.newVariables]: runtimeSystemVar2StoreType({
variables,
cloneVariables,
removeObj: externalProvider.externalWorkflowVariables,
userVariablesConfigs: data.chatConfig?.variables
}),
......@@ -1057,6 +1064,7 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
[DispatchNodeResponseKeyEnum.toolResponses]: workflowQueue.toolRunResponse,
[DispatchNodeResponseKeyEnum.newVariables]: runtimeSystemVar2StoreType({
variables,
cloneVariables,
removeObj: externalProvider.externalWorkflowVariables,
userVariablesConfigs: data.chatConfig?.variables
}),
......@@ -1092,6 +1100,15 @@ const getSystemVariables = async ({
const actualValue = anyValueDecrypt(val);
variablesMap[item.key] = valueTypeFormat(actualValue, item.valueType);
}
// 文件类型全局变量，签发成 string[] 格式
else if (item.type === VariableInputEnum.file) {
const vars = await presignVariablesFileUrls({
variables,
variableConfig: [item]
});
variablesMap[item.key] = vars?.[item.key]?.map((item: any) => item.url);
}
// API
else if (variables[item.label] !== undefined) {
variablesMap[item.key] = valueTypeFormat(variables[item.label], item.valueType);
......
packages/service/core/workflow/dispatch/init/workflowStart.tsx
......@@ -14,7 +14,7 @@ type Response = DispatchNodeResultType<{
[NodeOutputKeyEnum.userFiles]: string[];
}>;
export const dispatchWorkflowStart = (props: Record<string, any>): Response => {
export const dispatchWorkflowStart = async (props: Record<string, any>): Promise<Response> => {
const {
query,
variables,
......
packages/service/core/workflow/dispatch/interactive/formInput.ts
......@@ -8,6 +8,8 @@ import type {
} from '@fastgpt/global/core/workflow/runtime/type';
import type { UserInputFormItemType } from '@fastgpt/global/core/workflow/template/system/interactive/type';
import { addLog } from '../../../../common/system/log';
import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import { anyValueDecrypt } from '../../../../common/secret/utils';
type Props = ModuleDispatchProps<{
[NodeInputKeyEnum.description]: string;
......@@ -47,7 +49,7 @@ export const dispatchFormInput = async (props: Props): Promise<FormInputResponse
node.isEntry = false;
const { text } = chatValue2RuntimePrompt(query);
const userInputVal = (() => {
const rawUserInputVal: Record<string, any> = (() => {
try {
return JSON.parse(text);
} catch (error) {
......@@ -56,6 +58,32 @@ export const dispatchFormInput = async (props: Props): Promise<FormInputResponse
}
})();
const userInputVal = Object.entries(rawUserInputVal).reduce(
(acc, [key, value]) => {
const inputConfig = userInputForms.find((form) => form.key === key);
if (inputConfig?.type === FlowNodeInputTypeEnum.password) {
acc[key] = anyValueDecrypt(value);
} else if (inputConfig?.type === FlowNodeInputTypeEnum.fileSelect) {
if (Array.isArray(value)) {
acc[key] = value.map((file: any) => {
if (typeof file === 'object' && file.url) {
return file.url;
}
return file;
});
} else {
acc[key] = value;
}
} else {
acc[key] = value;
}
return acc;
},
{} as Record<string, any>
);
return {
data: {
...userInputVal,
......
packages/service/core/workflow/dispatch/tools/readFiles.ts
......@@ -7,15 +7,21 @@ import axios from 'axios';
import { serverRequestBaseUrl } from '../../../../common/api/serverRequest';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { detectFileEncoding, parseUrlToFileType } from '@fastgpt/global/common/file/tools';
import { readRawContentByFileBuffer } from '../../../../common/file/read/utils';
import { readS3FileContentByBuffer } from '../../../../common/file/read/utils';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { type ChatItemType, type UserChatItemValueItemType } from '@fastgpt/global/core/chat/type';
import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
import { addLog } from '../../../../common/system/log';
import { addRawTextBuffer, getRawTextBuffer } from '../../../../common/buffer/rawText/controller';
import { addMinutes } from 'date-fns';
import { addDays, addMinutes } from 'date-fns';
import { getNodeErrResponse } from '../utils';
import { isInternalAddress } from '../../../../common/system/utils';
import { replaceDatasetQuoteTextWithJWT } from '../../../dataset/utils';
import { getFileS3Key } from '../../../../common/s3/utils';
import { S3ChatSource } from '../../../../common/s3/sources/chat';
import path from 'path';
import { S3Buckets } from '../../../../common/s3/constants';
import { S3Sources } from '../../../../common/s3/type';
type Props = ModuleDispatchProps<{
[NodeInputKeyEnum.fileUrlList]: string[];
......@@ -152,11 +158,9 @@ export const getFileContentFromLinks = async ({
.map((url) => {
try {
// Check is system upload file
if (url.startsWith('/') || (requestOrigin && url.startsWith(requestOrigin))) {
// Remove the origin(Make intranet requests directly)
if (requestOrigin && url.startsWith(requestOrigin)) {
url = url.replace(requestOrigin, '');
}
const parsedURL = new URL(url, 'http://localhost:3000');
if (requestOrigin && parsedURL.origin === requestOrigin) {
url = url.replace(requestOrigin, '');
}
return url;
......@@ -185,6 +189,7 @@ export const getFileContentFromLinks = async ({
if (isInternalAddress(url)) {
return Promise.reject('Url is invalid');
}
// Get file buffer data
const response = await axios.get(url, {
baseURL: serverRequestBaseUrl,
......@@ -193,21 +198,39 @@ export const getFileContentFromLinks = async ({
const buffer = Buffer.from(response.data, 'binary');
const urlObj = new URL(url, 'http://localhost:3000');
const isChatExternalUrl = urlObj.pathname.startsWith(
`/${S3Buckets.private}/${S3Sources.chat}/`
);
// Get file name
const filename = (() => {
const { filename, extension, imageParsePrefix } = (() => {
const contentDisposition = response.headers['content-disposition'];
if (contentDisposition) {
const filenameRegex = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/;
const matches = filenameRegex.exec(contentDisposition);
if (matches != null && matches[1]) {
return decodeURIComponent(matches[1].replace(/['"]/g, ''));
const filename = decodeURIComponent(matches[1].replace(/['"]/g, ''));
return {
filename,
extension: path.extname(filename).replace('.', ''),
imageParsePrefix: `` // TODO: 需要根据是否是聊天对话里面的外部链接来决定
};
}
}
return url;
if (isChatExternalUrl) {
const filename = urlObj.pathname.split('/').pop() || 'file';
const extension = path.extname(filename).replace('.', '');
return {
filename,
extension,
imageParsePrefix: getFileS3Key.temp({ teamId, filename }).fileParsedPrefix
};
}
return S3ChatSource.parseChatUrl(url);
})();
// Extension
const extension = parseFileExtensionFromUrl(filename);
// Get encoding
const encoding = (() => {
......@@ -223,8 +246,7 @@ export const getFileContentFromLinks = async ({
return detectFileEncoding(buffer);
})();
// Read file
const { rawText } = await readRawContentByFileBuffer({
const { rawText } = await readS3FileContentByBuffer({
extension,
teamId,
tmbId,
......@@ -232,18 +254,27 @@ export const getFileContentFromLinks = async ({
encoding,
customPdfParse,
getFormatText: true,
imageKeyOptions: imageParsePrefix
? {
prefix: imageParsePrefix,
// 聊天对话里面上传的外部链接，解析出来的图片过期时间设置为1天，而且是存储在临时文件夹的
expiredTime: isChatExternalUrl ? addDays(new Date(), 1) : undefined
}
: undefined,
usageId
});
const replacedText = replaceDatasetQuoteTextWithJWT(rawText, addDays(new Date(), 90));
// Add to buffer
addRawTextBuffer({
sourceId: url,
sourceName: filename,
text: rawText,
text: replacedText,
expiredTime: addMinutes(new Date(), 20)
});
return formatResponseObject({ filename, url, content: rawText });
return formatResponseObject({ filename, url, content: replacedText });
} catch (error) {
return formatResponseObject({
filename: '',
......
packages/service/core/workflow/dispatch/tools/runUpdateVar.ts
......@@ -14,6 +14,8 @@ import { type ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/
import { runtimeSystemVar2StoreType } from '../utils';
import { isValidReferenceValue } from '@fastgpt/global/core/workflow/utils';
import { valueTypeFormat } from '@fastgpt/global/core/workflow/runtime/utils';
import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
import { z } from 'zod';
type Props = ModuleDispatchProps<{
[NodeInputKeyEnum.updateList]: TUpdateListItem[];
......@@ -25,6 +27,7 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>
chatConfig,
params,
variables,
cloneVariables,
runtimeNodes,
workflowStreamResponse,
externalProvider,
......@@ -33,6 +36,7 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>
const { updateList } = params;
const nodeIds = runtimeNodes.map((node) => node.nodeId);
const urlSchema = z.string().url();
const result = updateList.map((item) => {
const variable = item.variable;
......@@ -62,11 +66,19 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>
return valueTypeFormat(val, item.valueType);
} else {
return getReferenceVariableValue({
const val = getReferenceVariableValue({
value: item.value,
variables,
nodes: runtimeNodes
});
if (
Array.isArray(val) &&
val.every((url) => typeof url === 'string' && urlSchema.safeParse(url).success)
) {
return val.map((url) => parseUrlToFileType(url)).filter(Boolean);
}
return val;
}
})();
......@@ -94,6 +106,7 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>
event: SseResponseEventEnum.updateVariables,
data: runtimeSystemVar2StoreType({
variables,
cloneVariables,
removeObj: externalProvider.externalWorkflowVariables,
userVariablesConfigs: chatConfig?.variables
})
......
packages/service/core/workflow/dispatch/utils.ts
import { getErrText } from '@fastgpt/global/common/error/utils';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
import type { ChatItemType, UserChatItemFileItemType } from '@fastgpt/global/core/chat/type.d';
import { NodeOutputKeyEnum, VariableInputEnum } from '@fastgpt/global/core/workflow/constants';
import type { VariableItemType } from '@fastgpt/global/core/app/type';
import { encryptSecret } from '../../../common/secret/aes256gcm';
......@@ -119,10 +119,12 @@ export const checkQuoteQAValue = (quoteQA?: SearchDataResponseItemType[]) => {
/* remove system variable */
export const runtimeSystemVar2StoreType = ({
variables,
cloneVariables,
removeObj = {},
userVariablesConfigs = []
}: {
variables: Record<string, any>;
cloneVariables: Record<string, any>;
removeObj?: Record<string, string>;
userVariablesConfigs?: VariableItemType[];
}) => {
......@@ -152,6 +154,10 @@ export const runtimeSystemVar2StoreType = ({
};
}
}
// Remove URL from file variables
else if (item.type === VariableInputEnum.file) {
copyVariables[item.key] = cloneVariables[item.key];
}
});
return copyVariables;
......
packages/service/support/permission/auth/file.ts
......@@ -10,6 +10,9 @@ import { addMinutes } from 'date-fns';
import { parseHeaderCert } from './common';
import jwt from 'jsonwebtoken';
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import { S3Sources } from '../../../common/s3/type';
import { getS3DatasetSource, S3DatasetSource } from '../../../common/s3/sources/dataset';
import { isS3ObjectKey } from '../../../common/s3/utils';
export const authCollectionFile = async ({
fileId,
......@@ -17,28 +20,24 @@ export const authCollectionFile = async ({
...props
}: AuthModeType & {
fileId: string;
}): Promise<
AuthResponseType & {
file: DatasetFileSchema;
}
> => {
}): Promise<AuthResponseType> => {
const authRes = await parseHeaderCert(props);
const { teamId, tmbId } = authRes;
const file = await getFileById({ bucketName: BucketNameEnum.dataset, fileId });
if (!file) {
return Promise.reject(CommonErrEnum.fileNotFound);
}
if (file.metadata?.teamId !== teamId) {
return Promise.reject(CommonErrEnum.unAuthFile);
if (isS3ObjectKey(fileId, 'dataset')) {
const stat = await getS3DatasetSource().getDatasetFileStat(fileId);
if (!stat) return Promise.reject(CommonErrEnum.fileNotFound);
} else {
const file = await getFileById({ bucketName: BucketNameEnum.dataset, fileId });
if (!file) {
return Promise.reject(CommonErrEnum.fileNotFound);
}
if (file.metadata?.teamId !== teamId) {
return Promise.reject(CommonErrEnum.unAuthFile);
}
}
const permission = new Permission({
role: ReadRoleVal,
isOwner: file.metadata?.uid === tmbId || file.metadata?.tmbId === tmbId
});
const permission = new Permission({ role: ReadRoleVal, isOwner: true });
if (!permission.checkPer(per)) {
return Promise.reject(CommonErrEnum.unAuthFile);
......@@ -46,8 +45,7 @@ export const authCollectionFile = async ({
return {
...authRes,
permission,
file
permission
};
};
......
packages/service/support/permission/dataset/auth.ts
......@@ -19,11 +19,13 @@ import { MongoDatasetData } from '../../../core/dataset/data/schema';
import { type AuthModeType, type AuthResponseType } from '../type';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { type ParentIdType } from '@fastgpt/global/common/parentFolder/type';
import { DataSetDefaultRoleVal } from '@fastgpt/global/support/permission/dataset/constant';
import { getDatasetImagePreviewUrl } from '../../../core/dataset/image/utils';
import { i18nT } from '../../../../web/i18n/utils';
import { parseHeaderCert } from '../auth/common';
import { sumPer } from '@fastgpt/global/support/permission/utils';
import { getS3DatasetSource, S3DatasetSource } from '../../../common/s3/sources/dataset';
import { isS3ObjectKey, jwtSignS3ObjectKey } from '../../../common/s3/utils';
import { addHours } from 'date-fns';
export const authDatasetByTmbId = async ({
tmbId,
......@@ -250,12 +252,19 @@ export async function authDatasetData({
a: datasetData.a,
imageId: datasetData.imageId,
imagePreivewUrl: datasetData.imageId
? getDatasetImagePreviewUrl({
imageId: datasetData.imageId,
teamId: datasetData.teamId,
datasetId: datasetData.datasetId,
expiredMinutes: 30
})
? isS3ObjectKey(datasetData.imageId, 'dataset')
? // jwtSignS3ObjectKey(datasetData.imageId, addHours(new Date(), 1))
await getS3DatasetSource().createGetDatasetFileURL({
key: datasetData.imageId,
expiredHours: 1,
external: true
})
: getDatasetImagePreviewUrl({
imageId: datasetData.imageId,
teamId: datasetData.teamId,
datasetId: datasetData.datasetId,
expiredMinutes: 30
})
: undefined,
chunkIndex: datasetData.chunkIndex,
indexes: datasetData.indexes,
......
packages/service/support/permission/schema.ts
......@@ -240,6 +240,13 @@ try {
console.log(error);
}
ResourcePermissionSchema.pre('save', function (next) {
if (!this.tmbId && !this.groupId && !this.orgId) {
return next(new Error('At least one of tmbId, groupId, orgId must be present'));
}
next();
});
export const MongoResourcePermission = getMongoModel<ResourcePermissionType>(
ResourcePermissionCollectionName,
ResourcePermissionSchema
......
packages/service/type/env.d.ts
......@@ -32,6 +32,7 @@ declare global {
PASSWORD_LOGIN_LOCK_SECONDS?: string;
PASSWORD_EXPIRED_MONTH?: string;
MAX_LOGIN_SESSION?: string;
CHAT_MAX_QPM?: string;
// 安全配置
// 密码登录锁定时间
......
packages/web/components/common/Icon/constants.ts
......@@ -45,6 +45,7 @@ export const iconPaths = {
'common/disable': () => import('./icons/common/disable.svg'),
'common/downArrowFill': () => import('./icons/common/downArrowFill.svg'),
'common/download': () => import('./icons/common/download.svg'),
'common/downloadLine': () => import('./icons/common/downloadLine.svg'),
'common/edit': () => import('./icons/common/edit.svg'),
'common/editor/resizer': () => import('./icons/common/editor/resizer.svg'),
'common/ellipsis': () => import('./icons/common/ellipsis.svg'),
......
packages/web/components/common/Icon/icons/common/downloadLine.svg 0 → 100644
<svg viewBox="0 0 11 12" xmlns="http://www.w3.org/2000/svg">
<path d="M9.91667 10.4977C10.0714 10.4977 10.2197 10.5591 10.3291 10.6685C10.4385 10.7779 10.5 10.9263 10.5 11.081C10.5 11.2357 10.4385 11.3841 10.3291 11.4935C10.2197 11.6029 10.0714 11.6643 9.91667 11.6643H0.583333C0.428624 11.6643 0.280251 11.6029 0.170854 11.4935C0.0614582 11.3841 0 11.2357 0 11.081C0 10.9263 0.0614582 10.7779 0.170854 10.6685C0.280251 10.5591 0.428624 10.4977 0.583333 10.4977H9.91667ZM5.25 0C5.40471 0 5.55308 0.0614582 5.66248 0.170854C5.77187 0.280251 5.83333 0.428624 5.83333 0.583333V7.62942L8.6275 4.83525C8.73238 4.73043 8.87324 4.66949 9.02144 4.66481C9.16964 4.66013 9.31406 4.71206 9.42535 4.81004C9.53663 4.90803 9.60643 5.04472 9.62054 5.19232C9.63466 5.33993 9.59203 5.48737 9.50133 5.60467L9.45233 5.6595L5.66008 9.45408C5.55971 9.55461 5.42611 9.61505 5.28434 9.62407C5.14257 9.63309 5.00239 9.59007 4.89008 9.50308L4.83525 9.45467L1.0465 5.67233C0.941484 5.56755 0.880334 5.4267 0.875492 5.27843C0.87065 5.13016 0.92248 4.98562 1.02044 4.87421C1.11839 4.7628 1.25511 4.6929 1.40278 4.67873C1.55045 4.66455 1.69798 4.70717 1.81533 4.79792L1.87017 4.84633L4.66667 7.637V0.583333C4.66667 0.428624 4.72812 0.280251 4.83752 0.170854C4.94692 0.0614582 5.09529 0 5.25 0Z"/>
</svg>
packages/web/components/common/InputTypeSelector/configs.ts 0 → 100644
import {
VariableInputEnum,
variableConfigs,
WorkflowIOValueTypeEnum
} from '@fastgpt/global/core/workflow/constants';
import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import { i18nT } from '../../../i18n/utils';
export type InputTypeConfigItem = {
icon: string;
label: string;
value: string;
description?: string;
defaultValueType?: WorkflowIOValueTypeEnum;
};
export type PluginInputTypeConfigItem = {
icon: string;
label: string;
value: FlowNodeInputTypeEnum[];
defaultValueType: WorkflowIOValueTypeEnum;
description?: string;
};
export const getVariableInputTypeList = (): InputTypeConfigItem[][] => {
return variableConfigs
.map((group) =>
group
.filter((item) => item && item.value !== VariableInputEnum.textarea)
.map((item) => ({
icon: item.icon,
label: item.label,
value: item.value,
description: item.description,
defaultValueType: item.defaultValueType
}))
)
.filter((group) => group.length > 0);
};
export const getFormInputTypeList = (): InputTypeConfigItem[][] => {
return [
[
{
icon: 'core/workflow/inputType/input',
label: i18nT('common:core.workflow.inputType.textInput'),
value: FlowNodeInputTypeEnum.input,
defaultValueType: WorkflowIOValueTypeEnum.string
},
{
icon: 'core/workflow/inputType/password',
label: i18nT('common:core.workflow.inputType.password'),
value: FlowNodeInputTypeEnum.password,
defaultValueType: WorkflowIOValueTypeEnum.string
},
{
icon: 'core/workflow/inputType/numberInput',
label: i18nT('common:core.workflow.inputType.number input'),
value: FlowNodeInputTypeEnum.numberInput,
defaultValueType: WorkflowIOValueTypeEnum.number
},
{
icon: 'core/workflow/inputType/option',
label: i18nT('common:core.workflow.inputType.select'),
value: FlowNodeInputTypeEnum.select,
defaultValueType: WorkflowIOValueTypeEnum.string
},
{
icon: 'core/workflow/inputType/multipleSelect',
label: i18nT('common:core.workflow.inputType.multipleSelect'),
value: FlowNodeInputTypeEnum.multipleSelect,
defaultValueType: WorkflowIOValueTypeEnum.arrayString
},
{
icon: 'core/workflow/inputType/switch',
label: i18nT('common:core.workflow.inputType.switch'),
value: FlowNodeInputTypeEnum.switch,
defaultValueType: WorkflowIOValueTypeEnum.boolean
},
{
icon: 'core/workflow/inputType/timePointSelect',
label: i18nT('common:core.workflow.inputType.timePointSelect'),
value: FlowNodeInputTypeEnum.timePointSelect,
defaultValueType: WorkflowIOValueTypeEnum.string
},
{
icon: 'core/workflow/inputType/timeRangeSelect',
label: i18nT('common:core.workflow.inputType.timeRangeSelect'),
value: FlowNodeInputTypeEnum.timeRangeSelect,
defaultValueType: WorkflowIOValueTypeEnum.arrayString
}
],
[
{
icon: 'core/workflow/inputType/file',
label: i18nT('common:core.workflow.inputType.file'),
value: FlowNodeInputTypeEnum.fileSelect,
defaultValueType: WorkflowIOValueTypeEnum.arrayString
},
{
icon: 'core/workflow/inputType/selectLLM',
label: i18nT('common:core.workflow.inputType.selectLLMModel'),
value: FlowNodeInputTypeEnum.selectLLMModel,
defaultValueType: WorkflowIOValueTypeEnum.string
}
]
];
};
export const getPluginInputTypeRawList = (options?: {
hasDynamicInput?: boolean;
}): PluginInputTypeConfigItem[][] => {
const { hasDynamicInput = false } = options || {};
return [
[
{
icon: 'core/workflow/inputType/reference',
label: i18nT('common:core.workflow.inputType.Reference'),
value: [FlowNodeInputTypeEnum.reference],
defaultValueType: WorkflowIOValueTypeEnum.string
},
{
icon: 'core/workflow/inputType/input',
label: i18nT('common:core.workflow.inputType.textInput'),
value: [FlowNodeInputTypeEnum.input, FlowNodeInputTypeEnum.reference],
defaultValueType: WorkflowIOValueTypeEnum.string
},
{
icon: 'core/workflow/inputType/password',
label: i18nT('common:core.workflow.inputType.password'),
value: [FlowNodeInputTypeEnum.password],
defaultValueType: WorkflowIOValueTypeEnum.string
},
{
icon: 'core/workflow/inputType/numberInput',
label: i18nT('common:core.workflow.inputType.number input'),
value: [FlowNodeInputTypeEnum.numberInput, FlowNodeInputTypeEnum.reference],
defaultValueType: WorkflowIOValueTypeEnum.number
},
{
icon: 'core/workflow/inputType/option',
label: i18nT('common:core.workflow.inputType.select'),
value: [FlowNodeInputTypeEnum.select, FlowNodeInputTypeEnum.reference],
defaultValueType: WorkflowIOValueTypeEnum.string
},
{
icon: 'core/workflow/inputType/multipleSelect',
label: i18nT('common:core.workflow.inputType.multipleSelect'),
value: [FlowNodeInputTypeEnum.multipleSelect, FlowNodeInputTypeEnum.reference],
defaultValueType: WorkflowIOValueTypeEnum.arrayString
},
{
icon: 'core/workflow/inputType/switch',
label: i18nT('common:core.workflow.inputType.switch'),
value: [FlowNodeInputTypeEnum.switch, FlowNodeInputTypeEnum.reference],
defaultValueType: WorkflowIOValueTypeEnum.boolean
},
{
icon: 'core/workflow/inputType/timePointSelect',
label: i18nT('common:core.workflow.inputType.timePointSelect'),
value: [FlowNodeInputTypeEnum.timePointSelect, FlowNodeInputTypeEnum.reference],
defaultValueType: WorkflowIOValueTypeEnum.string
},
{
icon: 'core/workflow/inputType/timeRangeSelect',
label: i18nT('common:core.workflow.inputType.timeRangeSelect'),
value: [FlowNodeInputTypeEnum.timeRangeSelect, FlowNodeInputTypeEnum.reference],
defaultValueType: WorkflowIOValueTypeEnum.arrayString
}
],
[
{
icon: 'core/workflow/inputType/file',
label: i18nT('common:core.workflow.inputType.file'),
value: [FlowNodeInputTypeEnum.fileSelect, FlowNodeInputTypeEnum.reference],
defaultValueType: WorkflowIOValueTypeEnum.arrayString
},
{
icon: 'core/workflow/inputType/selectLLM',
label: i18nT('common:core.workflow.inputType.selectLLMModel'),
value: [FlowNodeInputTypeEnum.selectLLMModel, FlowNodeInputTypeEnum.reference],
defaultValueType: WorkflowIOValueTypeEnum.string
},
{
icon: 'core/workflow/inputType/selectDataset',
label: i18nT('common:core.workflow.inputType.selectDataset'),
value: [FlowNodeInputTypeEnum.selectDataset, FlowNodeInputTypeEnum.reference],
defaultValueType: WorkflowIOValueTypeEnum.selectDataset
},
...(hasDynamicInput
? []
: [
{
icon: 'core/workflow/inputType/dynamic',
label: i18nT('common:core.workflow.inputType.dynamicTargetInput'),
value: [FlowNodeInputTypeEnum.addInputParam],
defaultValueType: WorkflowIOValueTypeEnum.dynamic
}
])
],
[
{
icon: 'core/workflow/inputType/customVariable',
label: i18nT('common:core.workflow.inputType.custom'),
value: [FlowNodeInputTypeEnum.customVariable],
defaultValueType: WorkflowIOValueTypeEnum.string,
description: i18nT('app:variable.select type_desc')
},
{
icon: 'core/workflow/inputType/internal',
label: i18nT('common:core.workflow.inputType.internal'),
value: [FlowNodeInputTypeEnum.hidden],
defaultValueType: WorkflowIOValueTypeEnum.string,
description: i18nT('app:variable.internal_type_desc')
}
]
];
};
export const getPluginInputTypeList = (options?: {
hasDynamicInput?: boolean;
}): InputTypeConfigItem[][] => {
const rawList = getPluginInputTypeRawList(options);
return rawList.map((group) =>
group.map((item) => ({
icon: item.icon,
label: item.label,
value: item.value[0],
defaultValueType: item.defaultValueType,
description: item.description
}))
);
};
packages/web/components/common/InputTypeSelector/index.tsx 0 → 100644
import React from 'react';
import { Box, Flex } from '@chakra-ui/react';
import MyIcon from '../Icon';
import QuestionTip from '../MyTooltip/QuestionTip';
import type { InputTypeConfigItem } from './configs';
import { useTranslation } from 'next-i18next';
const InputTypeSelector = ({
inputTypeList,
selectedType,
onTypeChange
}: {
inputTypeList: InputTypeConfigItem[][];
selectedType: string;
onTypeChange: (type: string) => void;
}) => {
const { t } = useTranslation();
return (
<Flex flexDirection={'column'} gap={4}>
{inputTypeList.map((group, groupIndex) => {
return (
<Box
key={groupIndex}
display={'grid'}
gridTemplateColumns={'repeat(3, 1fr)'}
gap={4}
mt={3}
>
{group.map((item) => {
const isSelected = selectedType === item.value;
return (
<Box
display={'flex'}
key={item.label}
border={'1px solid'}
borderColor={isSelected ? 'primary.600' : 'myGray.250'}
p={3}
rounded={'6px'}
fontWeight={'medium'}
fontSize={'14px'}
alignItems={'center'}
cursor={'pointer'}
boxShadow={isSelected ? '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)' : 'none'}
_hover={{
'& > svg': {
color: 'primary.600'
},
'& > span': {
color: 'myGray.900'
},
borderColor: 'primary.600',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)'
}}
onClick={() => onTypeChange(item.value)}
>
<MyIcon
name={item.icon as any}
w={'20px'}
mr={1.5}
color={isSelected ? 'primary.600' : 'myGray.400'}
/>
<Box as="span" color={isSelected ? 'myGray.900' : 'inherit'} whiteSpace="nowrap">
{t(item.label)}
</Box>
{item.description && <QuestionTip label={t(item.description)} ml={1} />}
</Box>
);
})}
</Box>
);
})}
</Flex>
);
};
export default InputTypeSelector;
packages/web/components/common/Textarea/JsonEditor/index.tsx
......@@ -47,7 +47,11 @@ const options = {
scrollBeyondLastLine: false,
folding: false,
overviewRulerBorder: false,
tabSize: 2
tabSize: 2,
padding: {
top: 8,
bottom: 8
}
};
const JSONEditor = ({
......@@ -245,10 +249,9 @@ const JSONEditor = ({
borderWidth={'1px'}
borderRadius={'sm'}
borderColor={isInvalid ? 'red.500' : 'myGray.200'}
py={2}
height={height}
position={'relative'}
transition={'border-color 0.1s ease-in-out, box-shadow 0.1s ease-in-out'}
transition={'border-color 0.3s ease-in-out, box-shadow 0.3s ease-in-out'}
_focusWithin={
isInvalid
? {
......@@ -257,7 +260,8 @@ const JSONEditor = ({
}
: {
borderColor: 'primary.600',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)'
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)',
bg: 'white'
}
}
{...props}
......
packages/web/components/core/app/FileTypeSelector/index.tsx 0 → 100644
import React, { useState, useCallback } from 'react';
import { Box, Checkbox, VStack, Flex, Input } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import {
defaultFileExtensionTypes,
type FileExtensionKeyType
} from '@fastgpt/global/core/app/constants';
import MyIcon from '../../../common/Icon';
type FileTypeSelectorValue = {
canSelectFile?: boolean;
canSelectImg?: boolean;
canSelectVideo?: boolean;
canSelectAudio?: boolean;
canSelectCustomFileExtension?: boolean;
customFileExtensionList?: string[];
};
export const FileTypeSelectorPanel = ({
value,
onChange
}: {
value: FileTypeSelectorValue;
onChange: (value: FileTypeSelectorValue) => void;
}) => {
const { t } = useTranslation();
const [isAdding, setIsAdding] = useState(false);
const [newExtension, setNewExtension] = useState('.');
const handleTypeChange = (type: FileExtensionKeyType, checked: boolean) => {
onChange({
...value,
[type]: checked
});
};
const handleConfirmCustomExtension = useCallback(() => {
const currentList = value.customFileExtensionList || [];
if (newExtension !== '.' && !currentList.includes(newExtension)) {
onChange({
...value,
customFileExtensionList: [...currentList, newExtension]
});
}
setIsAdding(false);
setNewExtension('.');
}, [newExtension, value, onChange]);
const handleRemoveExtension = useCallback(
(ext: string) => {
const currentList = value.customFileExtensionList || [];
onChange({
...value,
customFileExtensionList: currentList.filter((item) => item !== ext)
});
},
[value, onChange]
);
return (
<VStack w="full" spacing={3} alignItems={'flex-start'}>
{Object.entries(defaultFileExtensionTypes).map(([type, exts]) =>
type === 'canSelectCustomFileExtension' ? (
<VStack key={type} w="full" spacing={2} alignItems={'flex-start'}>
<Checkbox
w="full"
alignItems={'flex-start'}
cursor="pointer"
isChecked={value.canSelectCustomFileExtension ?? false}
onChange={(e) => handleTypeChange(type as FileExtensionKeyType, e.target.checked)}
>
<Box color={'myGray.900'} lineHeight={1} mb={2}>
{t('app:upload_file_extension_type_canSelectCustomFileExtension')}
</Box>
<Flex gap={1} alignItems={'center'} flexWrap={'wrap'}>
{(value.customFileExtensionList || []).map((ext) => (
<Box
key={ext}
position={'relative'}
border="base"
borderRadius={'sm'}
color={'myGray.600'}
userSelect="none"
px={1}
py={0.5}
fontSize={'xs'}
minW={'50px'}
h={'22px'}
textAlign={'center'}
>
<Box>{ext}</Box>
<Flex
position="absolute"
top={0}
left={0}
right={0}
bottom={0}
alignItems="center"
justifyContent="center"
bg="rgba(255, 255, 255, 0.85)"
borderRadius={'sm'}
opacity={0}
cursor="pointer"
transition="opacity 0.2s"
_hover={{ opacity: 1 }}
onClick={(e) => {
e.stopPropagation();
e.preventDefault();
handleRemoveExtension(ext);
}}
>
<MyIcon
name={'delete'}
w={'14px'}
h={'14px'}
color={'red.600'}
_hover={{ color: 'red.700' }}
/>
</Flex>
</Box>
))}
<Flex
gap={1}
alignItems={'center'}
border="base"
borderRadius={'sm'}
cursor="pointer"
color={'myGray.600'}
_hover={{
color: 'primary.600',
borderColor: 'primary.600'
}}
_focusWithin={{
borderColor: 'primary.600'
}}
userSelect="none"
px={1}
py={0.5}
h={'22px'}
onClick={(e) => {
e.stopPropagation();
e.preventDefault();
setIsAdding(true);
}}
>
{isAdding ? (
<Input
variant={'unstyled'}
value={newExtension}
autoFocus
fontSize={'xs'}
w={'50px'}
maxLength={7}
onChange={(e) =>
setNewExtension(`.${e.target.value.replace(/^\./, '').trim()}`)
}
onBlur={handleConfirmCustomExtension}
onKeyDown={(e) => {
if (e.key.toLowerCase() !== 'enter') return;
handleConfirmCustomExtension();
}}
/>
) : (
<Flex gap={1} alignItems={'center'} h={'22px'}>
<MyIcon name={'common/add2'} w={'12px'} h={'12px'} />
<Box fontSize={'xs'}>
{t(
'app:upload_file_extension_type_canSelectCustomFileExtension_placeholder'
)}
</Box>
</Flex>
)}
</Flex>
</Flex>
</Checkbox>
</VStack>
) : (
<VStack key={type} w="full" spacing={2} alignItems={'flex-start'}>
<Checkbox
w="full"
alignItems={'flex-start'}
borderBottom="1px solid"
borderColor="myGray.200"
pb={3}
cursor="pointer"
isChecked={
value[type as FileExtensionKeyType] ??
(type === 'canSelectFile' || type === 'canSelectImg')
}
onChange={(e) => handleTypeChange(type as FileExtensionKeyType, e.target.checked)}
>
<Box color={'myGray.900'} lineHeight={1}>
{t(`app:upload_file_extension_type_${type}`)}
</Box>
<Box mt={1} fontSize={'xs'} color={'myGray.500'} wordBreak={'break-word'} w="full">
{exts.map((ext) => ext.slice(1)).join('/')}
</Box>
</Checkbox>
</VStack>
)
)}
</VStack>
);
};
packages/web/components/core/plugin/tool/ToolCard.tsx
......@@ -7,12 +7,6 @@ import MyIcon from '../../../common/Icon';
import { parseI18nString } from '@fastgpt/global/common/i18n/utils';
import { PluginStatusEnum } from '@fastgpt/global/core/plugin/type';
/*
3 种使用场景：
1. admin 视角插件市场：显示是否安装，无状态，显示安装/卸载
2. team 视角资源库：显示是否安装，状态文本，以及安装/卸载
3. 开放的插件市场：不显示任何状态，只显示下载按钮
*/
export type ToolCardItemType = {
id: string;
name: string;
......@@ -23,21 +17,35 @@ export type ToolCardItemType = {
downloadUrl?: string;
status?: number;
installed?: boolean;
update?: boolean;
downloadCount?: number;
};
/**
3 种使用场景：
1. admin 视角插件市场：显示是否安装，是否更新，无状态，显示安装/卸载
2. team 视角资源库：显示是否安装，不显示更新，状态文本，以及安装/卸载
3. 开放的插件市场：不显示任何状态，只显示下载按钮
*/
const ToolCard = ({
item,
systemTitle,
isLoading,
isInstallingOrDeleting,
isUpdating,
mode,
onClickButton,
onInstall,
onDelete,
onUpdate,
onClickCard
}: {
item: ToolCardItemType;
systemTitle?: string;
isLoading?: boolean;
isInstallingOrDeleting?: boolean;
isUpdating?: boolean;
mode: 'admin' | 'team' | 'marketplace';
onClickButton: (installed: boolean) => void;
onInstall: () => Promise<void>;
onDelete?: () => Promise<void>;
onUpdate?: () => Promise<void>;
onClickCard?: () => void;
}) => {
const { t, i18n } = useTranslation();
......@@ -76,7 +84,7 @@ const ToolCard = ({
};
}, [item.tags]);
const statusMap = useMemo(() => {
const statusLabel = useMemo(() => {
if (mode === 'marketplace') return null;
const pluginStatusMap: Record<number, { label: string; color: string; icon?: string } | null> =
......@@ -91,23 +99,27 @@ const ToolCard = ({
}
};
const installedStatusMap = item.installed
? {
label: t('app:toolkit_installed'),
color: 'myGray.500',
icon: 'common/check'
}
: null;
if (mode === 'admin') {
return installedStatusMap;
return item.installed
? {
label: t('app:toolkit_installed'),
color: 'myGray.500',
icon: 'common/check'
}
: null;
}
if (mode === 'team') {
if (item.status && pluginStatusMap[item.status]) {
return pluginStatusMap[item.status];
}
return installedStatusMap;
return item.installed
? {
label: t('app:toolkit_installed'),
color: 'myGray.500',
icon: 'common/check'
}
: null;
}
}, [item.installed, item.status]);
......@@ -122,23 +134,77 @@ const ToolCard = ({
display={'flex'}
flexDirection={'column'}
cursor={onClickCard ? 'pointer' : 'default'}
onClick={onClickCard}
position={'relative'}
onClick={() => {
if (isInstallingOrDeleting || isUpdating) return;
onClickCard?.();
}}
_hover={{
boxShadow: '0 4px 4px 0 rgba(19, 51, 107, 0.05), 0 0 1px 0 rgba(19, 51, 107, 0.08);',
'& .install-button': {
display: 'flex'
},
'& .update-button': {
display: 'flex'
},
// Only hide author info when there are multiple buttons
...(item.update && mode === 'admin'
? {
'& .author-info': {
display: 'none'
}
}
: {}),
'& .download-count': {
display: 'none'
}
}}
>
{/* Update badge in top-right corner */}
{item.update && mode === 'admin' && (
<Flex
alignItems="center"
position={'absolute'}
top={4}
right={4}
px={2}
py={0.5}
bg={'rgb(255, 247, 237)'}
color={'rgba(234,88,12,1)'}
fontSize={'12px'}
fontWeight={'medium'}
borderRadius={'0.5rem'}
borderColor={'rgba(255,237,213,1)'}
borderWidth={'1px'}
zIndex={1}
>
<svg
xmlns="http://www.w3.org/2000/svg"
width="12"
height="12"
viewBox="0 0 24 24"
fill="currentColor"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
strokeLinejoin="round"
aria-hidden="true"
>
<path d="M8.5 14.5A2.5 2.5 0 0 0 11 12c0-1.38-.5-2-1-3-1.072-2.143-.224-4.054 2-6 .5 2.5 2 4.9 4 6.5 2 1.6 3 3.5 3 5.5a7 7 0 1 1-14 0c0-1.153.433-2.294 1-3a2.5 2.5 0 0 0 2.5 2.5z"></path>
</svg>
{t('app:app.modules.has new version')}
</Flex>
)}
<HStack>
<Avatar src={item.icon} borderRadius={'sm'} w={'1.5rem'} />
<Box color={'myGray.900'} fontWeight={'medium'}>
{parseI18nString(item.name, i18n.language)}
</Box>
{statusMap && (
<Flex fontSize={'12px'} fontWeight={'medium'} color={statusMap.color} gap={1}>
{statusMap.icon && <MyIcon name={statusMap.icon as any} w={4} />}
{statusMap.label}
{statusLabel && (
<Flex fontSize={'12px'} fontWeight={'medium'} color={statusLabel.color} gap={1}>
{statusLabel.icon && <MyIcon name={statusLabel.icon as any} w={4} />}
{statusLabel.label}
</Flex>
)}
</HStack>
......@@ -194,36 +260,77 @@ const ToolCard = ({
</Flex>
<Flex w={'full'} fontSize={'mini'} alignItems={'end'} justifyContent={'space-between'}>
<Box color={'myGray.500'} mt={3}>{`by ${item.author || systemTitle || 'FastGPT'}`}</Box>
{mode === 'marketplace' ? (
<Button
className="install-button"
size={'sm'}
variant={'primary'}
onClick={(e) => {
e.stopPropagation();
onClickButton(false);
}}
isLoading={isLoading}
{...(!isLoading ? { display: 'none' } : {})}
>
{t('common:Download')}
</Button>
) : (
<Button
className="install-button"
{...(!isLoading ? { display: 'none' } : {})}
size={'sm'}
variant={item.installed ? 'primaryOutline' : 'primary'}
onClick={(e) => {
e.stopPropagation();
onClickButton(!item.installed);
}}
isLoading={isLoading}
>
{item.installed ? t('app:toolkit_uninstall') : t('app:toolkit_install')}
</Button>
)}
<Box
className="author-info"
color={'myGray.500'}
mt={3}
>{`by ${item.author || systemTitle || 'FastGPT'}`}</Box>
{/*TODO: when statistics is ready*/}
{/*<Flex flexDirection={'row'} gap={1} className="download-count" color={'myGray.500'} mt={3}>
<MyIcon name="common/downloadLine" />
{!item.downloadCount
? 0
: item.downloadCount < 1000
? `${item.downloadCount}`
: `${(item.downloadCount / 1000).toFixed(1)}k`}
</Flex>*/}
<Flex gap={2} alignItems={'center'} ml={'auto'}>
{mode === 'marketplace' ? (
<Button
className="install-button"
size={'sm'}
variant={'primary'}
onClick={(e) => {
e.stopPropagation();
onInstall();
}}
isLoading={isInstallingOrDeleting}
{...(!isInstallingOrDeleting ? { display: 'none' } : {})}
>
{t('common:Download')}
</Button>
) : (
<Button
className="install-button"
size={'sm'}
variant={item.installed ? 'primaryOutline' : 'primary'}
onClick={async (e) => {
e.stopPropagation();
if (item.installed) {
// delete
if (onDelete) {
return onDelete();
}
} else {
return onInstall();
}
}}
isLoading={isInstallingOrDeleting}
{...(!isInstallingOrDeleting ? { display: 'none' } : {})}
isDisabled={isUpdating}
>
{item.installed ? t('app:toolkit_uninstall') : t('app:toolkit_install')}
</Button>
)}
{/* Update button for admin mode when update is available */}
{item.update && mode === 'admin' && onUpdate && (
<Button
className="update-button"
size={'sm'}
variant={'primary'}
onClick={async (e) => {
e.stopPropagation();
return onUpdate();
}}
isLoading={isUpdating}
display={'none'}
>
{t('app:custom_plugin_update')}
</Button>
)}
</Flex>
</Flex>
</MyBox>
);
......
packages/web/components/core/plugin/tool/ToolDetailDrawer.tsx
......@@ -163,6 +163,8 @@ const ToolDetailDrawer = ({
onClose,
selectedTool,
onToggleInstall,
onUpdate,
isUpdating,
systemTitle,
onFetchDetail,
isLoading,
......@@ -172,6 +174,8 @@ const ToolDetailDrawer = ({
onClose: () => void;
selectedTool: ToolCardItemType;
onToggleInstall: (installed: boolean) => void;
onUpdate?: () => void;
isUpdating?: boolean;
systemTitle?: string;
onFetchDetail?: (toolId: string) => Promise<GetTeamToolDetailResponseType>;
isLoading?: boolean;
......@@ -183,7 +187,7 @@ const ToolDetailDrawer = ({
const [toolDetail, setToolDetail] = useState<
{ tools: Array<toolDetailType & { readme: string }>; downloadUrl: string } | undefined
>(undefined);
const [loading, setLoading] = useState(false);
const [loadingDetail, setLoading] = useState(false);
const [readmeContent, setReadmeContent] = useState<string>('');
const [isInstalled, setIsInstalled] = useState(selectedTool.installed);
......@@ -321,23 +325,44 @@ const ToolDetailDrawer = ({
<Box fontSize={'12px'} color="myGray.500" mt={3}>
{`by ${parentTool?.author || systemTitle || 'FastGPT'}`}
</Box>
<Flex mt={3}>
<Button
w="full"
variant={isInstalled ? 'primaryOutline' : 'primary'}
isLoading={isLoading || loading}
onClick={async () => {
onToggleInstall(!isInstalled);
if (mode === 'marketplace') return;
setIsInstalled(!isInstalled);
}}
>
{isDownload
? t('common:Download')
: isInstalled
? t('app:toolkit_uninstall')
: t('app:toolkit_install')}
</Button>
<Flex mt={3} gap={2}>
{/* Determine if we have two buttons */}
{(() => {
const hasUpdateButton = selectedTool.update && onUpdate && mode !== 'marketplace';
const buttonFlex = hasUpdateButton ? 1 : 1; // Both use flex=1, but when single button it fills the space
return (
<>
<Button
flex={buttonFlex}
variant={isInstalled ? 'primaryOutline' : 'primary'}
isLoading={isLoading || loadingDetail}
isDisabled={isUpdating}
onClick={async () => {
onToggleInstall(!isInstalled);
if (mode === 'marketplace') return;
setIsInstalled(!isInstalled);
}}
>
{isDownload
? t('common:Download')
: isInstalled
? t('app:toolkit_uninstall')
: t('app:toolkit_install')}
</Button>
{hasUpdateButton && (
<Button
variant="primary"
flex={1}
isLoading={isUpdating || loadingDetail}
onClick={onUpdate}
>
{t('app:custom_plugin_update')}
</Button>
)}
</>
);
})()}
</Flex>
{showPoint && (
......
packages/web/i18n/en/app.json
......@@ -43,9 +43,7 @@
"app.Version name": "Version Name",
"app.error.publish_unExist_app": "Release failed, please check whether the tool call is normal",
"app.error.unExist_app": "Some components are missing, please delete them",
"app.modules.click to update": "Click to Refresh",
"app.modules.has new version": "New Version Available",
"app.modules.not_found": "Not Found",
"app.version_current": "Current Version",
"app.version_initial": "Initial Version",
"app.version_name_tips": "Version name cannot be empty",
......@@ -153,7 +151,7 @@
"file_recover": "File will overwrite current content",
"file_types": "Optional file types",
"file_upload": "File Upload",
"file_upload_tip": "Once enabled, documents/images can be uploaded. Documents are retained for 7 days, images for 15 days. Using this feature may incur additional costs. To ensure a good experience, please choose an AI model with a larger context length when using this feature.",
"file_upload_tip": "Once enabled, you can configure the types of files that users can upload. Files are saved along with the conversation; deleting the conversation or deleting the application will clear the files. To ensure a good user experience, please select an AI model with a longer context length when using it.",
"find_more_tools": "Explore more",
"go_to_chat": "Go to Conversation",
"go_to_run": "Go to Execution",
......
packages/web/i18n/en/chat.json
......@@ -75,6 +75,7 @@
"query_extension_result": "Problem optimization results",
"question_tip": "From top to bottom, the response order of each module",
"read_raw_source": "Open the original text",
"images_collection_not_supported": "Image collection is not supported open the original file",
"reasoning_text": "Thinking process",
"release_cancel": "Release Cancel",
"release_send": "Release send, slide up to cancel",
......
packages/web/i18n/en/common.json
......@@ -922,8 +922,8 @@
"name": "name",
"name_is_empty": "Name Cannot Be Empty",
"navbar.Account": "Account",
"navbar.Chat": "portal",
"navbar.Config": "admini",
"navbar.Chat": "Portal",
"navbar.Config": "Admin",
"navbar.Datasets": "Dataset",
"navbar.Studio": "Studio",
"navbar.Toolkit": "Toolkit",
......
packages/web/i18n/zh-CN/app.json
......@@ -45,9 +45,7 @@
"app.Version name": "版本名称",
"app.error.publish_unExist_app": "发布失败，请检查工具调用是否正常",
"app.error.unExist_app": "部分组件缺失，请删除",
"app.modules.click to update": "点击更新",
"app.modules.has new version": "有新版本",
"app.modules.not_found": "组件缺失",
"app.modules.has new version": "新版本",
"app.version_current": "当前版本",
"app.version_initial": "初始版本",
"app.version_name_tips": "版本名称不能为空",
......@@ -157,7 +155,7 @@
"file_recover": "文件将覆盖当前内容",
"file_types": "可选文件类型",
"file_upload": "文件上传",
"file_upload_tip": "开启后，可以上传文档/图片。文档保留7天，图片保留15天。使用该功能可能产生较多额外费用。为保证使用体验，使用该功能时，请选择上下文长度较大的AI模型。",
"file_upload_tip": "开启后，可以配置用户可上传的文件类型。文件跟随对话保存，删除对话或删除应用均会清理文件。为保证使用体验，使用时请选择上下文长度较大的AI模型。",
"find_more_tools": "探索更多",
"go_to_chat": "去对话",
"go_to_run": "去运行",
......
packages/web/i18n/zh-CN/chat.json
......@@ -75,6 +75,7 @@
"query_extension_result": "问题优化结果",
"question_tip": "从上到下，为各个模块的响应顺序",
"read_raw_source": "打开原文",
"images_collection_not_supported": "图片数据集不支持打开原文",
"reasoning_text": "思考过程",
"release_cancel": "松开取消",
"release_send": "松开发送，上滑取消",
......
packages/web/i18n/zh-Hant/app.json
......@@ -43,9 +43,7 @@
"app.Version name": "版本名稱",
"app.error.publish_unExist_app": "發布失敗，請檢查工具呼叫是否正常",
"app.error.unExist_app": "部分元件遺失，請刪除",
"app.modules.click to update": "點選更新",
"app.modules.has new version": "有新版本",
"app.modules.not_found": "元件遺失",
"app.modules.has new version": "新版本",
"app.version_current": "目前版本",
"app.version_initial": "初始版本",
"app.version_name_tips": "版本名稱不能空白",
......@@ -152,7 +150,7 @@
"file_recover": "檔案將會覆蓋目前內容",
"file_types": "可選文件類型",
"file_upload": "檔案上傳",
"file_upload_tip": "開啟後，可以上傳文件／圖片。文件保留 7 天，圖片保留 15 天。使用這個功能可能產生較多額外費用。為了確保使用體驗，使用這個功能時，請選擇上下文長度較大的 AI 模型。",
"file_upload_tip": "開啟後，可以設定使用者可上傳的檔案類型。檔案跟隨對話儲存，刪除對話或刪除應用程式均會清理檔案。為保證使用體驗，使用時請選擇上下文長度較大的AI模型。",
"find_more_tools": "探索更多",
"go_to_chat": "前往對話",
"go_to_run": "前往執行",
......
packages/web/i18n/zh-Hant/chat.json
......@@ -75,6 +75,7 @@
"query_extension_result": "問題優化結果",
"question_tip": "由上至下，各個模組的回應順序",
"read_raw_source": "開啟原文",
"images_collection_not_supported": "圖片資料集不支持開啟原文",
"reasoning_text": "思考過程",
"release_cancel": "鬆開取消",
"release_send": "鬆開傳送，上滑取消",
......
projects/app/.env.template
......@@ -96,6 +96,8 @@ PASSWORD_LOGIN_LOCK_SECONDS=
PASSWORD_EXPIRED_MONTH=
# 最大登录客户端数量，默认为 10
MAX_LOGIN_SESSION=
# 工作流QPM
CHAT_MAX_QPM=5000
# 特殊配置
# 自定义跨域，不配置时，默认都允许跨域（逗号分割）
......
projects/app/package.json
{
"name": "app",
"version": "4.14.1",
"version": "4.14.3",
"private": false,
"scripts": {
"dev": "npm run build:workers && next dev",
......@@ -50,6 +50,7 @@
"next": "14.2.32",
"next-i18next": "15.4.2",
"nprogress": "^0.2.0",
"p-limit": "^7.2.0",
"qrcode": "^1.5.4",
"react": "18.3.1",
"react-day-picker": "^8.7.1",
......
projects/app/src/components/Markdown/img/Image.tsx
......@@ -3,11 +3,14 @@ import { Box, type ImageProps, Skeleton } from '@chakra-ui/react';
import MyPhotoView from '@fastgpt/web/components/common/Image/PhotoView';
import { useBoolean } from 'ahooks';
import { useTranslation } from 'next-i18next';
import type { AProps } from '../A';
const MdImage = ({ src, ...props }: { src?: string } & ImageProps) => {
const MdImage = ({
src,
...props
}: { src?: string } & ImageProps & { chatAuthData?: AProps['chatAuthData'] }) => {
const { t } = useTranslation();
const [isLoaded, { setTrue }] = useBoolean(false);
const [renderSrc, setRenderSrc] = useState(src);
if (src?.includes('base64') && !src.startsWith('data:image')) {
......
projects/app/src/components/Markdown/index.tsx
......@@ -54,7 +54,7 @@ const MarkdownRender = ({
}: Props) => {
const components = useCreation(() => {
return {
img: Image,
img: (props: any) => <Image {...props} alt={props.alt} chatAuthData={chatAuthData} />,
pre: RewritePre,
code: Code,
a: (props: any) => (
......@@ -145,8 +145,8 @@ function Code(e: any) {
return Component;
}
function Image({ src }: { src?: string }) {
return <MdImage src={src} />;
function Image({ src, chatAuthData }: { src?: string; chatAuthData?: AProps['chatAuthData'] }) {
return <MdImage src={src} chatAuthData={chatAuthData} />;
}
function RewritePre({ children }: any) {
......
projects/app/src/components/core/app/DatasetSelectModal.tsx
......@@ -32,12 +32,10 @@ import { useUserStore } from '@/web/support/user/useUserStore';
// Dataset selection modal component
export const DatasetSelectModal = ({
isOpen,
defaultSelectedDatasets = [],
onChange,
onClose
}: {
isOpen: boolean;
defaultSelectedDatasets: SelectedDatasetType[];
onChange: (e: SelectedDatasetType[]) => void;
onClose: () => void;
......@@ -142,7 +140,7 @@ export const DatasetSelectModal = ({
<MyModal
iconSrc="/imgs/workflow/db.png"
title={t('common:core.chat.Select dataset')}
isOpen={isOpen}
isOpen={true}
onClose={onClose}
minW="800px"
maxW={'800px'}
......
projects/app/src/components/core/app/formRender/FileSelector.tsx → projects/app/src/components/core/app/FileSelector/index.tsx
import type { DragEvent } from 'react';
import React, { useCallback, useEffect, useMemo, useState } from 'react';
import React, { useCallback, useEffect, useMemo, useState, useRef } from 'react';
import type { UserInputFileItemType } from '../../chat/ChatContainer/ChatBox/type';
import {
Box,
......@@ -29,7 +29,7 @@ import { useContextSelector } from 'use-context-selector';
import { POST } from '@/web/common/api/request';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { WorkflowAuthContext } from '@/components/core/chat/ChatContainer/context/workflowAuthContext';
import { WorkflowRuntimeContext } from '@/components/core/chat/ChatContainer/context/workflowRuntimeContext';
const FileSelector = ({
value,
......@@ -43,12 +43,10 @@ const FileSelector = ({
customFileExtensionList,
canLocalUpload,
canUrlUpload,
isDisabled = false,
onUploading
isDisabled = false
}: AppFileSelectConfigType & {
value: UserInputFileItemType[];
onChange: (e: any[]) => void;
onUploading?: (e: boolean) => void;
canLocalUpload?: boolean;
canUrlUpload?: boolean;
isDisabled?: boolean;
......@@ -57,9 +55,13 @@ const FileSelector = ({
const { toast } = useToast();
const { t } = useTranslation();
const appId = useContextSelector(WorkflowAuthContext, (v) => v.appId);
const chatId = useContextSelector(WorkflowAuthContext, (v) => v.chatId);
const outLinkAuthData = useContextSelector(WorkflowAuthContext, (v) => v.outLinkAuthData);
const appId = useContextSelector(WorkflowRuntimeContext, (v) => v.appId);
const chatId = useContextSelector(WorkflowRuntimeContext, (v) => v.chatId);
const outLinkAuthData = useContextSelector(WorkflowRuntimeContext, (v) => v.outLinkAuthData);
const setFileUploadingCount = useContextSelector(
WorkflowRuntimeContext,
(v) => v.setFileUploadingCount
);
const handleChangeFiles = useCallback(
(files: UserInputFileItemType[]) => {
......@@ -97,12 +99,14 @@ const FileSelector = ({
files.forEach((file) => {
file.status = 1;
file.process = 0;
});
handleChangeFiles(files);
await Promise.allSettled(
filterFiles.map(async (file) => {
if (!file.rawFile) return;
setFileUploadingCount((state) => state + 1);
try {
// Get Upload Post Presigned URL
......@@ -155,10 +159,12 @@ const FileSelector = ({
});
handleChangeFiles(files);
}
setFileUploadingCount((state) => state - 1);
})
);
},
[handleChangeFiles, appId, chatId, outLinkAuthData]
[handleChangeFiles, setFileUploadingCount, appId, chatId, outLinkAuthData]
);
// Selector props
......@@ -352,10 +358,6 @@ const FileSelector = ({
const isUploading = value.some((file) => !file.url && !file.error);
const disabled = isDisabled || isUploading;
useEffect(() => {
onUploading?.(isUploading);
}, [isUploading]);
return (
<>
{/* Selector */}
......@@ -423,7 +425,7 @@ const FileSelector = ({
zIndex={10}
/>
<Input
isDisabled={isMaxSelected || isDisabled}
isDisabled={isMaxSelected || disabled}
value={urlInput}
onChange={(e) => setUrlInput(e.target.value)}
onBlur={(e) => handleAddUrl(e.target.value)}
......@@ -467,7 +469,7 @@ const FileSelector = ({
{/* Status icon */}
<>
{!!file?.url || !!file?.error ? (
{!!file?.url || !!file?.error || file.process === undefined ? (
<IconButton
size={'xsSquare'}
borderRadius={'xs'}
......@@ -480,7 +482,7 @@ const FileSelector = ({
) : (
<HStack w={'24px'} h={'24px'} justifyContent={'center'}>
<CircularProgress
value={file?.process}
value={file.process}
color="primary.600"
bg={'white'}
size={'1.2rem'}
......
projects/app/src/components/core/app/VariableEditModal/index.tsx 0 → 100644
import React, { useCallback, useMemo, useEffect } from 'react';
import { Flex, Stack } from '@chakra-ui/react';
import { VariableInputEnum } from '@fastgpt/global/core/workflow/constants';
import type { VariableItemType } from '@fastgpt/global/core/app/type.d';
import { useForm } from 'react-hook-form';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useTranslation } from 'next-i18next';
import { useToast } from '@fastgpt/web/hooks/useToast';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import InputTypeConfig from '@/pageComponents/app/detail/WorkflowComponents/Flow/nodes/NodePluginIO/InputTypeConfig';
import { workflowSystemVariables } from '@/web/core/app/utils';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import InputTypeSelector from '@fastgpt/web/components/common/InputTypeSelector';
import { getVariableInputTypeList } from '@fastgpt/web/components/common/InputTypeSelector/configs';
import { addVariable } from '../VariableEdit';
import { useValidateFieldName, useSubmitErrorHandler } from '../utils/formValidation';
const VariableEditModal = ({
onClose,
variable,
variables,
onChange
}: {
onClose: () => void;
variable: VariableItemType;
variables: VariableItemType[];
onChange: (variables: VariableItemType[]) => void;
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const validateFieldName = useValidateFieldName();
const onSubmitError = useSubmitErrorHandler();
const form = useForm<VariableItemType>({
defaultValues: variable
});
const { setValue, reset, watch, getValues } = form;
const type = watch('type');
useEffect(() => {
reset(variable);
}, [variable, reset]);
const inputTypeList = useMemo(() => getVariableInputTypeList(), []);
const defaultValueType = useMemo(() => {
const item = inputTypeList.flat().find((item) => item.value === type);
return item?.defaultValueType;
}, [inputTypeList, type]);
const handleTypeChange = useCallback(
(newType: string) => {
const typeEnum = newType as VariableInputEnum;
const value = getValues();
const defaultValIsNumber = !isNaN(Number(value.defaultValue));
if (
typeEnum === VariableInputEnum.select ||
typeEnum === VariableInputEnum.multipleSelect ||
typeEnum === VariableInputEnum.file ||
(typeEnum === VariableInputEnum.numberInput && !defaultValIsNumber)
) {
setValue('defaultValue', '');
}
if (typeEnum === VariableInputEnum.datasetSelect && !value.datasetOptions) {
setValue('datasetOptions', []);
}
setValue('type', typeEnum);
},
[setValue, getValues]
);
const onSubmitSuccess = useCallback(
(data: VariableItemType, action: 'confirm' | 'continue') => {
data.label = data?.label?.trim();
const otherVariables = variables.filter((v) => v.key !== data.key);
const isValid = validateFieldName(data.label, {
existingKeys: otherVariables.flatMap((v) => [v.key, v.label]),
systemVariables: workflowSystemVariables,
currentKey: data.key
});
if (!isValid) {
return;
}
if (
data.type === VariableInputEnum.custom ||
data.type === VariableInputEnum.internal ||
data.type === VariableInputEnum.switch
) {
data.required = false;
} else {
data.valueType = inputTypeList
.flat()
.find((item) => item.value === data.type)?.defaultValueType;
}
Object.keys(data).forEach((key) => {
if (data[key as keyof VariableItemType] === undefined) {
delete data[key as keyof VariableItemType];
}
});
const submittedData = data.key ? data : { ...data, key: getNanoid(8) };
const updatedVariables =
submittedData.key && variables.some((v) => v.key === submittedData.key)
? variables.map((item) => (item.key === submittedData.key ? submittedData : item))
: [...variables, submittedData];
onChange(updatedVariables);
if (action === 'confirm') {
onClose();
} else if (action === 'continue') {
toast({
status: 'success',
title: t('common:add_success')
});
reset({
...addVariable(),
defaultValue: ''
});
}
},
[variables, inputTypeList, onChange, reset, onClose, validateFieldName, toast, t]
);
return (
<MyModal
iconSrc="core/app/simpleMode/variable"
title={t('common:core.module.Variable Setting')}
isOpen={true}
onClose={onClose}
maxW={['90vw', '1078px']}
w={'100%'}
isCentered
>
<Flex h={'560px'}>
<Stack p={8}>
<FormLabel color={'myGray.600'} fontWeight={'medium'}>
{t('workflow:Variable.Variable type')}
</FormLabel>
<InputTypeSelector
inputTypeList={inputTypeList}
selectedType={type}
onTypeChange={handleTypeChange}
/>
</Stack>
<InputTypeConfig
form={form}
type={'variable'}
isEdit={!!variable?.key}
inputType={type}
defaultValueType={defaultValueType}
onClose={onClose}
onSubmitSuccess={onSubmitSuccess}
onSubmitError={onSubmitError}
/>
</Flex>
</MyModal>
);
};
export default VariableEditModal;
projects/app/src/components/core/app/formRender/LabelAndForm.tsx
import type { BoxProps } from '@chakra-ui/react';
import { Box, Flex } from '@chakra-ui/react';
import { Box, Flex, FormControl, FormErrorMessage } from '@chakra-ui/react';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import React from 'react';
......@@ -66,32 +66,45 @@ const LabelAndFormRender = ({
if (typeof value === 'number' || typeof value === 'boolean') return true;
if (inputType === InputTypeEnum.password && props.minLength) {
if (!value || typeof value !== 'object' || !value.value) return false;
return value.value.length >= props.minLength;
if (value.value.length < props.minLength) {
return t(`common:min_length`, { minLenth: props.minLength });
}
return true;
}
if (!required) return true;
if (inputType === InputTypeEnum.fileSelect) {
if (!value || !Array.isArray(value) || value.length === 0) {
return t('common:required');
}
return true;
}
return !!value;
},
...(!!props?.minLength
? {
minLength: {
value: props.minLength,
message: t(`common:min_length`, { minLength: props.minLength })
message: t(`common:min_length`, { minLenth: props.minLength })
}
}
: {})
}}
render={({ field: { onChange, value }, fieldState: { error } }) => {
return (
<InputRender
inputType={inputType}
isRichText={false}
value={value}
onChange={onChange}
placeholder={placeholder}
isInvalid={!!error}
{...props}
/>
<FormControl isInvalid={!!error}>
<InputRender
inputType={inputType}
isRichText={false}
value={value}
onChange={onChange}
placeholder={placeholder}
isInvalid={!!error}
{...props}
/>
{error && <FormErrorMessage>{error.message}</FormErrorMessage>}
</FormControl>
);
}}
/>
......
projects/app/src/components/core/app/formRender/index.tsx
......@@ -12,10 +12,11 @@ import AIModelSelector from '../../../Select/AIModelSelector';
import TimeInput from './TimeInput';
import { useSafeTranslation } from '@fastgpt/web/hooks/useSafeTranslation';
import { isSecretValue } from '@fastgpt/global/common/secret/utils';
import FileSelector from '@/components/core/app/formRender/FileSelector';
import FileSelector from '../FileSelector/index';
import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
import { useMemoEnhance } from '@fastgpt/web/hooks/useMemoEnhance';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/type/io';
const InputRender = (props: InputRenderProps) => {
const {
......@@ -200,6 +201,7 @@ const InputRender = (props: InputRenderProps) => {
return (
<AIModelSelector
{...commonProps}
cacheModel={false}
list={
llmModelList?.map((item) => ({
value: item.model,
......@@ -217,7 +219,6 @@ const InputRender = (props: InputRenderProps) => {
value={files}
onChange={onChange}
isDisabled={isDisabled}
onUploading={props.setUploading}
maxFiles={props.maxFiles}
canSelectFile={props.canSelectFile}
canSelectImg={props.canSelectImg}
......@@ -232,7 +233,7 @@ const InputRender = (props: InputRenderProps) => {
}
if (inputType === InputTypeEnum.selectDataset) {
const list = props.dataset?.map((item: any) => ({
const list = props.datasetOptions?.map((item: SelectedDatasetType) => ({
label: item.name,
value: item.datasetId,
icon: item.avatar,
......@@ -240,7 +241,7 @@ const InputRender = (props: InputRenderProps) => {
}));
const selectedValues = Array.isArray(value)
? value.map((item: any) => (typeof item === 'string' ? item : item.datasetId))
? value.map((item: SelectedDatasetType) => item.datasetId)
: [];
return (
......@@ -252,14 +253,12 @@ const InputRender = (props: InputRenderProps) => {
onSelect={(selectedVals) => {
onChange(
selectedVals.map((val) => {
const item = list?.find((l) => l.value === val);
return item
? {
name: item.label,
datasetId: item.value,
icon: item.icon
}
: { name: val, datasetId: val, icon: '' };
const selectedItem = list?.find((item) => item.value === val);
return {
name: selectedItem?.label || '',
datasetId: selectedItem?.value || '',
icon: selectedItem?.icon || ''
};
})
);
}}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment