Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit f732bc52 by Xianquan Committed by GitHub
Options

feat: unify browser object storage multipart uploads (#7411) 

* feat: optimize object storage multipart upload

* feat: unify browser object storage uploads

* chore: update storage package and pro submodule

* Bump deps

* fix: configure home chat custom PDF parsing

* fix: extend dataset parsing timeout

* fix: ocr comments

* fix: resolve pro submodule conflict

* chore: remove invalid multipart benchmark command

* refactor: select multipart uploads by file size

---------

Co-authored-by: Archer <545436317@qq.com>
parent 8fe60f24
Showing with 3057 additions and 400 deletions
document/data/doc-last-modified.json
......@@ -329,8 +329,8 @@
"content/self-host/upgrading/4-15/4154.mdx": "2026-07-30T11:22:58+08:00",
"content/self-host/upgrading/4-15/4155.en.mdx": "2026-07-30T11:22:58+08:00",
"content/self-host/upgrading/4-15/4155.mdx": "2026-07-30T11:22:58+08:00",
"content/self-host/upgrading/4-16/41601.en.mdx": "2026-08-03T10:30:00+08:00",
"content/self-host/upgrading/4-16/41601.mdx": "2026-08-03T10:30:00+08:00",
"content/self-host/upgrading/4-16/41601.en.mdx": "2026-07-30T16:03:06+08:00",
"content/self-host/upgrading/4-16/41601.mdx": "2026-07-30T16:03:06+08:00",
"content/self-host/upgrading/outdated/40.en.mdx": "2026-07-25T00:27:20+08:00",
"content/self-host/upgrading/outdated/40.mdx": "2026-07-25T00:27:20+08:00",
"content/self-host/upgrading/outdated/41.en.mdx": "2026-07-25T00:27:20+08:00",
......@@ -473,4 +473,4 @@
"content/self-host/upgrading/upgrade-intruction.mdx": "2026-04-26T21:08:47+08:00",
"content/toc.en.mdx": "2026-07-29T19:55:49+08:00",
"content/toc.mdx": "2026-07-29T19:55:49+08:00"
}
}
\ No newline at end of file
packages/global/common/file/constants.ts
......@@ -6,6 +6,11 @@ export enum BucketNameEnum {
export const documentFileType = '.txt, .docx, .csv, .xlsx, .pdf, .md, .html, .pptx';
/** S3 Multipart 和各兼容 provider 统一支持的最大分片数量。 */
export const MAX_MULTIPART_PART_COUNT = 10_000;
/** 用于确认最终对象确实由当前 Multipart session 生成的不可变 metadata key。 */
export const MULTIPART_OBJECT_MARKER_METADATA_KEY = 'fastgptMultipartMarker';
/** 图片数据集创建/追加图片（multer 直传）与 ImageDataset、InsertImageModal 的 fileType 一致 */
export const datasetImageCollectionFileType = '.jpg, .jpeg, .png';
......
packages/global/common/file/s3/type.ts
import z from 'zod';
import { IntSchema } from '../../zod';
export const CreatePostPresignedUrlResponseSchema = z.object({
export const PresignFileUploadParamsSchema = z.object({
filename: z.string().min(1),
size: IntSchema.optional()
});
export type PresignFileUploadParams = z.infer<typeof PresignFileUploadParamsSchema>;
const CreatePostPresignedUrlBaseResponseSchema = z.object({
url: z.string().nonempty(),
key: z.string().nonempty(),
headers: z.record(z.string(), z.string()),
previewUrl: z.string().nonempty(),
maxSize: z.number().positive().optional() // bytes
});
export const CreatePostPresignedUrlSingleResponseSchema =
CreatePostPresignedUrlBaseResponseSchema.extend({
uploadMode: z.literal('single').meta({ description: '单请求 PUT 上传' })
});
export const CreatePostPresignedUrlMultipartResponseSchema =
CreatePostPresignedUrlBaseResponseSchema.extend({
uploadMode: z.literal('multipart').meta({ description: 'Multipart 分片上传' }),
completeUrl: z.string().min(1).meta({ description: '完成 Multipart 上传的接口地址' }),
abortUrl: z.string().min(1).meta({ description: '取消 Multipart 上传的接口地址' }),
partSize: IntSchema.positive().meta({
example: 8388608,
description: '单个分片大小，单位 byte'
}),
concurrency: IntSchema.positive().meta({
example: 3,
description: '建议的并发分片数'
}),
maxRetry: IntSchema.meta({ example: 3, description: '单个分片最大重试次数' })
});
export const CreatePostPresignedUrlResponseSchema = z
.discriminatedUnion('uploadMode', [
CreatePostPresignedUrlSingleResponseSchema,
CreatePostPresignedUrlMultipartResponseSchema
])
.meta({ description: 'S3 单 PUT 或 Multipart 上传参数' });
export type CreatePostPresignedUrlResponseType = z.infer<
typeof CreatePostPresignedUrlResponseSchema
>;
packages/global/common/file/s3TTL/type.ts
......@@ -3,4 +3,9 @@ export type S3TtlSchemaType = {
bucketName: string;
minioKey: string;
expiredTime: Date;
multipart?: {
uploadId: string;
objectMarker?: string;
totalSize?: number;
};
};
packages/global/openapi/core/dataset/file/api.ts
......@@ -124,6 +124,9 @@ export type GetRawTextPreviewChunksResponse = z.infer<typeof GetPreviewChunksRes
/* ============================================================================
* API: 获取知识库文件上传预签名 URL
* Route: POST /api/core/dataset/file/presignDatasetFilePostUrl
* Method: POST
* Description: 根据文件大小返回单 PUT 或 Multipart 上传会话
* Tags: ['Dataset', 'File', 'Write']
* ============================================================================ */
export const PresignDatasetFilePostUrlBodySchema = z.object({
filename: z.string().min(1).meta({
......@@ -133,17 +136,84 @@ export const PresignDatasetFilePostUrlBodySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '目标知识库 ID'
}),
size: IntSchema.positive().optional().meta({
example: 52428800,
description: '文件大小，单位 byte；旧客户端不传时继续使用单 PUT'
})
});
export type PresignDatasetFilePostUrlBody = z.infer<typeof PresignDatasetFilePostUrlBodySchema>;
export const PresignDatasetFilePostUrlResponseSchema = CreatePostPresignedUrlResponseSchema.meta({
description: 'S3 预签名上传 URL 及相关头信息'
description: 'S3 单 PUT 或 Multipart 上传参数'
});
export type PresignDatasetFilePostUrlResponse = z.infer<
typeof PresignDatasetFilePostUrlResponseSchema
>;
export const DatasetFileUploadTokenPathSchema = z.object({
token: z.string().min(20).max(64).meta({
example: 'wZ6rm5X4l6Ygm1oDQX8JbA',
description: '服务端签发的 opaque 上传 token'
})
});
export const UploadDatasetFileMultipartPartQuerySchema = z.object({
partNumber: IntSchema.positive().meta({
example: 1,
description: '待上传的分片编号，从 1 开始'
})
});
/* ============================================================================
* API: 完成知识库文件 Multipart 上传
* Route: POST /api/system/file/u/{token}/complete
* Method: POST
* Description: 校验分片清单并合并生成最终对象
* Tags: ['Dataset', 'File', 'Write']
* ============================================================================ */
export const CompleteDatasetFileMultipartUploadBodySchema = z.object({
parts: z
.array(
z.object({
partNumber: IntSchema.positive().meta({
example: 1,
description: '分片编号，从 1 开始'
}),
etag: z.string().min(1).meta({
example: '"etag-value"',
description: '对象存储返回的分片 ETag'
})
})
)
.min(1)
.meta({ description: '已上传分片清单，必须连续且按编号升序排列' })
});
export type CompleteDatasetFileMultipartUploadBody = z.infer<
typeof CompleteDatasetFileMultipartUploadBodySchema
>;
export const CompleteDatasetFileMultipartUploadResponseSchema = z.object({
key: z.string().min(1).meta({ description: '完成上传后的对象 key' })
});
export type CompleteDatasetFileMultipartUploadResponse = z.infer<
typeof CompleteDatasetFileMultipartUploadResponseSchema
>;
export const UploadDatasetFileMultipartPartResponseSchema = z.object({
etag: z.string().min(1).meta({ description: '当前分片的 ETag' })
});
export type UploadDatasetFileMultipartPartResponse = z.infer<
typeof UploadDatasetFileMultipartPartResponseSchema
>;
export const AbortDatasetFileMultipartUploadResponseSchema = z.undefined().meta({
description: '取消成功'
});
export type AbortDatasetFileMultipartUploadResponse = z.infer<
typeof AbortDatasetFileMultipartUploadResponseSchema
>;
/* ============================================================================
* API: 获取搜索测试图片上传预签名 URL
* Route: POST /api/core/dataset/file/presignSearchTestImage
......@@ -159,6 +229,10 @@ export const PresignSearchTestImageBodySchema = z.object({
filename: z.string().min(1).meta({
example: 'demo.png',
description: '待上传图片文件名'
}),
size: IntSchema.positive().optional().meta({
example: 1048576,
description: '文件大小，单位 byte；用于自动选择单 PUT 或 Multipart'
})
});
export type PresignSearchTestImageBody = z.infer<typeof PresignSearchTestImageBodySchema>;
......
packages/global/openapi/core/dataset/file/index.ts
import { z } from 'zod';
import type { OpenAPIPath } from '../../../type';
import { DevApiTagsMap } from '../../../tag';
import {
......@@ -6,10 +7,16 @@ import {
GetPreviewChunksBodySchema,
GetPreviewChunksResponseSchema,
GetRawTextPreviewChunksBodySchema,
AbortDatasetFileMultipartUploadResponseSchema,
CompleteDatasetFileMultipartUploadBodySchema,
CompleteDatasetFileMultipartUploadResponseSchema,
DatasetFileUploadTokenPathSchema,
PresignDatasetFilePostUrlBodySchema,
PresignDatasetFilePostUrlResponseSchema,
PresignSearchTestImageBodySchema,
PresignSearchTestImageResponseSchema
PresignSearchTestImageResponseSchema,
UploadDatasetFileMultipartPartQuerySchema,
UploadDatasetFileMultipartPartResponseSchema
} from './api';
export const DatasetFilePath: OpenAPIPath = {
......@@ -75,7 +82,7 @@ export const DatasetFilePath: OpenAPIPath = {
},
responses: {
200: {
description: '成功返回预签名上传 URL、key、请求头和最大文件大小',
description: '成功返回单 PUT 或 Multipart 上传参数',
content: {
'application/json': {
schema: PresignDatasetFilePostUrlResponseSchema
......@@ -85,6 +92,77 @@ export const DatasetFilePath: OpenAPIPath = {
}
}
},
'/system/file/u/{token}': {
put: {
summary: '上传知识库文件 Multipart 分片',
description: '通过 presign 返回的 token 代理上传单个 Multipart 分片',
tags: [DevApiTagsMap.datasetFile],
requestParams: {
path: DatasetFileUploadTokenPathSchema,
query: UploadDatasetFileMultipartPartQuerySchema.partial()
},
responses: {
200: {
description: '成功返回当前分片 ETag 或单 PUT 上传结果',
content: {
'application/json': {
schema: z.union([
UploadDatasetFileMultipartPartResponseSchema,
z.object({ success: z.literal(true) })
])
}
}
}
}
}
},
'/system/file/u/{token}/complete': {
post: {
summary: '完成知识库文件 Multipart 上传',
description: '校验分片清单并合并生成最终对象',
tags: [DevApiTagsMap.datasetFile],
requestParams: {
path: DatasetFileUploadTokenPathSchema
},
requestBody: {
content: {
'application/json': {
schema: CompleteDatasetFileMultipartUploadBodySchema
}
}
},
responses: {
200: {
description: '成功完成 Multipart 上传',
content: {
'application/json': {
schema: CompleteDatasetFileMultipartUploadResponseSchema
}
}
}
}
}
},
'/system/file/u/{token}/abort': {
post: {
summary: '取消知识库文件 Multipart 上传',
description: '取消未完成的 Multipart 上传并清理对象存储分片',
tags: [DevApiTagsMap.datasetFile],
requestParams: {
path: DatasetFileUploadTokenPathSchema
},
responses: {
200: {
description: '成功取消 Multipart 上传',
content: {
'application/json': {
schema: AbortDatasetFileMultipartUploadResponseSchema
}
}
}
}
}
},
'/core/dataset/file/presignSearchTestImage': {
post: {
summary: '获取搜索测试图片上传预签名 URL',
......
packages/service/common/file/parseTimeout.ts 0 → 100644
import { serviceEnv } from '../../env';
export const MIN_BACKEND_FILE_OPERATION_TIMEOUT_SECONDS = 600;
/**
* 获取后端文件读取和解析使用的有效总 timeout。
*
* 文件解析 worker 继续使用原始环境变量值；HTTP provider 和外部文件流
* 至少保留 10 分钟的等待时间，避免较小的 worker 配置反向缩短已有行为。
*/
export const getBackendFileOperationTimeoutMs = () =>
Math.max(MIN_BACKEND_FILE_OPERATION_TIMEOUT_SECONDS, serviceEnv.PARSE_FILE_TIMEOUT_SECONDS) *
1000;
packages/service/common/file/read/utils.ts
......@@ -11,6 +11,7 @@ import { readRawContentFromBuffer } from '../../../worker/function';
import { getLogger, LogCategories } from '../../logger';
import { getImageBuffer } from '../image/utils';
import { uploadParsedPdfImage } from './image';
import { getBackendFileOperationTimeoutMs } from '../parseTimeout';
const logger = getLogger(LogCategories.MODULE.DATASET.FILE);
......@@ -132,7 +133,7 @@ export const readFileContentByBuffer = async ({
markdown: string;
error?: object | string;
}>(url, data, {
timeout: 600000,
timeout: getBackendFileOperationTimeoutMs(),
headers: {
...data.getHeaders(),
Authorization: token ? `Bearer ${token}` : undefined
......
packages/service/common/s3/accessLink/type.ts
import z from 'zod';
import { StorageObjectKeySchema, UploadConstraintsSchema } from '../contracts/type';
import { MAX_MULTIPART_PART_COUNT } from '@fastgpt/global/common/file/constants';
import { StorageObjectKeySchema } from '../contracts/type';
import { UploadFileHintSchema, UploadPolicySchema } from '../uploadPolicy/type';
import { S3_DOWNLOAD_URL_BATCH_MAX_SIZE } from '@fastgpt-sdk/storage/access-link';
......@@ -74,42 +75,92 @@ export type VerifiedS3DownloadAccess = z.infer<typeof VerifiedS3DownloadAccessSc
export const S3UploadTokenSchema = UrlSafeTokenSchema.min(20).max(64);
export const S3UploadTokenHashSchema = HexSha256Schema;
const PositiveSafeIntegerSchema = z
.number()
.int()
.positive()
.refine(Number.isSafeInteger, 'Must be a safe integer');
export const S3MultipartUploadSessionSchema = z.object({
uploadId: z.string().min(1),
partSize: PositiveSafeIntegerSchema,
totalSize: PositiveSafeIntegerSchema,
status: z.enum(['active', 'completing', 'completed', 'aborted']),
completionAttemptId: z.string().min(1).optional(),
completingAt: z.coerce.date().optional(),
completedAt: z.coerce.date().optional(),
abortedAt: z.coerce.date().optional()
});
export type S3MultipartUploadSession = z.infer<typeof S3MultipartUploadSessionSchema>;
/** 创建 session 时只允许服务端初始化的 active 状态，禁止伪造完成/取消时间。 */
const CreateS3MultipartUploadSessionSchema = z
.object({
uploadId: z.string().min(1),
partSize: PositiveSafeIntegerSchema,
totalSize: PositiveSafeIntegerSchema,
status: z.literal('active'),
completingAt: z.never().optional(),
completedAt: z.never().optional(),
abortedAt: z.never().optional()
})
.strict()
.superRefine(({ totalSize, partSize }, context) => {
if (Math.ceil(totalSize / partSize) > MAX_MULTIPART_PART_COUNT) {
context.addIssue({
code: z.ZodIssueCode.custom,
path: ['partSize'],
message: `Multipart upload cannot exceed ${MAX_MULTIPART_PART_COUNT} parts`
});
}
});
export const S3UploadSessionSchema = z.object({
tokenHash: S3UploadTokenHashSchema,
bucketName: S3AccessBucketNameSchema,
objectKey: S3AccessObjectKeySchema,
maxSize: z.number().positive(),
uploadConstraints: UploadConstraintsSchema,
uploadPolicy: UploadPolicySchema.optional(),
maxSize: PositiveSafeIntegerSchema,
uploadPolicy: UploadPolicySchema,
fileHint: UploadFileHintSchema.optional(),
metadata: z.record(z.string(), z.string()).optional(),
createTime: z.coerce.date(),
expiresAt: z.coerce.date(),
usedAt: z.coerce.date().optional(),
revokedAt: z.coerce.date().optional()
revokedAt: z.coerce.date().optional(),
multipart: S3MultipartUploadSessionSchema.optional()
});
export type S3UploadSessionType = z.infer<typeof S3UploadSessionSchema>;
export const CreateS3UploadAccessUrlParamsSchema = z.object({
bucketName: S3AccessBucketNameSchema,
objectKey: S3AccessObjectKeySchema,
expiredTime: z.coerce.date(),
maxSize: z.number().positive(),
uploadConstraints: UploadConstraintsSchema,
uploadPolicy: UploadPolicySchema.optional(),
fileHint: UploadFileHintSchema.optional(),
metadata: z.record(z.string(), z.string()).optional()
});
export const CreateS3UploadAccessUrlParamsSchema = z
.object({
bucketName: S3AccessBucketNameSchema,
objectKey: S3AccessObjectKeySchema,
expiredTime: z.coerce.date(),
maxSize: PositiveSafeIntegerSchema,
uploadPolicy: UploadPolicySchema,
fileHint: UploadFileHintSchema.optional(),
metadata: z.record(z.string(), z.string()).optional(),
multipart: CreateS3MultipartUploadSessionSchema.optional()
})
.superRefine(({ maxSize, multipart }, context) => {
if (multipart && multipart.totalSize > maxSize) {
context.addIssue({
code: z.ZodIssueCode.custom,
path: ['multipart', 'totalSize'],
message: 'Multipart total size exceeds maxSize'
});
}
});
export type CreateS3UploadAccessUrlParams = z.infer<typeof CreateS3UploadAccessUrlParamsSchema>;
export const S3ProxyUploadPayloadSchema = S3UploadSessionSchema.pick({
bucketName: true,
objectKey: true,
maxSize: true,
uploadConstraints: true,
uploadPolicy: true,
fileHint: true,
metadata: true
metadata: true,
multipart: true
});
export type S3ProxyUploadPayload = z.infer<typeof S3ProxyUploadPayloadSchema>;
......
packages/service/common/s3/accessLink/uploadSession/schema.ts
import { getLogger, LogCategories } from '../../../logger';
import { defineIndex, getMongoModel, Schema } from '../../../mongo';
import type { S3UploadSessionType } from '../type';
export const S3UploadSessionCollectionName = 's3_upload_sessions';
const logger = getLogger(LogCategories.INFRA.MONGO);
const S3UploadSessionMongoSchema = new Schema({
tokenHash: {
type: String,
......@@ -23,11 +20,10 @@ const S3UploadSessionMongoSchema = new Schema({
type: Number,
required: true
},
uploadConstraints: {
uploadPolicy: {
type: Object,
required: true
},
uploadPolicy: Object,
fileHint: Object,
metadata: Object,
createTime: {
......@@ -39,7 +35,11 @@ const S3UploadSessionMongoSchema = new Schema({
required: true
},
usedAt: Date,
revokedAt: Date
revokedAt: Date,
multipart: {
type: Object,
required: false
}
});
defineIndex(S3UploadSessionMongoSchema, {
......
packages/service/common/s3/accessLink/uploadSession/service.ts
......@@ -8,7 +8,7 @@ import { s3AccessLinkService } from '../accessLinkService';
/**
* 创建一次上传会话并返回短上传 URL。
*
* 上传 session 承载 maxSize/uploadConstraints/metadata 等服务端策略，不按 objectKey 复用，
* 上传 session 承载 maxSize/uploadPolicy/metadata 等服务端策略，不按 objectKey 复用，
* 避免重复 PUT、覆盖对象和策略变更不生效。
*/
export const createS3UploadAccessUrl = async (params: unknown) => {
......@@ -20,6 +20,48 @@ export const verifyS3UploadSessionToken = async (token: string): Promise<S3Proxy
return S3ProxyUploadPayloadSchema.parse(await s3AccessLinkService.verifyUploadToken(token));
};
/** 校验并返回 Multipart session，供分片、完成和取消流程复用同一 token。 */
export const verifyS3MultipartUploadSessionToken = async (token: string) => {
const payload = await verifyS3UploadSessionToken(token);
if (!payload.multipart) {
throw new Error('Not a multipart upload session');
}
return payload;
};
/** 将已占用完成权的 Multipart session 从 completing 原子地标记为 completed。 */
export const markS3MultipartUploadCompleted = (
token: string,
completionAttemptId: string,
completedAt?: Date
) => {
return s3AccessLinkService.markMultipartCompleted({ token, completionAttemptId, completedAt });
};
/** 原子地占用 Multipart 完成权，防止 complete 与 abort 并发操作同一个 provider upload。 */
export const markS3MultipartUploadCompleting = (token: string, completingAt?: Date) => {
return s3AccessLinkService.markMultipartCompleting({ token, completingAt });
};
/** 仅在 completing 租约过期后重新占用完成权，允许网络超时后的 complete 重试。 */
export const retryS3MultipartUploadCompleting = (token: string, reclaimBefore: Date) => {
return s3AccessLinkService.markMultipartCompleting({ token, reclaimBefore });
};
/** 将已占用完成权但 provider complete 失败的 session 标记为 aborted。 */
export const markS3MultipartUploadCompleteFailed = (
token: string,
completionAttemptId: string,
abortedAt?: Date
) => {
return s3AccessLinkService.markMultipartCompleteFailed({ token, completionAttemptId, abortedAt });
};
/** 将 Multipart session 从 active 原子地标记为 aborted。 */
export const markS3MultipartUploadAborted = (token: string, abortedAt?: Date) => {
return s3AccessLinkService.markMultipartAborted({ token, abortedAt });
};
export const revokeS3UploadSessionToken = (token: string) => {
return s3AccessLinkService.revokeUploadToken(token);
};
packages/service/common/s3/accessLink/uploadSession/store.ts
......@@ -29,6 +29,82 @@ export const mongoS3UploadSessionStore: S3UploadSessionStore = {
}
);
},
markMultipartCompleting: async ({
tokenHash,
completionAttemptId,
completingAt,
reclaimBefore
}) => {
const statusFilter = reclaimBefore
? {
$or: [
{ 'multipart.status': 'active' },
{
'multipart.status': 'completing',
$or: [
{ 'multipart.completingAt': { $lte: reclaimBefore } },
{ 'multipart.completingAt': { $exists: false } }
]
}
]
}
: { 'multipart.status': 'active' };
const result = await MongoS3UploadSession.updateOne(
{ tokenHash, ...statusFilter },
{
$set: {
'multipart.status': 'completing',
'multipart.completionAttemptId': completionAttemptId,
'multipart.completingAt': completingAt
}
}
);
return result.modifiedCount === 1 ? completionAttemptId : null;
},
markMultipartCompleted: async ({ tokenHash, completionAttemptId, completedAt }) => {
const result = await MongoS3UploadSession.updateOne(
{
tokenHash,
'multipart.status': 'completing',
'multipart.completionAttemptId': completionAttemptId
},
{
$set: {
'multipart.status': 'completed',
'multipart.completedAt': completedAt
}
}
);
return result.modifiedCount === 1;
},
markMultipartCompleteFailed: async ({ tokenHash, completionAttemptId, abortedAt }) => {
const result = await MongoS3UploadSession.updateOne(
{
tokenHash,
'multipart.status': 'completing',
'multipart.completionAttemptId': completionAttemptId
},
{
$set: {
'multipart.status': 'aborted',
'multipart.abortedAt': abortedAt
}
}
);
return result.modifiedCount === 1;
},
markMultipartAborted: async ({ tokenHash, abortedAt }) => {
const result = await MongoS3UploadSession.updateOne(
{ tokenHash, 'multipart.status': 'active' },
{
$set: {
'multipart.status': 'aborted',
'multipart.abortedAt': abortedAt
}
}
);
return result.modifiedCount === 1;
},
revoke: async ({ tokenHash, revokedAt }) => {
await MongoS3UploadSession.updateOne(
{ tokenHash },
......
packages/service/common/s3/config/constants.ts
......@@ -17,6 +17,17 @@ export const getSystemMaxFileSize = () => global.feConfigs.uploadFileMaxSize ||
export const S3_KEY_PATH_INVALID_CHARS = /[|\\/]/;
/** 达到该大小的浏览器直传文件切换到代理层 Multipart 上传。 */
export const S3_MULTIPART_UPLOAD_THRESHOLD_BYTES = 32 * 1024 * 1024;
/** 首期固定分片大小，超过 S3/OSS/COS 常见最小分片限制。 */
export const S3_MULTIPART_PART_SIZE_BYTES = 8 * 1024 * 1024;
export const S3_MULTIPART_CONCURRENCY = 3;
export const S3_MULTIPART_MAX_RETRY = 3;
export { MAX_MULTIPART_PART_COUNT } from '@fastgpt/global/common/file/constants';
export const S3_MULTIPART_SESSION_EXPIRE_HOURS = 3;
/** provider complete 发生网络超时后，保留完成权的短租约，过期后允许同一 uploadId 重试。 */
export const S3_MULTIPART_COMPLETING_LEASE_MS = 5 * 60 * 1000;
type BucketStorageOptions = {
publicBucket: string;
privateBucket: string;
......
packages/service/common/s3/contracts/type.ts
......@@ -6,6 +6,7 @@ import {
UploadPolicySchema
} from '../uploadPolicy/type';
import { assertStorageObjectKey } from '@fastgpt-sdk/storage';
import type { MultipartUploadPart, StorageUploadBody } from '@fastgpt-sdk/storage';
/** FastGPT 入口与底层 Storage SDK 共用同一套对象 key 规范。 */
export const StorageObjectKeySchema = z.string().superRefine((key, context) => {
......@@ -64,18 +65,93 @@ export type CreatePostPresignedUrlParams = z.infer<typeof CreatePostPresignedUrl
export const CreatePostPresignedUrlOptionsSchema = z.object({
expiredHours: z.number().positive().optional().describe('小时'),
maxFileSize: z.number().positive().optional().describe('MB'),
uploadConstraints: UploadConstraintsInputSchema.optional()
uploadPolicy: UploadPolicySchema.optional()
});
export type CreatePostPresignedUrlOptions = z.infer<typeof CreatePostPresignedUrlOptionsSchema>;
export const CreatePostPresignedUrlResultSchema = z.object({
const CreatePostPresignedUrlBaseResultSchema = z.object({
url: z.string().nonempty(),
key: z.string().nonempty(),
headers: z.record(z.string(), z.string()),
previewUrl: z.string().nonempty(),
maxSize: z.number().positive().optional()
});
export const CreatePresignedPutUrlResultSchema = CreatePostPresignedUrlBaseResultSchema.extend({
uploadMode: z.literal('single')
});
export type CreatePresignedPutUrlResult = z.infer<typeof CreatePresignedPutUrlResultSchema>;
export const CreatePostPresignedUrlResultSchema = z.discriminatedUnion('uploadMode', [
CreatePresignedPutUrlResultSchema,
CreatePostPresignedUrlBaseResultSchema.extend({
uploadMode: z.literal('multipart'),
completeUrl: z.string().nonempty(),
abortUrl: z.string().nonempty(),
partSize: z.number().int().positive(),
concurrency: z.number().int().positive(),
maxRetry: z.number().int().nonnegative()
})
]);
export type CreatePostPresignedUrlResult = z.infer<typeof CreatePostPresignedUrlResultSchema>;
export const CreateMultipartUploadAccessUrlParamsSchema = CreatePostPresignedUrlParamsSchema.extend(
{
size: z
.number()
.int()
.positive()
.refine(Number.isSafeInteger, 'Multipart size must be a safe integer')
}
);
export type CreateMultipartUploadAccessUrlParams = z.infer<
typeof CreateMultipartUploadAccessUrlParamsSchema
>;
export const CreateMultipartUploadAccessUrlOptionsSchema =
CreatePostPresignedUrlOptionsSchema.extend({
partSize: z
.number()
.int()
.positive()
.refine(Number.isSafeInteger, 'Multipart part size must be a safe integer')
.optional(),
concurrency: z.number().int().positive().optional(),
maxRetry: z.number().int().nonnegative().optional()
});
export type CreateMultipartUploadAccessUrlOptions = z.infer<
typeof CreateMultipartUploadAccessUrlOptionsSchema
>;
export const CreateMultipartUploadAccessUrlResultSchema =
CreatePostPresignedUrlBaseResultSchema.extend({
uploadMode: z.literal('multipart'),
completeUrl: z.string().nonempty(),
abortUrl: z.string().nonempty(),
partSize: z.number().int().positive(),
concurrency: z.number().int().positive(),
maxRetry: z.number().int().nonnegative()
});
export type CreateMultipartUploadAccessUrlResult = z.infer<
typeof CreateMultipartUploadAccessUrlResultSchema
>;
export type UploadMultipartPartAccessParams = {
token: string;
partNumber: number;
body: StorageUploadBody;
contentLength: number;
};
export type CompleteMultipartUploadAccessParams = {
token: string;
parts: MultipartUploadPart[];
};
export type AbortMultipartUploadAccessParams = {
token: string;
};
export const CreateGetPresignedUrlParamsSchema = z.object({
key: StorageObjectKeySchema,
expiredHours: z.number().positive().optional(),
......
packages/service/common/s3/lifecycle/cleanup.ts
......@@ -6,6 +6,10 @@ import { TimerIdEnum } from '../../system/timerLock/constants';
const logger = getLogger(LogCategories.INFRA.S3);
/**
* 扫描过期的 S3 TTL 记录；Multipart 记录先 Abort 远端分片，普通对象才提交删除任务。
* bucket 暂不可用时保留 Multipart 记录，等待后续 cron 重试，避免丢失 uploadId。
*/
export async function clearExpiredMinioFiles() {
try {
const expiredFiles = await MongoS3TTL.find({
......@@ -25,26 +29,49 @@ export async function clearExpiredMinioFiles() {
try {
const bucketName = file.bucketName;
const bucket = global.s3BucketMap[bucketName];
const multipartUploadId = file.multipart?.uploadId?.trim();
const hasMultipartMarker = file.multipart !== undefined && file.multipart !== null;
if (bucket) {
await bucket.addDeleteJob({ key: file.minioKey });
if (hasMultipartMarker) {
if (!multipartUploadId) {
throw new Error('Invalid Multipart TTL record: uploadId is missing');
}
// Multipart TTL 只代表未完成的远端分片，不应提交最终对象删除任务。
await bucket.abortMultipartUploadByUploadId({
key: file.minioKey,
uploadId: multipartUploadId,
objectMarker: file.multipart?.objectMarker,
totalSize: file.multipart?.totalSize
});
} else {
await bucket.addDeleteJob({ key: file.minioKey });
}
await MongoS3TTL.deleteOne({ _id: file._id });
success++;
logger.info('Deleted expired S3 object', {
logger.info('Cleaned expired S3 file', {
key: file.minioKey,
bucketName: file.bucketName
bucketName: file.bucketName,
multipart: hasMultipartMarker
});
} else {
logger.warn('S3 bucket not found for expired file', {
bucketName: file.bucketName,
key: file.minioKey
});
await MongoS3TTL.deleteOne({ minioKey: file.minioKey, bucketName: file.bucketName });
logger.info('Cleanup the expired document in MongoDB of S3 TTL', {
key: file.minioKey,
bucketName: file.bucketName
});
if (!hasMultipartMarker) {
await MongoS3TTL.deleteOne({ minioKey: file.minioKey, bucketName: file.bucketName });
logger.info('Removed expired S3 TTL without bucket', {
key: file.minioKey,
bucketName: file.bucketName
});
} else {
logger.info('Deferred expired Multipart cleanup because bucket is unavailable', {
key: file.minioKey,
bucketName: file.bucketName
});
}
}
} catch (error) {
fail++;
......
packages/service/common/s3/models/ttl.ts
......@@ -3,6 +3,27 @@ import { type S3TtlSchemaType } from '@fastgpt/global/common/file/s3TTL/type';
const collectionName = 's3_ttls';
const S3MultipartTTLSchema = new Schema(
{
uploadId: {
type: String,
required: true,
trim: true
},
objectMarker: {
type: String,
required: false,
trim: true
},
totalSize: {
type: Number,
required: false,
min: 1
}
},
{ _id: false }
);
const S3TTLSchema = new Schema({
bucketName: {
type: String,
......@@ -15,6 +36,10 @@ const S3TTLSchema = new Schema({
expiredTime: {
type: Date,
required: true
},
multipart: {
type: S3MultipartTTLSchema,
required: false
}
});
......
packages/service/common/s3/sources/avatar/index.ts
......@@ -3,7 +3,7 @@ import type { ClientSession } from 'mongoose';
import { getFileS3Key } from '../../utils';
import { MongoS3TTL } from '../../models/ttl';
import { S3PublicBucket } from '../../buckets/public';
import { avatarAllowedExtensions } from '../../utils/uploadConstraints';
import { avatarAllowedExtensions, createUploadConstraints } from '../../utils/uploadConstraints';
class S3AvatarSource extends S3PublicBucket {
constructor() {
......@@ -17,22 +17,28 @@ class S3AvatarSource extends S3PublicBucket {
async createUploadAvatarURL({
filename,
teamId,
size,
autoExpired = true
}: {
filename: string;
teamId: string;
size?: number;
autoExpired?: boolean;
}) {
const { fileKey } = getFileS3Key.avatar({ teamId, filename });
const uploadPolicy = createUploadConstraints({
filename,
uploadConstraints: {
allowedExtensions: avatarAllowedExtensions
}
});
return this.createPresignedPutUrl(
{ filename, rawKey: fileKey },
return this.createUploadAccessUrl(
{ filename, rawKey: fileKey, ...(size !== undefined ? { size } : {}) },
{
expiredHours: autoExpired ? 1 : undefined, // 1 Hours
maxFileSize: 5, // 5MB
uploadConstraints: {
allowedExtensions: avatarAllowedExtensions
}
uploadPolicy
}
);
}
......
packages/service/common/s3/sources/chat/index.ts
......@@ -14,6 +14,7 @@ import path from 'path';
import { getFormatedFilename } from '../../utils';
import type { ChatS3SourceType } from './type';
import { ChatSourceTypeEnum } from '@fastgpt/global/core/chat/constants';
import { createUploadConstraints } from '../../utils/uploadConstraints';
const getChatFileS3Key = ({
sourceType,
......@@ -98,7 +99,18 @@ export class S3ChatSource extends S3PrivateBucket {
extensionRules
} = ChatFileUploadSchema.parse(params);
const { fileKey } = getChatFileS3Key({ sourceType, sourceId, chatId, uId, filename });
return await this.createPresignedPutUrl(
const uploadPolicy = createUploadConstraints({
filename,
...(contentType ? { contentType } : {}),
...(declaredExtension ? { declaredExtension } : {}),
...(declaredFilename ? { declaredFilename } : {}),
...(size !== undefined ? { size } : {}),
uploadConstraints: {
allowedExtensions,
extensionRules
}
});
return await this.createUploadAccessUrl(
{
rawKey: fileKey,
filename,
......@@ -110,10 +122,7 @@ export class S3ChatSource extends S3PrivateBucket {
{
expiredHours: expiredTime ? differenceInHours(expiredTime, new Date()) : 1,
maxFileSize,
uploadConstraints: {
allowedExtensions,
extensionRules
}
uploadPolicy
}
);
}
......
packages/service/common/s3/sources/dataset/index.ts
......@@ -19,7 +19,7 @@ import { getLogger, LogCategories } from '../../../logger';
import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
import { readFileContentByBuffer } from '../../../file/read/utils';
import { ensureTextContentTypeCharset, isTextLikeFile, resolveMimeType } from '../../utils/mime';
import { datasetAllowedExtensions } from '../../utils/uploadConstraints';
import { createUploadConstraints, datasetAllowedExtensions } from '../../utils/uploadConstraints';
import { getFileS3Key, truncateFilename } from '../../utils';
import { isAuthorizedDatasetFileS3Key } from './key';
import type { S3RawTextSource } from '../rawText';
......@@ -53,18 +53,30 @@ export class S3DatasetSource extends S3PrivateBucket {
return await this.createPreviewUrl({ key, expiredHours, responseContentType });
}
// 上传链接
async createUploadDatasetFileURL(params: CreateUploadDatasetFileParams) {
const { filename, datasetId, maxFileSize } = CreateUploadDatasetFileParamsSchema.parse(params);
const { filename, datasetId, maxFileSize, size } =
CreateUploadDatasetFileParamsSchema.parse(params);
const { fileKey } = getFileS3Key.dataset({ datasetId, filename });
return await this.createPresignedPutUrl(
{ rawKey: fileKey, filename },
const uploadPolicy = createUploadConstraints({
filename,
source: 'local-file',
...(size !== undefined ? { size } : {}),
uploadConstraints: {
allowedExtensions: datasetAllowedExtensions
}
});
return await this.createUploadAccessUrl(
{
rawKey: fileKey,
filename,
source: 'local-file',
...(size !== undefined ? { size } : {})
},
{
expiredHours: 3,
maxFileSize,
uploadConstraints: {
allowedExtensions: datasetAllowedExtensions
}
uploadPolicy
}
);
}
......
packages/service/common/s3/sources/dataset/type.ts
......@@ -5,7 +5,8 @@ import z from 'zod';
export const CreateUploadDatasetFileParamsSchema = z.object({
filename: z.string().nonempty(),
datasetId: ObjectIdSchema,
maxFileSize: z.number().positive().optional()
maxFileSize: z.number().positive().optional(),
size: z.number().int().positive().optional()
});
export type CreateUploadDatasetFileParams = z.infer<typeof CreateUploadDatasetFileParamsSchema>;
......
packages/service/common/s3/utils/assert.ts 0 → 100644
import { InvalidObjectNameError, InvalidXMLError, S3Error } from 'minio';
import type { MultipartUploadPart, S3MultipartUploadSession } from '@fastgpt-sdk/storage';
type MultipartUploadStatus = S3MultipartUploadSession['status'];
/** 只允许 active session 接收新的 Multipart 分片。 */
export const assertActiveMultipartSession = (status: MultipartUploadStatus) => {
if (status !== 'active') {
throw new Error(`Multipart upload session is ${status}`);
}
};
/** complete 允许复用已经被当前请求占用的 completing session。 */
export const assertCompletableMultipartSession = (status: MultipartUploadStatus) => {
if (status !== 'active' && status !== 'completing') {
throw new Error(`Multipart upload session is ${status}`);
}
};
/** 完成前校验分片编号和 ETag，确保客户端不能合并缺失或重复的 part。 */
export const assertCompleteMultipartParts = ({
parts,
totalSize,
partSize
}: {
parts: MultipartUploadPart[];
totalSize: number;
partSize: number;
}) => {
const expectedPartCount = Math.ceil(totalSize / partSize);
if (parts.length !== expectedPartCount) {
throw new Error('Multipart parts count does not match total size');
}
parts.forEach((part, index) => {
const expectedPartNumber = index + 1;
if (part.partNumber !== expectedPartNumber || !part.etag.trim()) {
throw new Error('Multipart parts must be continuous and have an ETag');
}
});
};
/** 根据 session 计算指定分片的准确长度，只有最后一个分片允许小于 partSize。 */
export const getExpectedMultipartPartLength = ({
partNumber,
totalSize,
partSize
}: {
partNumber: number;
totalSize: number;
partSize: number;
}) => {
const partCount = Math.ceil(totalSize / partSize);
if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > partCount) {
throw new Error('Multipart part number is out of range');
}
return partNumber === partCount ? totalSize - partSize * (partCount - 1) : partSize;
};
/** 将对象存储返回的常见“对象不存在”错误转换为可幂等处理的判断结果。 */
export const isFileNotFoundError = (error: unknown): boolean => {
if (error && typeof error === 'object') {
const value = error as {
code?: unknown;
name?: unknown;
status?: unknown;
statusCode?: unknown;
$metadata?: { httpStatusCode?: unknown };
};
const statusCodes = [value.status, value.statusCode, value.$metadata?.httpStatusCode].map(
(status) => Number(status)
);
if (statusCodes.includes(404)) return true;
if (
[value.code, value.name].some((item) =>
['NotFound', 'NoSuchKey', 'NoSuchObject'].includes(String(item))
)
) {
return true;
}
}
if (error instanceof S3Error) {
return (
error.code === 'NoSuchKey' ||
error.code === 'InvalidObjectName' ||
error.message === 'Not Found' ||
error.message ===
'The request signature we calculated does not match the signature you provided. Check your key and signing method.' ||
error.message.includes('Resource name contains bad components') ||
error.message.includes('Object name contains unsupported characters.')
);
}
if (error instanceof InvalidObjectNameError || error instanceof InvalidXMLError) {
return true;
}
return false;
};
packages/service/common/s3/utils/uploadConstraints.ts
......@@ -4,7 +4,7 @@ import {
type FileExtensionKeyType
} from '@fastgpt/global/core/app/constants';
import type { AppFileSelectConfigType } from '@fastgpt/global/core/app/type/config.schema';
import type { UploadConstraintsInput, UploadConstraints } from '../contracts/type';
import type { UploadConstraintsInput } from '../contracts/type';
import {
createUploadExtensionRulesFromFileSelectConfig,
normalizeAllowedExtensions,
......@@ -12,6 +12,7 @@ import {
parseAllowedExtensions
} from '../uploadPolicy/utils';
import { createUploadPolicy } from '../uploadPolicy/service';
import type { UploadPolicy } from '../uploadPolicy/type';
const uploadConfigKeys: FileExtensionKeyType[] = [
'canSelectFile',
......@@ -61,7 +62,7 @@ export const createUploadConstraints = ({
declaredFilename?: string;
source?: 'local-file' | 'remote-url' | 'server-generated';
size?: number;
}): UploadConstraints => {
}): UploadPolicy => {
return createUploadPolicy({
hint: {
filename,
......
packages/service/common/s3/validation/upload.ts
......@@ -27,8 +27,8 @@ export const getUploadInspectBytes = (
/**
* 校验上传文件内容并返回最终写入 metadata 的文件信息。
*
* 兼容旧调用方的 `filename + uploadConstraints` 入参；新短上传链路应优先传
* `fileHint + uploadPolicy`，避免把客户端 hint、服务端策略和内容 evidence 混在一起。
* 新短上传链路传入已经固定的 `fileHint + uploadPolicy`；`uploadConstraints` 仅保留给
* 旧的直接调用方现场构建策略，不作为 upload session 或 proxy payload 的字段。
*/
export async function validateUploadFile({
buffer,
......@@ -39,7 +39,7 @@ export async function validateUploadFile({
}: {
buffer: Buffer;
filename?: string;
uploadConstraints: UploadConstraints;
uploadConstraints?: UploadConstraints;
uploadPolicy?: UploadPolicy;
fileHint?: UploadFileHint;
}) {
......@@ -47,7 +47,7 @@ export async function validateUploadFile({
filename: filename || 'file'
};
const policy =
uploadPolicy ||
uploadPolicy ??
createUploadPolicy({
hint,
uploadConstraints
......
packages/service/core/dataset/read.ts
......@@ -20,6 +20,7 @@ import { getFileS3Key, isS3ObjectKey } from '../../common/s3/utils';
import { isAuthorizedDatasetFileS3Key } from '../../common/s3/sources/dataset/key';
import { getLogger, LogCategories } from '../../common/logger';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { getBackendFileOperationTimeoutMs } from '../../common/file/parseTimeout';
const logger = getLogger(LogCategories.MODULE.DATASET.FILE);
......@@ -73,10 +74,20 @@ export const readFileRawTextByUrl = async ({
maxFileSize?: number;
}) => {
const extension = parseFileExtensionFromUrl(url);
const downloadTimeoutMs = getBackendFileOperationTimeoutMs();
const downloadDeadline = Date.now() + downloadTimeoutMs;
const getRemainingDownloadMs = () => Math.max(0, downloadDeadline - Date.now());
const getDownloadRequestTimeout = (maxTimeoutMs: number) => {
const remainingMs = getRemainingDownloadMs();
if (remainingMs <= 0) {
throw new Error(`File download timeout after ${downloadTimeoutMs / 1000} seconds`);
}
return Math.min(maxTimeoutMs, remainingMs);
};
// Check file size
try {
const headResponse = await axios.head(url, { timeout: 10000 });
const headResponse = await axios.head(url, { timeout: getDownloadRequestTimeout(10000) });
const contentLength = parseInt(
getAxiosHeaderValue(headResponse.headers['content-length']) || '0'
);
......@@ -87,6 +98,7 @@ export const readFileRawTextByUrl = async ({
);
}
} catch (error) {
if (getRemainingDownloadMs() <= 0) throw error;
logger.warn('File HEAD request failed, skip size precheck', { url, error });
}
......@@ -96,7 +108,7 @@ export const readFileRawTextByUrl = async ({
url: url,
responseType: 'stream',
maxContentLength: maxFileSize,
timeout: 30000
timeout: getDownloadRequestTimeout(30000)
});
// 优化：直接从 stream 转换为 buffer，避免 arraybuffer 中间步骤
......@@ -115,10 +127,11 @@ export const readFileRawTextByUrl = async ({
};
// Stream timeout
const streamTimeoutMs = getRemainingDownloadMs();
const timeoutId = setTimeout(() => {
cleanup();
reject('File download timeout after 30 seconds');
}, 600000);
reject(new Error(`File download timeout after ${downloadTimeoutMs / 1000} seconds`));
}, streamTimeoutMs);
response.data.on('data', (chunk: Buffer) => {
if (isAborted) return;
......@@ -140,6 +153,10 @@ export const readFileRawTextByUrl = async ({
clearTimeout(timeoutId);
try {
if (getRemainingDownloadMs() <= 0) {
throw new Error(`File download timeout after ${downloadTimeoutMs / 1000} seconds`);
}
// 合并所有 chunks 为单个 buffer
const buffer = Buffer.concat(chunks as unknown as Uint8Array[]);
......@@ -166,6 +183,10 @@ export const readFileRawTextByUrl = async ({
});
});
if (getRemainingDownloadMs() <= 0) {
throw new Error(`File download timeout after ${downloadTimeoutMs / 1000} seconds`);
}
resolve({ rawText });
} catch (error) {
cleanup();
......
packages/service/env.ts
......@@ -165,6 +165,9 @@ export const serviceEnv = createEnv({
TEXTIN_SECRET_CODE: z.string().optional().meta({
description: '合合信息 Textin 服务 Secret Code'
}),
HOME_CHAT_CUSTOM_PDF_PARSE: BoolSchema.default(false).meta({
description: '首页聊天是否启用 PDF 增强解析'
}),
// ==================== 数据库与缓存 ====================
// Redisg
......
packages/service/test/common/file/parseTimeout.test.ts 0 → 100644
import { beforeEach, describe, expect, it, vi } from 'vitest';
const { mockEnv } = vi.hoisted(() => ({
mockEnv: {
PARSE_FILE_TIMEOUT_SECONDS: 600
}
}));
vi.mock('@fastgpt/service/env', () => ({
serviceEnv: mockEnv
}));
const { getBackendFileOperationTimeoutMs } =
await import('@fastgpt/service/common/file/parseTimeout');
describe('getBackendFileOperationTimeoutMs', () => {
beforeEach(() => {
mockEnv.PARSE_FILE_TIMEOUT_SECONDS = 600;
});
it('配置小于 10 分钟时保留最小后端等待时间', () => {
mockEnv.PARSE_FILE_TIMEOUT_SECONDS = 300;
expect(getBackendFileOperationTimeoutMs()).toBe(600000);
});
it.each([600, 1200, 6000])('按环境变量秒数转换为毫秒: %s', (seconds) => {
mockEnv.PARSE_FILE_TIMEOUT_SECONDS = seconds;
expect(getBackendFileOperationTimeoutMs()).toBe(seconds * 1000);
});
});
packages/service/test/common/file/read/utils.test.ts
......@@ -11,7 +11,8 @@ const {
mockDoc2xParsePDF,
mockTextinParsePDF,
mockUploadImage2S3Bucket,
mockGetImageBuffer
mockGetImageBuffer,
mockEnv
} = vi.hoisted(() => ({
mockReadRawContentFromBuffer: vi.fn(async ({ extension, buffer, encoding }: any) => {
if (extension === 'txt') {
......@@ -42,7 +43,10 @@ const {
mockGetImageBuffer: vi.fn().mockResolvedValue({
buffer: Buffer.from('image-bytes'),
mime: 'image/png'
})
}),
mockEnv: {
PARSE_FILE_TIMEOUT_SECONDS: 600
}
}));
vi.mock('@fastgpt/service/worker/function', () => ({
......@@ -90,6 +94,10 @@ vi.mock('@fastgpt/service/common/file/image/utils', () => ({
getImageBuffer: mockGetImageBuffer
}));
vi.mock('@fastgpt/service/env', () => ({
serviceEnv: mockEnv
}));
import {
readRawTextByLocalFile,
readFileContentByBuffer
......@@ -143,6 +151,7 @@ describe('readFileContentByBuffer', () => {
beforeEach(() => {
vi.clearAllMocks();
global.systemEnv = {} as any;
mockEnv.PARSE_FILE_TIMEOUT_SECONDS = 600;
});
it('should parse a txt buffer', async () => {
......@@ -247,6 +256,7 @@ describe('readFileContentByBuffer', () => {
});
it('should use custom URL service for pdf when configured', async () => {
mockEnv.PARSE_FILE_TIMEOUT_SECONDS = 1200;
global.systemEnv = {
customPdfParse: { url: 'http://custom-pdf-service.com/parse', key: 'test-key' }
} as any;
......@@ -270,6 +280,11 @@ describe('readFileContentByBuffer', () => {
});
expect(result.rawText).toBe('custom-service-parsed-text');
expect(mockAxiosPost).toHaveBeenCalledWith(
'http://custom-pdf-service.com/parse',
expect.anything(),
expect.objectContaining({ timeout: 1200000 })
);
});
it('should upload custom URL service base64 and http markdown images with shared handler', async () => {
......
packages/service/test/common/s3/accessLink.test.ts
......@@ -177,7 +177,7 @@ describe('s3 access link', () => {
objectKey: 'chat/app/user/chat/public-prefix.txt',
expiredTime: getFutureDate(10),
maxSize: 1024,
uploadConstraints: {
uploadPolicy: {
defaultContentType: 'text/plain',
allowedExtensions: ['.txt']
}
......@@ -277,10 +277,6 @@ describe('s3 access link', () => {
objectKey: 'chat/app/user/chat/file.txt',
expiredTime: getFutureDate(10),
maxSize: 1024,
uploadConstraints: {
defaultContentType: 'text/plain',
allowedExtensions: ['.txt']
},
uploadPolicy: {
defaultContentType: 'text/plain',
allowedExtensions: ['.txt'],
......@@ -313,6 +309,7 @@ describe('s3 access link', () => {
expect(sessions).toHaveLength(1);
expect(sessions[0]?.tokenHash).toMatch(/^[a-f0-9]{64}$/);
expect(sessions[0]?.tokenHash).not.toBe(token);
expect(sessions[0]).not.toHaveProperty('uploadConstraints');
await expect(verifyS3UploadSessionToken(token)).resolves.toMatchObject({
bucketName: 'fastgpt-private',
......@@ -341,6 +338,144 @@ describe('s3 access link', () => {
expect(usedSession?.usedAt).toBeInstanceOf(Date);
});
it('stores multipart state and prevents complete/abort races from overwriting terminal state', async () => {
const {
createS3UploadAccessUrl,
verifyS3MultipartUploadSessionToken,
markS3MultipartUploadCompleting,
markS3MultipartUploadCompleted,
markS3MultipartUploadAborted,
MongoS3UploadSession
} = await loadAccessLinkModules();
const url = await createS3UploadAccessUrl({
bucketName: 'fastgpt-private',
objectKey: 'dataset/team-1/large-file.pdf',
expiredTime: getFutureDate(30),
maxSize: 100 * 1024 * 1024,
uploadPolicy: {
defaultContentType: 'application/pdf'
},
multipart: {
uploadId: 'upload-1',
partSize: 8 * 1024 * 1024,
totalSize: 50 * 1024 * 1024,
status: 'active'
}
});
const token = extractLastPathSegment(url);
const completedAt = new Date();
await expect(verifyS3MultipartUploadSessionToken(token)).resolves.toMatchObject({
multipart: {
uploadId: 'upload-1',
status: 'active',
partSize: 8 * 1024 * 1024,
totalSize: 50 * 1024 * 1024
}
});
const completionAttemptId = await markS3MultipartUploadCompleting(token);
expect(completionAttemptId).toEqual(expect.any(String));
expect(await markS3MultipartUploadCompleted(token, completionAttemptId!, completedAt)).toBe(
true
);
expect(await markS3MultipartUploadAborted(token)).toBe(false);
const session = await MongoS3UploadSession.findOne({}).lean();
expect(session?.multipart).toMatchObject({
uploadId: 'upload-1',
status: 'completed',
completedAt
});
});
it('restricts Multipart create payloads to active sessions within size and part limits', async () => {
const { CreateS3UploadAccessUrlParamsSchema } = await loadAccessLinkModules();
const baseParams = {
bucketName: 'fastgpt-private',
objectKey: 'dataset/team-1/large-file.pdf',
expiredTime: getFutureDate(30),
maxSize: 100 * 1024 * 1024,
uploadPolicy: {
defaultContentType: 'application/pdf'
},
multipart: {
uploadId: 'upload-1',
partSize: 8 * 1024 * 1024,
totalSize: 50 * 1024 * 1024,
status: 'active' as const
}
};
expect(
CreateS3UploadAccessUrlParamsSchema.safeParse({
...baseParams,
multipart: { ...baseParams.multipart, status: 'completed' }
}).success
).toBe(false);
expect(
CreateS3UploadAccessUrlParamsSchema.safeParse({
...baseParams,
multipart: {
...baseParams.multipart,
totalSize: baseParams.maxSize + 1
}
}).success
).toBe(false);
expect(
CreateS3UploadAccessUrlParamsSchema.safeParse({
...baseParams,
maxSize: 10001,
multipart: {
...baseParams.multipart,
partSize: 1,
totalSize: 10001
}
}).success
).toBe(false);
expect(
CreateS3UploadAccessUrlParamsSchema.safeParse({
...baseParams,
multipart: {
...baseParams.multipart,
completedAt: new Date()
}
}).success
).toBe(false);
});
it('marks an active multipart session aborted idempotently', async () => {
const {
createS3UploadAccessUrl,
markS3MultipartUploadCompleted,
markS3MultipartUploadAborted,
MongoS3UploadSession
} = await loadAccessLinkModules();
const url = await createS3UploadAccessUrl({
bucketName: 'fastgpt-private',
objectKey: 'dataset/team-1/aborted-file.pdf',
expiredTime: getFutureDate(30),
maxSize: 100 * 1024 * 1024,
uploadPolicy: {
defaultContentType: 'application/pdf'
},
multipart: {
uploadId: 'upload-2',
partSize: 8 * 1024 * 1024,
totalSize: 50 * 1024 * 1024,
status: 'active'
}
});
const token = extractLastPathSegment(url);
expect(await markS3MultipartUploadAborted(token)).toBe(true);
expect(await markS3MultipartUploadAborted(token)).toBe(false);
expect(await markS3MultipartUploadCompleted(token, 'stale-attempt')).toBe(false);
const session = await MongoS3UploadSession.findOne({}).lean();
expect(session?.multipart?.status).toBe('aborted');
expect(session?.multipart?.abortedAt).toBeInstanceOf(Date);
});
it('rejects expired and revoked upload sessions', async () => {
const { createS3UploadAccessUrl, verifyS3UploadSessionToken, revokeS3UploadSessionToken } =
await loadAccessLinkModules();
......@@ -350,7 +485,7 @@ describe('s3 access link', () => {
objectKey: 'chat/app/user/chat/expired.txt',
expiredTime: getFutureDate(-10),
maxSize: 1024,
uploadConstraints: {
uploadPolicy: {
defaultContentType: 'text/plain'
}
})
......@@ -361,7 +496,7 @@ describe('s3 access link', () => {
objectKey: 'chat/app/user/chat/revoked.txt',
expiredTime: getFutureDate(10),
maxSize: 1024,
uploadConstraints: {
uploadPolicy: {
defaultContentType: 'text/plain'
}
})
......
packages/service/test/common/s3/buckets/uploadAccess.test.ts 0 → 100644
import { afterEach, describe, expect, it, vi } from 'vitest';
const loadBucketModule = async () => {
vi.resetModules();
const [{ S3BaseBucket }, constants] = await Promise.all([
vi.importActual<typeof import('@fastgpt/service/common/s3/buckets/base')>(
'@fastgpt/service/common/s3/buckets/base'
),
import('@fastgpt/service/common/s3/config/constants')
]);
return { S3BaseBucket, threshold: constants.S3_MULTIPART_UPLOAD_THRESHOLD_BYTES };
};
afterEach(() => {
vi.resetModules();
vi.restoreAllMocks();
});
describe('S3BaseBucket automatic upload access URL', () => {
it('selects Multipart for uploads at or above the threshold', async () => {
const { S3BaseBucket, threshold } = await loadBucketModule();
const bucket = Object.create(S3BaseBucket.prototype) as InstanceType<typeof S3BaseBucket>;
const createPresignedPutUrl = vi.fn();
const createMultipartUploadAccessUrl = vi.fn().mockResolvedValue({ uploadMode: 'multipart' });
Object.assign(bucket, { createPresignedPutUrl, createMultipartUploadAccessUrl });
const result = await bucket.createUploadAccessUrl(
{
rawKey: 'dataset/file.bin',
filename: 'file.bin',
size: threshold
},
{ maxFileSize: 100 }
);
expect(result).toMatchObject({ uploadMode: 'multipart' });
expect(createMultipartUploadAccessUrl).toHaveBeenCalledWith(
{
rawKey: 'dataset/file.bin',
filename: 'file.bin',
size: threshold
},
{ maxFileSize: 100 }
);
expect(createPresignedPutUrl).not.toHaveBeenCalled();
});
it('keeps single PUT below the threshold or when size is unavailable', async () => {
const { S3BaseBucket, threshold } = await loadBucketModule();
const cases = [{ size: threshold - 1 }, { size: undefined }];
for (const testCase of cases) {
const bucket = Object.create(S3BaseBucket.prototype) as InstanceType<typeof S3BaseBucket>;
const createPresignedPutUrl = vi.fn().mockResolvedValue({ uploadMode: 'single' });
const createMultipartUploadAccessUrl = vi.fn();
Object.assign(bucket, { createPresignedPutUrl, createMultipartUploadAccessUrl });
await bucket.createUploadAccessUrl({
rawKey: 'dataset/file.bin',
filename: 'file.bin',
...(testCase.size === undefined ? {} : { size: testCase.size })
});
expect(createPresignedPutUrl).toHaveBeenCalledTimes(1);
expect(createMultipartUploadAccessUrl).not.toHaveBeenCalled();
}
});
});
packages/service/test/common/s3/config/constants.test.ts
......@@ -52,6 +52,22 @@ describe('s3 storage constants', () => {
);
});
it('uses the expected Multipart upload defaults', async () => {
const {
S3_MULTIPART_UPLOAD_THRESHOLD_BYTES,
S3_MULTIPART_PART_SIZE_BYTES,
S3_MULTIPART_CONCURRENCY,
S3_MULTIPART_MAX_RETRY,
S3_MULTIPART_SESSION_EXPIRE_HOURS
} = await loadConstants();
expect(S3_MULTIPART_UPLOAD_THRESHOLD_BYTES).toBe(32 * 1024 * 1024);
expect(S3_MULTIPART_PART_SIZE_BYTES).toBe(8 * 1024 * 1024);
expect(S3_MULTIPART_CONCURRENCY).toBe(3);
expect(S3_MULTIPART_MAX_RETRY).toBe(3);
expect(S3_MULTIPART_SESSION_EXPIRE_HOURS).toBe(3);
});
it('rewrites external URLs with the CDN endpoint', async () => {
vi.stubEnv('STORAGE_EXTERNAL_ENDPOINT', 'https://s3.example.com');
vi.stubEnv('STORAGE_S3_CDN_ENDPOINT', 'https://cdn.example.com/files');
......
packages/service/test/common/s3/lifecycle/cleanup.test.ts 0 → 100644
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { createVitestStorageMock } from '@fastgpt-sdk/storage';
import { MongoS3TTL } from '@fastgpt/service/common/s3/models/ttl';
const { S3BaseBucket } = await vi.importActual<
typeof import('@fastgpt/service/common/s3/buckets/base')
>('@fastgpt/service/common/s3/buckets/base');
const { clearExpiredMinioFiles } = await vi.importActual<
typeof import('@fastgpt/service/common/s3/lifecycle/cleanup')
>('@fastgpt/service/common/s3/lifecycle/cleanup');
const bucketName = 'fastgpt-private';
const fileKey = 'dataset/team-1/expired-file.bin';
const createExpiredTtl = async (multipart?: { uploadId: string }) =>
MongoS3TTL.create({
bucketName,
minioKey: fileKey,
expiredTime: new Date(Date.now() - 1000),
...(multipart ? { multipart } : {})
});
describe('S3 TTL cleanup', () => {
let originalS3BucketMap: typeof global.s3BucketMap;
beforeEach(() => {
originalS3BucketMap = global.s3BucketMap;
});
afterEach(() => {
global.s3BucketMap = originalS3BucketMap;
vi.restoreAllMocks();
});
it('aborts expired Multipart parts and removes only the TTL record', async () => {
const storage = createVitestStorageMock({ vi, bucketName });
const bucket = new S3BaseBucket(storage, undefined);
const upload = await storage.createMultipartUpload({ key: fileKey });
await storage.uploadMultipartPart({
key: fileKey,
uploadId: upload.uploadId,
partNumber: 1,
body: Buffer.from('partial'),
contentLength: 7
});
const addDeleteJob = vi.spyOn(bucket, 'addDeleteJob');
await createExpiredTtl({ uploadId: upload.uploadId });
global.s3BucketMap = { [bucketName]: bucket };
await clearExpiredMinioFiles();
expect(storage.abortMultipartUpload).toHaveBeenCalledWith({
key: fileKey,
uploadId: upload.uploadId
});
expect(storage.__multipartUploads.size).toBe(0);
expect(addDeleteJob).not.toHaveBeenCalled();
expect(await MongoS3TTL.find({ minioKey: fileKey, bucketName })).toHaveLength(0);
});
it('removes an expired Multipart TTL after provider-side cleanup already ran', async () => {
const storage = createVitestStorageMock({ vi, bucketName });
const bucket = new S3BaseBucket(storage, undefined);
const addDeleteJob = vi.spyOn(bucket, 'addDeleteJob');
const uploadId = 'provider-cleaned-upload';
vi.spyOn(storage, 'abortMultipartUpload').mockRejectedValueOnce({ name: 'NoSuchUpload' });
await createExpiredTtl({ uploadId });
global.s3BucketMap = { [bucketName]: bucket };
await clearExpiredMinioFiles();
expect(storage.abortMultipartUpload).toHaveBeenCalledWith({
key: fileKey,
uploadId
});
expect(addDeleteJob).not.toHaveBeenCalled();
expect(await MongoS3TTL.find({ minioKey: fileKey, bucketName })).toHaveLength(0);
});
it('keeps the Multipart TTL record when the bucket is unavailable', async () => {
await createExpiredTtl({ uploadId: 'retry-later-upload' });
global.s3BucketMap = {};
await clearExpiredMinioFiles();
expect(await MongoS3TTL.find({ minioKey: fileKey, bucketName })).toHaveLength(1);
});
it('submits normal expired objects for deletion and then removes their TTL record', async () => {
const storage = createVitestStorageMock({ vi, bucketName });
const bucket = new S3BaseBucket(storage, undefined);
const addDeleteJob = vi.spyOn(bucket, 'addDeleteJob').mockResolvedValue(undefined);
await createExpiredTtl();
global.s3BucketMap = { [bucketName]: bucket };
await clearExpiredMinioFiles();
expect(addDeleteJob).toHaveBeenCalledWith({ key: fileKey });
expect(await MongoS3TTL.find({ minioKey: fileKey, bucketName })).toHaveLength(0);
});
it('rejects a Multipart TTL without an uploadId', async () => {
await expect(
MongoS3TTL.create({
bucketName,
minioKey: fileKey,
expiredTime: new Date(Date.now() - 1000),
multipart: {}
})
).rejects.toThrow();
});
});
packages/service/test/common/s3/sources/dataset.test.ts 0 → 100644
import { describe, expect, it, vi } from 'vitest';
vi.resetModules();
const { S3_MULTIPART_UPLOAD_THRESHOLD_BYTES } = await vi.importActual<
typeof import('@fastgpt/service/common/s3/config/constants')
>('@fastgpt/service/common/s3/config/constants');
const { S3DatasetSource } = await vi.importActual<
typeof import('@fastgpt/service/common/s3/sources/dataset')
>('@fastgpt/service/common/s3/sources/dataset');
const datasetId = '507f1f77bcf86cd799439011';
const createSourceWithUploadMocks = () => {
const source = Object.create(S3DatasetSource.prototype) as InstanceType<typeof S3DatasetSource>;
const createUploadAccessUrl = vi.fn().mockResolvedValue({ uploadMode: 'single' });
Object.assign(source, { createUploadAccessUrl });
return { source, createUploadAccessUrl };
};
describe('S3DatasetSource upload access parameters', () => {
it('passes the file size and dataset policy to the shared upload selector', async () => {
const { source, createUploadAccessUrl } = createSourceWithUploadMocks();
await source.createUploadDatasetFileURL({
filename: 'file.pdf',
datasetId,
size: S3_MULTIPART_UPLOAD_THRESHOLD_BYTES - 1,
maxFileSize: 100
});
expect(createUploadAccessUrl).toHaveBeenCalledWith(
expect.objectContaining({
filename: 'file.pdf',
size: S3_MULTIPART_UPLOAD_THRESHOLD_BYTES - 1,
source: 'local-file'
}),
expect.objectContaining({
expiredHours: 3,
maxFileSize: 100,
uploadPolicy: expect.objectContaining({
defaultContentType: expect.any(String),
allowedExtensions: expect.any(Array)
})
})
);
});
it('keeps the automatic selector compatible with calls without a size hint', async () => {
const { source, createUploadAccessUrl } = createSourceWithUploadMocks();
await source.createUploadDatasetFileURL({
filename: 'file.pdf',
datasetId,
maxFileSize: 100
});
expect(createUploadAccessUrl).toHaveBeenCalledWith(
expect.objectContaining({
rawKey: expect.stringMatching(/^dataset\/507f1f77bcf86cd799439011\//),
filename: 'file.pdf',
source: 'local-file'
}),
expect.objectContaining({
expiredHours: 3,
maxFileSize: 100,
uploadPolicy: expect.objectContaining({
defaultContentType: expect.any(String),
allowedExtensions: expect.any(Array)
})
})
);
});
});
packages/service/test/common/s3/utils/assert.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import { InvalidObjectNameError, InvalidXMLError, S3Error } from 'minio';
import { isFileNotFoundError } from '@fastgpt/service/common/s3/utils/assert';
describe('isFileNotFoundError', () => {
it('recognizes provider-neutral 404 response shapes', () => {
const errors = [
{ statusCode: 404 },
{ status: 404 },
{ $metadata: { httpStatusCode: 404 } },
{ code: 'NotFound' },
{ name: 'NoSuchKey' },
{ name: 'NoSuchObject' }
];
errors.forEach((error) => {
expect(isFileNotFoundError(error)).toBe(true);
});
});
it('recognizes MinIO not-found errors', () => {
const s3Error = new S3Error('Not Found');
s3Error.code = 'NoSuchKey';
expect(isFileNotFoundError(s3Error)).toBe(true);
expect(isFileNotFoundError(new InvalidObjectNameError())).toBe(true);
expect(isFileNotFoundError(new InvalidXMLError())).toBe(true);
});
it('does not classify unrelated errors as missing objects', () => {
expect(isFileNotFoundError(undefined)).toBe(false);
expect(isFileNotFoundError('Not Found')).toBe(false);
expect(isFileNotFoundError(new Error('Not Found'))).toBe(false);
expect(isFileNotFoundError({ statusCode: 500 })).toBe(false);
expect(isFileNotFoundError({ code: 'AccessDenied' })).toBe(false);
});
});
packages/service/test/core/dataset/read.test.ts
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { DatasetSourceReadTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { PassThrough, Readable } from 'node:stream';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const mocks = vi.hoisted(() => ({
getDatasetFileRawText: vi.fn()
getDatasetFileRawText: vi.fn(),
axios: vi.fn(),
axiosHead: vi.fn(),
readFileContentByBuffer: vi.fn()
}));
vi.mock('@fastgpt/service/common/s3/sources/dataset', () => ({
......@@ -12,7 +16,21 @@ vi.mock('@fastgpt/service/common/s3/sources/dataset', () => ({
})
}));
import { readDatasetSourceRawText } from '@fastgpt/service/core/dataset/read';
vi.mock('@fastgpt/service/common/api/axios', async (importOriginal) => {
const mod = await importOriginal<typeof import('@fastgpt/service/common/api/axios')>();
return {
...mod,
axios: Object.assign(mocks.axios, {
head: mocks.axiosHead
})
};
});
vi.mock('@fastgpt/service/common/file/read/utils', () => ({
readFileContentByBuffer: mocks.readFileContentByBuffer
}));
import { readDatasetSourceRawText, readFileRawTextByUrl } from '@fastgpt/service/core/dataset/read';
describe('readDatasetSourceRawText', () => {
beforeEach(() => {
......@@ -21,6 +39,8 @@ describe('readDatasetSourceRawText', () => {
filename: 'demo.pdf',
rawText: 'demo content'
});
mocks.axiosHead.mockResolvedValue({ headers: {} });
mocks.readFileContentByBuffer.mockResolvedValue({ rawText: 'downloaded content' });
});
it('rejects a local dataset file key that is not under the authorized dataset id', async () => {
......@@ -59,3 +79,60 @@ describe('readDatasetSourceRawText', () => {
);
});
});
describe('readFileRawTextByUrl', () => {
beforeEach(() => {
vi.useFakeTimers();
vi.clearAllMocks();
mocks.axiosHead.mockResolvedValue({ headers: {} });
mocks.readFileContentByBuffer.mockResolvedValue({ rawText: 'downloaded content' });
});
afterEach(() => {
vi.useRealTimers();
});
it('在统一下载 deadline 内保留 30 秒建连 timeout，并在流结束后解析文件内容', async () => {
mocks.axios.mockResolvedValue({
data: Readable.from([Buffer.from('pdf-content')])
});
await expect(
readFileRawTextByUrl({
teamId: 'team-a',
tmbId: 'tmb-a',
url: 'https://example.com/file.pdf',
relatedId: 'external-file-a',
datasetId: 'dataset-a'
})
).resolves.toEqual({ rawText: 'downloaded content' });
expect(mocks.axios).toHaveBeenCalledWith(
expect.objectContaining({
responseType: 'stream',
timeout: 30000
})
);
});
it('流读取超过后端有效 timeout 时终止下载并抛出 Error', async () => {
const stream = new PassThrough();
mocks.axios.mockResolvedValue({ data: stream });
const resultPromise = readFileRawTextByUrl({
teamId: 'team-a',
tmbId: 'tmb-a',
url: 'https://example.com/file.pdf',
relatedId: 'external-file-a',
datasetId: 'dataset-a'
});
const resultAssertion = expect(resultPromise).rejects.toThrow(
'File download timeout after 600 seconds'
);
await vi.advanceTimersByTimeAsync(600000);
await resultAssertion;
expect(stream.destroyed).toBe(true);
});
});
packages/service/test/env.test.ts
......@@ -17,6 +17,7 @@ const originalEnv = {
AES256_SECRET_KEY: process.env.AES256_SECRET_KEY,
INVOKE_TOKEN_SECRET: process.env.INVOKE_TOKEN_SECRET,
SOMARK_API_KEY: process.env.SOMARK_API_KEY,
HOME_CHAT_CUSTOM_PDF_PARSE: process.env.HOME_CHAT_CUSTOM_PDF_PARSE,
PRO_URL: process.env.PRO_URL,
PRO_TOKEN: process.env.PRO_TOKEN,
VITEST: process.env.VITEST,
......@@ -54,6 +55,7 @@ describe('serviceEnv', () => {
vi.stubEnv('AES256_SECRET_KEY', originalEnv.AES256_SECRET_KEY);
vi.stubEnv('INVOKE_TOKEN_SECRET', originalEnv.INVOKE_TOKEN_SECRET);
vi.stubEnv('SOMARK_API_KEY', originalEnv.SOMARK_API_KEY);
vi.stubEnv('HOME_CHAT_CUSTOM_PDF_PARSE', originalEnv.HOME_CHAT_CUSTOM_PDF_PARSE);
vi.stubEnv('PRO_URL', originalEnv.PRO_URL);
vi.stubEnv('PRO_TOKEN', originalEnv.PRO_TOKEN);
vi.stubEnv('VITEST', originalEnv.VITEST);
......@@ -95,6 +97,20 @@ describe('serviceEnv', () => {
});
});
it('disables home chat custom PDF parsing by default and supports enabling it', async () => {
vi.stubEnv('FILE_TOKEN_KEY', 'filetokenkey');
vi.stubEnv('AES256_SECRET_KEY', 'fastgptsecret');
vi.stubEnv('INVOKE_TOKEN_SECRET', validInvokeTokenSecret);
vi.stubEnv('HOME_CHAT_CUSTOM_PDF_PARSE', undefined);
const defaultEnv = await importServiceEnv();
expect(defaultEnv.serviceEnv.HOME_CHAT_CUSTOM_PDF_PARSE).toBe(false);
vi.stubEnv('HOME_CHAT_CUSTOM_PDF_PARSE', 'true');
const enabledEnv = await importServiceEnv();
expect(enabledEnv.serviceEnv.HOME_CHAT_CUSTOM_PDF_PARSE).toBe(true);
});
it('validates SYSTEM_MAX_STRING_LENGTH_M during service env init', async () => {
vi.stubEnv('FILE_TOKEN_KEY', 'filetokenkey');
vi.stubEnv('AES256_SECRET_KEY', 'fastgptsecret');
......
packages/service/test/thirdProvider/doc2x.test.ts
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const { requestMock, axiosPutMock, getImageBufferMock } = vi.hoisted(() => ({
const { requestMock, axiosPutMock, getImageBufferMock, mockEnv } = vi.hoisted(() => ({
requestMock: vi.fn(),
axiosPutMock: vi.fn(),
getImageBufferMock: vi.fn()
getImageBufferMock: vi.fn(),
mockEnv: {
PARSE_FILE_TIMEOUT_SECONDS: 600
}
}));
vi.mock('@fastgpt/service/common/api/axios', () => ({
......@@ -19,6 +22,10 @@ vi.mock('@fastgpt/service/common/file/image/utils', () => ({
getImageBuffer: getImageBufferMock
}));
vi.mock('@fastgpt/service/env', () => ({
serviceEnv: mockEnv
}));
const { useDoc2xServer } = await import('@fastgpt/service/thirdProvider/doc2x');
const mockDoc2xSuccess = (md: string) => {
......@@ -52,6 +59,8 @@ const mockDoc2xSuccess = (md: string) => {
describe('useDoc2xServer', () => {
beforeEach(() => {
vi.clearAllMocks();
requestMock.mockReset();
mockEnv.PARSE_FILE_TIMEOUT_SECONDS = 600;
vi.useFakeTimers();
axiosPutMock.mockResolvedValue({
status: 200,
......@@ -77,12 +86,15 @@ describe('useDoc2xServer', () => {
await vi.runAllTimersAsync();
const result = await resultPromise;
expect(getImageBufferMock).toHaveBeenCalledWith('https://img.example.com/a.png');
expect(getImageBufferMock).toHaveBeenCalledWith('https://img.example.com/a.png', {
timeoutMs: 180000
});
expect(uploadImage).toHaveBeenCalledWith({
type: 'http',
url: 'https://img.example.com/a.png',
mime: 'image/png',
buffer: Buffer.from('image-bytes')
buffer: Buffer.from('image-bytes'),
signal: expect.any(AbortSignal)
});
expect(result).toEqual({
pages: 1,
......@@ -104,8 +116,12 @@ describe('useDoc2xServer', () => {
await vi.runAllTimersAsync();
const result = await resultPromise;
expect(getImageBufferMock).toHaveBeenNthCalledWith(1, 'https://img.example.com/a.png');
expect(getImageBufferMock).toHaveBeenNthCalledWith(2, 'https://img.example.com/b.png');
expect(getImageBufferMock).toHaveBeenNthCalledWith(1, 'https://img.example.com/a.png', {
timeoutMs: 180000
});
expect(getImageBufferMock).toHaveBeenNthCalledWith(2, 'https://img.example.com/b.png', {
timeoutMs: 180000
});
expect(uploadImage).toHaveBeenCalledTimes(2);
expect(result.text).toBe(
'a ![](dataset/ds1/file-parsed/a.png) b ![](dataset/ds1/file-parsed/b.png)'
......@@ -128,7 +144,8 @@ describe('useDoc2xServer', () => {
type: 'base64',
mime: 'image/png',
base64: 'iVBORw0KGgo=',
dataUrl: 'data:image/png;base64,iVBORw0KGgo='
dataUrl: 'data:image/png;base64,iVBORw0KGgo=',
signal: expect.any(AbortSignal)
});
expect(result.text).toBe('hello ![img](dataset/ds1/file-parsed/base64.png)');
});
......@@ -153,4 +170,165 @@ describe('useDoc2xServer', () => {
expect(getImageBufferMock).not.toHaveBeenCalled();
expect(result.text).toBe('hello ![](https://img.example.com/a.png)');
});
it('Doc2x 返回 failed 状态时立即失败，不继续轮询', async () => {
requestMock.mockReset();
requestMock
.mockResolvedValueOnce({
data: {
code: 'ok',
data: {
uid: 'uid-failed',
url: 'https://upload.example.com/file'
}
}
})
.mockResolvedValueOnce({
data: {
code: 'ok',
msg: 'invalid pdf',
data: {
status: 'failed',
result: {
pages: []
}
}
}
});
const resultPromise = useDoc2xServer({ apiKey: 'api-key' }).parsePDF(Buffer.from('pdf'));
const resultAssertion = expect(resultPromise).rejects.toThrow(
'[Doc2x] Failed to get result (uid: uid-failed): invalid pdf'
);
await vi.runAllTimersAsync();
await resultAssertion;
expect(requestMock).toHaveBeenCalledTimes(2);
});
it('Doc2x 返回未知状态时立即失败，不快速自旋', async () => {
requestMock.mockReset();
requestMock
.mockResolvedValueOnce({
data: {
code: 'ok',
data: {
uid: 'uid-unknown',
url: 'https://upload.example.com/file'
}
}
})
.mockResolvedValueOnce({
data: {
code: 'ok',
data: {
status: 'queued',
result: {
pages: []
}
}
}
});
const resultPromise = useDoc2xServer({ apiKey: 'api-key' }).parsePDF(Buffer.from('pdf'));
const resultAssertion = expect(resultPromise).rejects.toThrow(
'[Doc2x] Failed to get result (uid: uid-unknown): unknown status queued'
);
await vi.runAllTimersAsync();
await resultAssertion;
expect(requestMock).toHaveBeenCalledTimes(2);
});
it('轮询总时长跟随后端有效 timeout，不受固定 120 次限制', async () => {
mockEnv.PARSE_FILE_TIMEOUT_SECONDS = 1200;
requestMock.mockReset();
let statusCalls = 0;
requestMock.mockImplementation(({ url }: { url: string }) => {
if (url === '/v2/parse/preupload') {
return Promise.resolve({
data: {
code: 'ok',
data: {
uid: 'uid-long-running',
url: 'https://upload.example.com/file'
}
}
});
}
statusCalls += 1;
return Promise.resolve({
data: {
code: 'ok',
data:
statusCalls === 121
? {
status: 'success',
result: {
pages: [{ md: 'long-running result' }]
}
}
: {
status: 'processing',
progress: statusCalls,
result: {
pages: []
}
}
}
});
});
const resultPromise = useDoc2xServer({ apiKey: 'api-key' }).parsePDF(Buffer.from('pdf'));
await vi.runAllTimersAsync();
const result = await resultPromise;
expect(statusCalls).toBe(121);
expect(result).toEqual({
pages: 1,
text: 'long-running result'
});
});
it('达到整体 deadline 后停止状态轮询', async () => {
requestMock.mockReset();
let statusCalls = 0;
requestMock.mockImplementation(({ url }: { url: string }) => {
if (url === '/v2/parse/preupload') {
return Promise.resolve({
data: {
code: 'ok',
data: {
uid: 'uid-timeout',
url: 'https://upload.example.com/file'
}
}
});
}
statusCalls += 1;
return Promise.resolve({
data: {
code: 'ok',
data: {
status: 'processing',
progress: statusCalls,
result: {
pages: []
}
}
}
});
});
const resultPromise = useDoc2xServer({ apiKey: 'api-key' }).parsePDF(Buffer.from('pdf'));
const resultAssertion = expect(resultPromise).rejects.toThrow(
'[Doc2x] Failed to get result (uid: uid-timeout): Process timeout'
);
await vi.runAllTimersAsync();
await resultAssertion;
expect(statusCalls).toBe(119);
});
});
packages/service/test/thirdProvider/somark.test.ts
import FormData from 'form-data';
import { beforeEach, describe, expect, it, vi } from 'vitest';
const { postMock } = vi.hoisted(() => ({
postMock: vi.fn()
const { postMock, createProxyAxiosMock, mockEnv } = vi.hoisted(() => ({
postMock: vi.fn(),
createProxyAxiosMock: vi.fn(() => ({
post: postMock
})),
mockEnv: {
PARSE_FILE_TIMEOUT_SECONDS: 600
}
}));
vi.mock('@fastgpt/service/common/api/axios', () => ({
createProxyAxios: vi.fn(() => ({
post: postMock
}))
createProxyAxios: createProxyAxiosMock
}));
vi.mock('@fastgpt/service/env', () => ({
serviceEnv: mockEnv
}));
const { useSomarkServer } = await import('@fastgpt/service/thirdProvider/somark');
......@@ -35,6 +43,7 @@ const mockSomarkSuccess = () => {
describe('useSomarkServer', () => {
beforeEach(() => {
vi.clearAllMocks();
mockEnv.PARSE_FILE_TIMEOUT_SECONDS = 600;
});
it('通过同步接口解析 PDF 并返回 Markdown 与页数', async () => {
......@@ -57,6 +66,7 @@ describe('useSomarkServer', () => {
})
})
);
expect(createProxyAxiosMock).toHaveBeenCalledWith(expect.objectContaining({ timeout: 600000 }));
const form = postMock.mock.calls[0][1] as FormData;
const body = form.getBuffer().toString();
......
packages/service/test/thirdProvider/textin.test.ts
import { beforeEach, describe, expect, it, vi } from 'vitest';
const { postMock, getImageBufferMock } = vi.hoisted(() => ({
const { postMock, getImageBufferMock, createProxyAxiosMock, mockEnv } = vi.hoisted(() => ({
postMock: vi.fn(),
getImageBufferMock: vi.fn()
getImageBufferMock: vi.fn(),
createProxyAxiosMock: vi.fn(() => ({
post: postMock
})),
mockEnv: {
PARSE_FILE_TIMEOUT_SECONDS: 600
}
}));
vi.mock('@fastgpt/service/common/api/axios', () => ({
createProxyAxios: vi.fn(() => ({
post: postMock
}))
createProxyAxios: createProxyAxiosMock
}));
vi.mock('@fastgpt/service/common/file/image/utils', () => ({
getImageBuffer: getImageBufferMock
}));
vi.mock('@fastgpt/service/env', () => ({
serviceEnv: mockEnv
}));
const { useTextinServer } = await import('@fastgpt/service/thirdProvider/textin');
const mockTextinSuccess = (markdown: string) => {
......@@ -32,6 +40,7 @@ const mockTextinSuccess = (markdown: string) => {
describe('useTextinServer', () => {
beforeEach(() => {
vi.clearAllMocks();
mockEnv.PARSE_FILE_TIMEOUT_SECONDS = 600;
getImageBufferMock.mockResolvedValue({
buffer: Buffer.from('image-bytes'),
mime: 'image/png'
......@@ -60,6 +69,7 @@ describe('useTextinServer', () => {
})
})
);
expect(createProxyAxiosMock).toHaveBeenCalledWith(expect.objectContaining({ timeout: 600000 }));
expect(uploadImage).toHaveBeenCalledWith({
type: 'base64',
mime: 'image/png',
......
packages/service/thirdProvider/somark/index.ts
......@@ -2,6 +2,7 @@ import FormData from 'form-data';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { createProxyAxios } from '../../common/api/axios';
import { getLogger, LogCategories } from '../../common/logger';
import { getBackendFileOperationTimeoutMs } from '../../common/file/parseTimeout';
type SomarkResponse = {
code?: number;
......@@ -28,7 +29,7 @@ export const useSomarkServer = ({ apiKey }: { apiKey: string }) => {
const logger = getLogger(LogCategories.MODULE.DATASET.FILE);
const instance = createProxyAxios({
baseURL: 'https://somark.ai/api/v1',
timeout: 600000
timeout: getBackendFileOperationTimeoutMs()
});
const parsePDF = async (fileBuffer: Buffer) => {
......
packages/service/thirdProvider/textin/index.ts
......@@ -4,6 +4,7 @@ import { getLogger, LogCategories } from '../../common/logger';
import { parseMarkdownBase64Images } from '@fastgpt/global/common/string/markdown';
import { type UploadedFileResult } from '../../worker/readFile/type';
import { getImageBuffer } from '../../common/file/image/utils';
import { getBackendFileOperationTimeoutMs } from '../../common/file/parseTimeout';
type TextinImageUploadHandler = (
params:
......@@ -26,7 +27,7 @@ export const useTextinServer = ({ appId, secretCode }: { appId: string; secretCo
// Init request
const instance = createProxyAxios({
baseURL: 'https://api.textin.com/ai/service/v1',
timeout: 300000,
timeout: getBackendFileOperationTimeoutMs(),
headers: {
'x-ti-app-id': appId,
'x-ti-secret-code': secretCode
......
packages/service/worker/function.ts
......@@ -37,7 +37,7 @@ const getReadFileWorker = () =>
getWorkerController<ReadFileWorkerProps, ReadFileResponse>({
name: WorkerNameEnum.readFile,
maxReservedThreads: serviceEnv.PARSE_FILE_WORKERS,
// 单任务超时：默认 300s（5min），由 PARSE_FILE_TIMEOUT_SECONDS（秒）配置
// 单任务超时：默认 600s（10min），由 PARSE_FILE_TIMEOUT_SECONDS（秒）配置
taskTimeoutMs: serviceEnv.PARSE_FILE_TIMEOUT_SECONDS * 1000,
// mammoth/xlsx/pdf-parse 历史上有 module 级缓存与潜在内存泄漏，定期回收 worker
maxTasksPerWorker: 100
......
packages/web/common/file/hooks/useUploadAvatar.tsx
import { base64ToFile, fileToBase64, putFileToS3 } from '../utils';
import { base64ToFile, fileToBase64 } from '../utils';
import { S3FileUploader } from '../uploader';
import { compressBase64Img } from '../img';
import { useToast } from '../../../hooks/useToast';
import { useCallback, useRef, useTransition } from 'react';
import { useTranslation } from 'next-i18next';
import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants';
import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type';
import type {
CreatePostPresignedUrlResponseType,
PresignFileUploadParams
} from '@fastgpt/global/common/file/s3/type';
export const useUploadAvatar = (
api: (params: { filename: string }) => Promise<CreatePostPresignedUrlResponseType>,
api: (params: PresignFileUploadParams) => Promise<CreatePostPresignedUrlResponseType>,
{
onSuccess,
maxW = 300,
......@@ -48,17 +52,17 @@ export const useUploadAvatar = (
}),
file.name
);
const { url, key, headers } = await api({ filename: file.name });
const uploadResult = await api({ filename: file.name, size: compressed.size });
await putFileToS3({
url,
const uploader = new S3FileUploader({
...uploadResult,
file: compressed,
headers,
onSuccess() {
onSuccess?.(`${imageBaseUrl}${key}`);
onSuccess?.(`${imageBaseUrl}${uploadResult.key}`);
},
t
});
await uploader.upload();
});
},
[t, toast, api, onSuccess]
......
packages/web/common/file/uploader/constants.ts 0 → 100644
/** 单个 Multipart 分片和控制请求的超时时间。 */
export const MULTIPART_REQUEST_TIMEOUT = 120 * 1000;
export const MULTIPART_RETRY_BASE_DELAY = 300;
export const SINGLE_REQUEST_TIMEOUT = 5 * 60 * 1000;
packages/web/common/file/uploader/index.ts 0 → 100644
import { abortMultipartFile, uploadMultipartFile } from './multipart';
import { uploadSingleFile } from './single';
import type { S3FileUploaderParams } from './types';
/**
* 统一执行浏览器到 FastGPT 对象存储代理的文件上传。
*
* 未声明 uploadMode 的旧响应按 single 处理；新的 presign 响应会携带完整的模式参数。
*/
export class S3FileUploader {
constructor(private readonly params: S3FileUploaderParams) {}
/** 根据预签名响应选择 single PUT 或 Multipart 上传。 */
async upload(): Promise<void> {
if (this.params.uploadMode === 'multipart') {
return uploadMultipartFile(this.params);
}
return uploadSingleFile(this.params);
}
/** 最佳努力清理已签发但尚未开始或已中断的 Multipart session。 */
async abort(): Promise<void> {
if (this.params.uploadMode !== 'multipart') return;
await abortMultipartFile(this.params.abortUrl).catch(() => undefined);
}
}
export type { S3FileUploaderParams } from './types';
packages/web/common/file/uploader/multipart.ts 0 → 100644
import axios from 'axios';
import { parseS3UploadError } from '@fastgpt/global/common/error/s3';
import { MULTIPART_REQUEST_TIMEOUT, MULTIPART_RETRY_BASE_DELAY } from './constants';
import type { MultipartUploadPart, S3FileUploaderMultipartParams } from './types';
import {
createMultipartAbortError,
appendUrlSearchParam,
getMultipartPartCount,
getMultipartPartRange,
isUploadAbortError,
throwIfAborted,
waitForMultipartRetry
} from './utils';
const uploadMultipartPartWithRetry = async ({
url,
file,
partNumber,
partSize,
headers,
maxRetry,
signal,
onProgress
}: {
url: string;
file: File;
partNumber: number;
partSize: number;
headers?: Record<string, string>;
maxRetry: number;
signal: AbortSignal;
onProgress: (loaded: number) => void;
}): Promise<MultipartUploadPart> => {
const { start, end, size } = getMultipartPartRange({
fileSize: file.size,
partSize,
partNumber
});
const partUrl = appendUrlSearchParam({
url,
key: 'partNumber',
value: String(partNumber)
});
for (let attempt = 0; ; attempt++) {
throwIfAborted(signal);
try {
const response = await axios.put(partUrl, file.slice(start, end), {
headers: {
...headers
},
onUploadProgress: (event) => {
onProgress(Math.min(event.loaded, size));
},
signal,
timeout: MULTIPART_REQUEST_TIMEOUT
});
const etag = response.data?.data?.etag ?? response.data?.etag;
if (typeof etag !== 'string' || !etag) {
throw new Error('Multipart part response missing etag');
}
onProgress(size);
return { partNumber, etag };
} catch (error) {
if (isUploadAbortError(error, signal) || attempt >= maxRetry) {
throw error;
}
onProgress(0);
await waitForMultipartRetry(MULTIPART_RETRY_BASE_DELAY * 2 ** attempt, signal);
}
}
};
const postMultipartAbort = (abortUrl: string) =>
axios.post(abortUrl, undefined, {
timeout: MULTIPART_REQUEST_TIMEOUT
});
/** 使用独立请求清理 Multipart session；调用方可在 presign 返回后但上传尚未开始时使用。 */
export const abortMultipartFile = async (abortUrl: string) => {
await postMultipartAbort(abortUrl);
};
/** 执行 Multipart 分片调度、完成和失败后的远端清理。 */
export const uploadMultipartFile = async (params: S3FileUploaderMultipartParams): Promise<void> => {
const partCount = getMultipartPartCount(params.file.size, params.partSize);
if (!Number.isInteger(params.concurrency) || params.concurrency <= 0) {
throw new Error('Multipart concurrency must be a positive integer');
}
if (!Number.isInteger(params.maxRetry) || params.maxRetry < 0) {
throw new Error('Multipart max retry must be a non-negative integer');
}
const requestController = new AbortController();
const requestSignal = requestController.signal;
const onExternalAbort = () => {
requestController.abort(params.signal?.reason ?? createMultipartAbortError());
};
params.signal?.addEventListener('abort', onExternalAbort, { once: true });
const loadedByPart = new Array<number>(partCount).fill(0);
const parts = new Array<MultipartUploadPart | undefined>(partCount);
const reportProgress = () => {
params.onProgress?.(
Math.min(
params.file.size,
loadedByPart.reduce((total, loaded) => total + loaded, 0)
),
params.file.size
);
};
let nextPartNumber = 1;
let firstUploadError: unknown;
const worker = async () => {
while (true) {
const partNumber = nextPartNumber++;
if (partNumber > partCount) return;
try {
const part = await uploadMultipartPartWithRetry({
url: params.url,
file: params.file,
partNumber,
partSize: params.partSize,
headers: params.headers,
maxRetry: params.maxRetry,
signal: requestSignal,
onProgress: (loaded) => {
loadedByPart[partNumber - 1] = loaded;
reportProgress();
}
});
parts[partNumber - 1] = part;
} catch (error) {
firstUploadError ??= error;
if (!requestSignal.aborted) requestController.abort(error);
throw error;
}
}
};
try {
throwIfAborted(params.signal);
reportProgress();
await Promise.all(
Array.from({ length: Math.min(params.concurrency, partCount) }, () => worker())
);
throwIfAborted(params.signal);
const completedParts = parts
.filter((part): part is MultipartUploadPart => !!part)
.sort((left, right) => left.partNumber - right.partNumber);
if (completedParts.length !== partCount) {
throw new Error('Multipart parts are incomplete');
}
await axios.post(
params.completeUrl,
{ parts: completedParts },
{
signal: requestSignal,
timeout: MULTIPART_REQUEST_TIMEOUT
}
);
} catch (error) {
const uploadError = firstUploadError ?? error;
await abortMultipartFile(params.abortUrl).catch(() => undefined);
if (isUploadAbortError(uploadError, params.signal)) {
throw uploadError;
}
throw parseS3UploadError({ t: params.t, error: uploadError, maxSize: params.maxSize });
} finally {
params.signal?.removeEventListener('abort', onExternalAbort);
}
params.onProgress?.(params.file.size, params.file.size);
params.onSuccess?.();
};
packages/web/common/file/uploader/single.ts 0 → 100644
import axios from 'axios';
import { parseS3UploadError } from '@fastgpt/global/common/error/s3';
import { SINGLE_REQUEST_TIMEOUT } from './constants';
import type { S3FileUploaderSingleParams } from './types';
import { isUploadAbortError, throwIfAborted } from './utils';
/** 执行单 PUT 上传，并将 Axios 进度事件转换成统一的字节进度回调。 */
export const uploadSingleFile = async (params: S3FileUploaderSingleParams): Promise<void> => {
try {
throwIfAborted(params.signal);
params.onProgress?.(0, params.file.size);
await axios.put(params.url, params.file, {
headers: {
...params.headers
},
onUploadProgress: (event) => {
params.onProgress?.(Math.min(event.loaded, params.file.size), params.file.size);
},
signal: params.signal,
timeout: SINGLE_REQUEST_TIMEOUT
});
} catch (error) {
if (isUploadAbortError(error, params.signal)) {
throw error;
}
throw parseS3UploadError({ t: params.t, error, maxSize: params.maxSize });
}
params.onProgress?.(params.file.size, params.file.size);
params.onSuccess?.();
};
packages/web/common/file/uploader/types.ts 0 → 100644
export type S3UploadTranslation = (key: string, options?: Record<string, unknown>) => string;
export type S3FileUploaderBaseParams = {
url: string;
file: File;
headers?: Record<string, string>;
onProgress?: (loaded: number, total: number) => void;
onSuccess?: () => void;
signal?: AbortSignal;
maxSize?: number;
t: S3UploadTranslation;
};
export type S3FileUploaderSingleParams = S3FileUploaderBaseParams & {
/** 未提供上传模式的旧接口仍按单 PUT 处理。 */
uploadMode?: 'single';
};
export type S3FileUploaderMultipartParams = S3FileUploaderBaseParams & {
uploadMode: 'multipart';
completeUrl: string;
abortUrl: string;
partSize: number;
concurrency: number;
maxRetry: number;
};
export type S3FileUploaderParams = S3FileUploaderSingleParams | S3FileUploaderMultipartParams;
export type MultipartUploadPart = {
partNumber: number;
etag: string;
};
packages/web/common/file/uploader/utils.ts 0 → 100644
import axios from 'axios';
import { MAX_MULTIPART_PART_COUNT } from '@fastgpt/global/common/file/constants';
/** 创建统一的上传取消错误，兼容没有 DOMException 的运行环境。 */
export const createMultipartAbortError = () => {
if (typeof DOMException !== 'undefined') {
return new DOMException('The multipart upload was aborted', 'AbortError');
}
const error = new Error('The multipart upload was aborted');
error.name = 'AbortError';
return error;
};
/** 判断请求错误是否属于用户取消或上传器内部取消。 */
export const isUploadAbortError = (error: unknown, signal?: AbortSignal) => {
if (signal?.aborted || axios.isCancel(error)) return true;
if (!error || typeof error !== 'object') return false;
const typedError = error as { name?: string; code?: string };
return (
typedError.name === 'AbortError' ||
typedError.name === 'CanceledError' ||
typedError.code === 'ERR_CANCELED'
);
};
/** 在开始新的上传请求前检查外部或内部取消信号。 */
export const throwIfAborted = (signal?: AbortSignal) => {
if (signal?.aborted) {
throw signal.reason ?? createMultipartAbortError();
}
};
/** 校验文件和分片大小，并返回 Multipart 分片数量。 */
export const getMultipartPartCount = (fileSize: number, partSize: number) => {
if (!Number.isInteger(fileSize) || fileSize <= 0) {
throw new Error('Multipart file size must be a positive integer');
}
if (!Number.isInteger(partSize) || partSize <= 0) {
throw new Error('Multipart part size must be a positive integer');
}
if (!Number.isSafeInteger(fileSize) || !Number.isSafeInteger(partSize)) {
throw new Error('Multipart file size and part size must be safe integers');
}
const partCount = Math.ceil(fileSize / partSize);
if (partCount > MAX_MULTIPART_PART_COUNT) {
throw new Error(`Multipart upload cannot exceed ${MAX_MULTIPART_PART_COUNT} parts`);
}
return partCount;
};
/** 根据 part number 计算 File.slice 的边界，只有最后一个分片可以小于 partSize。 */
export const getMultipartPartRange = ({
fileSize,
partSize,
partNumber
}: {
fileSize: number;
partSize: number;
partNumber: number;
}) => {
const partCount = getMultipartPartCount(fileSize, partSize);
if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > partCount) {
throw new Error('Multipart part number is out of range');
}
const start = (partNumber - 1) * partSize;
const end = Math.min(start + partSize, fileSize);
return { start, end, size: end - start };
};
/** 使用 URL API 添加查询参数，并保留调用方原本的相对/绝对 URL 形式。 */
export const appendUrlSearchParam = ({
url,
key,
value
}: {
url: string;
key: string;
value: string;
}) => {
const hashIndex = url.indexOf('#');
const hash = hashIndex === -1 ? '' : url.slice(hashIndex);
const urlWithoutHash = hashIndex === -1 ? url : url.slice(0, hashIndex);
const queryIndex = urlWithoutHash.indexOf('?');
const pathname = queryIndex === -1 ? urlWithoutHash : urlWithoutHash.slice(0, queryIndex);
const query = queryIndex === -1 ? '' : urlWithoutHash.slice(queryIndex + 1);
const searchParams = new URLSearchParams(query);
searchParams.set(key, value);
return `${pathname}?${searchParams.toString()}${hash}`;
};
/** 等待当前分片的指数退避时间，并允许取消立即结束等待。 */
export const waitForMultipartRetry = (delay: number, signal: AbortSignal) =>
new Promise<void>((resolve, reject) => {
if (signal.aborted) {
reject(signal.reason ?? createMultipartAbortError());
return;
}
const timer = setTimeout(() => {
signal.removeEventListener('abort', onAbort);
resolve();
}, delay);
const onAbort = () => {
clearTimeout(timer);
signal.removeEventListener('abort', onAbort);
reject(signal.reason ?? createMultipartAbortError());
};
signal.addEventListener('abort', onAbort, { once: true });
});
packages/web/common/file/utils.ts
import { getErrText } from '@fastgpt/global/common/error/utils';
import Papa from 'papaparse';
import { type AxiosProgressEvent } from 'axios';
import axios from 'axios';
import { parseS3UploadError } from '@fastgpt/global/common/error/s3';
export const loadFile2Buffer = ({ file, onError }: { file: File; onError?: (err: any) => void }) =>
new Promise<ArrayBuffer>((resolve, reject) => {
......@@ -123,50 +120,3 @@ export const base64ToFile = (base64: string, filename: string) => {
}
return new File([u8arr], filename, { type: mime });
};
export const putFileToS3 = async ({
headers,
url,
file,
onSuccess,
onUploadProgress,
signal,
maxSize,
t
}: {
headers?: Record<string, string>;
url: string;
file: File;
onSuccess?: () => void;
onUploadProgress?: (progressEvent: AxiosProgressEvent) => void;
signal?: AbortSignal;
maxSize?: number;
t: any;
}) => {
try {
const res = await axios.put(url, file, {
headers: {
...headers
},
onUploadProgress,
signal,
timeout: 5 * 60 * 1000
});
if (res.status === 200) {
onSuccess?.();
}
} catch (error) {
if (
axios.isCancel(error) ||
(typeof error === 'object' &&
error !== null &&
((error as { name?: string }).name === 'AbortError' ||
(error as { name?: string }).name === 'CanceledError' ||
(error as { code?: string }).code === 'ERR_CANCELED'))
) {
return Promise.reject(error);
}
return Promise.reject(parseS3UploadError({ t, error, maxSize }));
}
};
packages/web/i18n/en/common.json
......@@ -1185,6 +1185,7 @@
"upgrade": "upgrade",
"upload_file": "Upload File",
"upload_file_error": "File Upload Failed",
"empty_file": "Empty files cannot be uploaded",
"usage_records": "Usage records",
"user.Account": "Account",
"user.No_right_to_reset_password": "You do not have the right to reset the password",
......
packages/web/i18n/zh-CN/common.json
......@@ -1185,6 +1185,7 @@
"upgrade": "升级",
"upload_file": "上传文件",
"upload_file_error": "上传文件失败",
"empty_file": "不能上传空文件",
"usage_records": "使用记录",
"user.Account": "账号",
"user.No_right_to_reset_password": "没有重置密码的权限",
......
packages/web/i18n/zh-Hant/common.json
......@@ -1174,6 +1174,7 @@
"upgrade": "升級",
"upload_file": "上傳檔案",
"upload_file_error": "上傳檔案失敗",
"empty_file": "不能上傳空檔案",
"user.Account": "帳戶",
"user.No_right_to_reset_password": "沒有重置密碼的權限",
"user.Old password is error": "舊密碼錯誤",
......
pnpm-workspace.yaml
......@@ -82,7 +82,7 @@ catalog:
request-ip: ^3.3.0
tsdown: 0.21.10
tsx: ^4.20.6
turbo: 2.10.5
turbo: 2.10.7
typescript: ^6.0.3
undici: ^7.28.0
vaul: ^1.1.2
......
pro @ e6c7fe1c
Subproject commit 7d3e9e1aaf7728faaae221bee43986c466f38b16
Subproject commit e6c7fe1c8036e34d4ba6e8d06189ad2ea9ab3b11
projects/app/src/components/core/app/FileSelector/index.tsx
......@@ -39,7 +39,7 @@ import { getErrText } from '@fastgpt/global/common/error/utils';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { WorkflowRuntimeContext } from '@/components/core/chat/ChatContainer/context/workflowRuntimeContext';
import { useSafeTranslation } from '@fastgpt/web/hooks/useSafeTranslation';
import { putFileToS3 } from '@fastgpt/web/common/file/utils';
import { S3FileUploader } from '@fastgpt/web/common/file/uploader';
import {
getFileSelectorDisplayIcon,
hasFileSelectorError,
......@@ -420,21 +420,21 @@ const FileSelector = ({
...chatAuthTarget,
chatId
};
const { url, key, headers, previewUrl } =
const uploadResult =
fileUploadMode === 'draft'
? await getUploadDraftChatFilePresignedUrl({
...uploadParams,
fileSelectConfig
})
: await getUploadChatFilePresignedUrl(uploadParams);
const { key, previewUrl } = uploadResult;
await putFileToS3({
url,
const uploader = new S3FileUploader({
...uploadResult,
file: file.rawFile,
headers,
onUploadProgress: (e) => {
if (!e.total) return;
const percent = Math.round((e.loaded / e.total) * 100);
onProgress: (loaded, total) => {
if (!total) return;
const percent = Math.round((loaded / total) * 100);
files.forEach((item) => {
if (item.id === file.id) {
item.process = percent;
......@@ -445,6 +445,7 @@ const FileSelector = ({
t,
maxSize
});
await uploader.upload();
// Update file url and key
markFileSelectorUploadSuccess({
......
projects/app/src/components/core/chat/ChatContainer/ChatBox/hooks/useFileUpload.tsx
......@@ -19,7 +19,7 @@ import {
getUploadDraftChatFilePresignedUrl
} from '@/web/common/file/api';
import { getUploadFileType } from '@fastgpt/global/core/app/constants';
import { putFileToS3 } from '@fastgpt/web/common/file/utils';
import { S3FileUploader } from '@fastgpt/web/common/file/uploader';
import { getUploadChatFileType } from '../utils/file';
import { type ChatSourceTarget, useChatAuthApiTarget } from '@/web/core/chat/utils';
import {
......@@ -340,7 +340,7 @@ export const useFileUpload = (props: UseFileUploadOptions) => {
...chatAuthTarget,
chatId
};
const { url, key, headers, maxSize, previewUrl } =
const uploadResult =
fileUploadMode === 'draft'
? await getUploadDraftChatFilePresignedUrl(
{
......@@ -352,8 +352,20 @@ export const useFileUpload = (props: UseFileUploadOptions) => {
: await getUploadChatFilePresignedUrl(uploadParams, {
cancelToken: task.controller
});
const { key, previewUrl } = uploadResult;
task.key = key;
const uploader = new S3FileUploader({
...uploadResult,
file: rawFile,
onProgress: (loaded, total) => {
if (!total) return;
const percent = Math.round((loaded / total) * 100);
updateFileByUploadId(uploadId, { process: percent, status: 1 });
},
signal: task.controller.signal,
t
});
if (
!canApplyUploadResult({
files: fileListRef.current,
......@@ -361,23 +373,12 @@ export const useFileUpload = (props: UseFileUploadOptions) => {
canceled: task.canceled
})
) {
await uploader.abort();
return;
}
// Upload File to S3
await putFileToS3({
url,
file: rawFile,
headers,
onUploadProgress: (e) => {
if (!e.total) return;
const percent = Math.round((e.loaded / e.total) * 100);
updateFileByUploadId(uploadId, { process: percent, status: 1 });
},
signal: task.controller.signal,
t,
maxSize
});
await uploader.upload();
// Update file url and key
updateFileByUploadId(uploadId, {
......
projects/app/src/pageComponents/app/detail/components/QuickCreateDatasetModal.tsx
import React, { useState, useMemo } from 'react';
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useTranslation } from 'next-i18next';
import { useForm } from 'react-hook-form';
import {
......@@ -31,7 +31,7 @@ import FileSelector, {
type SelectFileItemType
} from '@/pageComponents/dataset/detail/Import/components/FileSelector';
import { useRouter } from 'next/router';
import { putFileToS3 } from '@fastgpt/web/common/file/utils';
import { S3FileUploader } from '@fastgpt/web/common/file/uploader';
import type { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
const QuickCreateDatasetModal = ({
......@@ -54,6 +54,20 @@ const QuickCreateDatasetModal = ({
const defaultVLLM = defaultModels.datasetImageLLM?.model;
const [selectFiles, setSelectFiles] = useState<ImportSourceItemType[]>([]);
const uploadControllers = useRef(new Map<string, AbortController>());
useEffect(() => {
return () => {
uploadControllers.current.forEach((controller) => controller.abort());
uploadControllers.current.clear();
};
}, []);
const handleClose = useCallback(() => {
uploadControllers.current.forEach((controller) => controller.abort());
uploadControllers.current.clear();
onClose();
}, [onClose]);
const successFiles = useMemo(
() => selectFiles.filter((item) => item.dbFileId && !item.errorMsg),
......@@ -81,18 +95,27 @@ const QuickCreateDatasetModal = ({
async (files: SelectFileItemType[]) => {
await Promise.all(
files.map(async ({ fileId, file }) => {
const controller = new AbortController();
uploadControllers.current.set(fileId, controller);
try {
const { url, key, headers, maxSize } = await getUploadTempFilePresignedUrl({
filename: file.name
});
const uploadResult = await getUploadTempFilePresignedUrl(
{
filename: file.name,
size: file.size
},
{
cancelToken: controller
}
);
const { key } = uploadResult;
await putFileToS3({
url,
const uploader = new S3FileUploader({
...uploadResult,
file,
headers,
onUploadProgress: (e) => {
if (!e.total) return;
const percent = Math.round((e.loaded / e.total) * 100);
onProgress: (loaded, total) => {
if (!total) return;
const percent = Math.round((loaded / total) * 100);
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
......@@ -107,23 +130,30 @@ const QuickCreateDatasetModal = ({
);
},
t,
maxSize,
onSuccess: () => {
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
dbFileId: key,
isUploading: false,
uploadedFileRate: 100
}
: item
)
);
}
signal: controller.signal
});
if (controller.signal.aborted) {
await uploader.abort();
return;
}
await uploader.upload();
if (controller.signal.aborted) return;
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
dbFileId: key,
isUploading: false,
uploadedFileRate: 100
}
: item
)
);
} catch (error) {
if (controller.signal.aborted) return;
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
......@@ -135,6 +165,8 @@ const QuickCreateDatasetModal = ({
: item
)
);
} finally {
uploadControllers.current.delete(fileId);
}
})
);
......@@ -197,7 +229,7 @@ const QuickCreateDatasetModal = ({
return (
<MyModal
isOpen={true}
onClose={onClose}
onClose={handleClose}
title={t('app:Create_dataset')}
minW={'800px'}
ml={'20px'}
......@@ -333,7 +365,7 @@ const QuickCreateDatasetModal = ({
</Box>
</Flex>
<Flex gap={3}>
<Button variant={'whiteBase'} onClick={onClose}>
<Button variant={'whiteBase'} onClick={handleClose}>
{t('common:Cancel')}
</Button>
<Button
......
projects/app/src/pageComponents/dataset/detail/Import/components/RenderFiles.tsx
......@@ -20,10 +20,12 @@ import MyTag from '@fastgpt/web/components/common/Tag/index';
export const RenderUploadFiles = ({
files,
setFiles
setFiles,
onCancelUpload
}: {
files: ImportSourceItemType[];
setFiles: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
onCancelUpload?: (fileId: string) => void;
}) => {
const { t } = useTranslation();
......@@ -85,7 +87,15 @@ export const RenderUploadFiles = ({
</Td>
<Td>{item.sourceSize}</Td>
<Td>
{!item.isUploading && (
{item.isUploading ? (
<IconButton
variant={'grayDanger'}
size={'sm'}
icon={<MyIcon name={'close'} w={'14px'} />}
aria-label={t('common:Cancel')}
onClick={() => onCancelUpload?.(item.id)}
/>
) : (
<Flex alignItems={'center'} gap={4}>
<IconButton
variant={'grayDanger'}
......
projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileLocal.tsx
import React, { useCallback, useEffect, useMemo, useState } from 'react';
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { type ImportSourceItemType } from '@/web/core/dataset/type';
import { Box, Button } from '@chakra-ui/react';
import FileSelector, { type SelectFileItemType } from '../components/FileSelector';
......@@ -14,7 +14,7 @@ import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { getFileIcon } from '@fastgpt/global/common/file/icon';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { getUploadDatasetFilePresignedUrl } from '@/web/core/dataset/api/file';
import { putFileToS3 } from '@fastgpt/web/common/file/utils';
import { S3FileUploader } from '@fastgpt/web/common/file/uploader';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'));
const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
......@@ -49,9 +49,17 @@ const SelectFile = React.memo(function SelectFile() {
...source
}))
);
const uploadControllers = useRef(new Map<string, AbortController>());
const successFiles = useMemo(() => selectFiles.filter((item) => !item.errorMsg), [selectFiles]);
useEffect(() => {
return () => {
uploadControllers.current.forEach((controller) => controller.abort());
uploadControllers.current.clear();
};
}, []);
useEffect(() => {
setSources(successFiles);
}, [setSources, successFiles]);
......@@ -66,51 +74,64 @@ const SelectFile = React.memo(function SelectFile() {
{
await Promise.all(
files.map(async ({ fileId, file }) => {
try {
const { url, key, headers, maxSize } = await getUploadDatasetFilePresignedUrl({
filename: file.name,
datasetId
});
const controller = new AbortController();
uploadControllers.current.set(fileId, controller);
// Upload File to S3
await putFileToS3({
url,
file,
headers,
maxSize,
onUploadProgress: (e) => {
if (!e.total) return;
const percent = Math.round((e.loaded / e.total) * 100);
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
uploadedFileRate: item.uploadedFileRate
? Math.max(percent, item.uploadedFileRate)
: percent
}
: item
)
);
try {
const uploadResult = await getUploadDatasetFilePresignedUrl(
{
filename: file.name,
datasetId,
size: file.size
},
t,
onSuccess: () => {
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
dbFileId: key,
isUploading: false,
uploadedFileRate: 100
}
: item
)
);
{
cancelToken: controller
}
);
const updateProgress = (loaded: number, total: number) => {
if (!total) return;
const percent = Math.min(100, Math.round((loaded / total) * 100));
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
uploadedFileRate: Math.max(item.uploadedFileRate ?? 0, percent)
}
: item
)
);
};
const uploader = new S3FileUploader({
...uploadResult,
file,
signal: controller.signal,
onProgress: updateProgress,
t
});
if (controller.signal.aborted) {
await uploader.abort();
return;
}
await uploader.upload();
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
dbFileId: uploadResult.key,
isUploading: false,
uploadedFileRate: 100
}
: item
)
);
} catch (error) {
if (controller.signal.aborted) return;
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
......@@ -122,6 +143,8 @@ const SelectFile = React.memo(function SelectFile() {
: item
)
);
} finally {
uploadControllers.current.delete(fileId);
}
})
);
......@@ -152,12 +175,21 @@ const SelectFile = React.memo(function SelectFile() {
}
);
const cancelUpload = useCallback((fileId: string) => {
uploadControllers.current.get(fileId)?.abort();
setSelectFiles((state) => state.filter((file) => file.id !== fileId));
}, []);
return (
<Box>
<FileSelector fileType={fileType} selectFiles={selectFiles} onSelectFiles={onSelectFiles} />
{/* render files */}
<RenderUploadFiles files={selectFiles} setFiles={setSelectFiles} />
<RenderUploadFiles
files={selectFiles}
setFiles={setSelectFiles}
onCancelUpload={cancelUpload}
/>
<Box textAlign={'right'} mt={5}>
<Button isDisabled={successFiles.length === 0 || uploading} onClick={onclickNext}>
......
projects/app/src/pageComponents/dataset/detail/Test/hooks/useSearchTestImages.tsx
......@@ -5,7 +5,7 @@ import { getUploadSearchTestImagePresignedUrl } from '@/web/core/dataset/api/fil
import { useUserStore } from '@/web/support/user/useUserStore';
import { imageFileType } from '@fastgpt/global/common/file/constants';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { putFileToS3 } from '@fastgpt/web/common/file/utils';
import { S3FileUploader } from '@fastgpt/web/common/file/uploader';
import { useToast } from '@fastgpt/web/hooks/useToast';
import {
IMAGE_EXTENSION_SET,
......@@ -57,8 +57,16 @@ export const useSearchTestImages = ({
.catch(() => undefined));
const maxImageSize =
(planStatus?.standard?.maxUploadFileSize ?? uploadFileMaxSize ?? 500) * 1024 * 1024;
const validImageFiles = imageFiles.filter((file) => file.size <= maxImageSize);
if (validImageFiles.length < imageFiles.length) {
const nonEmptyImageFiles = imageFiles.filter((file) => file.size > 0);
if (nonEmptyImageFiles.length < imageFiles.length) {
toast({
status: 'warning',
title: t('file:empty_file')
});
}
const validImageFiles = nonEmptyImageFiles.filter((file) => file.size <= maxImageSize);
if (validImageFiles.length < nonEmptyImageFiles.length) {
toast({
status: 'warning',
title: t('file:some_file_size_exceeds_limit', {
......@@ -85,18 +93,18 @@ export const useSearchTestImages = ({
try {
const uploadedImages = await Promise.all(
uploadFiles.map(async (file) => {
const { url, key, headers, maxSize, previewUrl } =
await getUploadSearchTestImagePresignedUrl({
datasetId,
filename: file.name
});
await putFileToS3({
url,
headers,
const uploadResult = await getUploadSearchTestImagePresignedUrl({
datasetId,
filename: file.name,
size: file.size
});
const { key, previewUrl } = uploadResult;
const uploader = new S3FileUploader({
...uploadResult,
file,
maxSize,
t
});
await uploader.upload();
return { key, previewUrl };
})
);
......
projects/app/src/pages/api/common/file/presignAvatarPostUrl.ts
......@@ -2,14 +2,20 @@ import type { ApiRequestProps, ApiResponseType } from '@fastgpt/next/type';
import { NextAPI } from '@/service/middleware/entry';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { getS3AvatarSource } from '@fastgpt/service/common/s3/sources/avatar';
import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { BoolSchema } from '@fastgpt/global/common/zod';
import type z from 'zod';
import {
PresignFileUploadParamsSchema,
type CreatePostPresignedUrlResponseType
} from '@fastgpt/global/common/file/s3/type';
export type updateAvatarQuery = Record<string, never>;
export type updateAvatarBody = {
filename: string;
autoExpired?: boolean;
};
export const UpdateAvatarBodySchema = PresignFileUploadParamsSchema.extend({
autoExpired: BoolSchema.optional()
});
export type updateAvatarBody = z.infer<typeof UpdateAvatarBodySchema>;
export type updateAvatarResponse = CreatePostPresignedUrlResponseType;
......@@ -17,12 +23,16 @@ async function handler(
req: ApiRequestProps<updateAvatarBody, updateAvatarQuery>,
_: ApiResponseType<updateAvatarResponse>
): Promise<updateAvatarResponse> {
const { filename, autoExpired } = req.body;
const { filename, size, autoExpired } = parseApiInput({
req,
bodySchema: UpdateAvatarBodySchema
}).body;
const { teamId } = await authCert({ req, authToken: true });
return await getS3AvatarSource().createUploadAvatarURL({
teamId,
filename,
size,
autoExpired
});
}
......
projects/app/src/pages/api/common/file/presignTempFilePostUrl.ts
import type { ApiRequestProps } from '@fastgpt/next/type';
import { NextAPI } from '@/service/middleware/entry';
import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type';
import {
PresignFileUploadParamsSchema,
type CreatePostPresignedUrlResponseType,
type PresignFileUploadParams
} from '@fastgpt/global/common/file/s3/type';
import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
import { TeamDatasetCreatePermissionVal } from '@fastgpt/global/support/permission/user/constant';
import { getFileS3Key } from '@fastgpt/service/common/s3/utils';
......@@ -8,15 +12,17 @@ import { S3PrivateBucket } from '@fastgpt/service/common/s3/buckets/private';
import { authFrequencyLimit } from '@fastgpt/service/common/system/frequencyLimit/utils';
import { addSeconds } from 'date-fns';
import { getTeamPlanStatus } from '@fastgpt/service/support/wallet/sub/utils';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
export type PresignTempFilePostUrlParams = {
filename: string;
};
export type PresignTempFilePostUrlParams = PresignFileUploadParams;
async function handler(
req: ApiRequestProps<PresignTempFilePostUrlParams>
): Promise<CreatePostPresignedUrlResponseType> {
const { filename } = req.body;
const { filename, size } = parseApiInput({
req,
bodySchema: PresignFileUploadParamsSchema
}).body;
const { teamId, tmbId } = await authUserPer({
req,
......@@ -35,8 +41,8 @@ async function handler(
const bucket = new S3PrivateBucket();
const { fileKey } = getFileS3Key.temp({ teamId, filename });
return await bucket.createPresignedPutUrl(
{ rawKey: fileKey, filename },
return await bucket.createUploadAccessUrl(
{ rawKey: fileKey, filename, ...(size !== undefined ? { size } : {}) },
{
expiredHours: 1,
maxFileSize: planStatus.standard?.maxUploadFileSize ?? global.feConfigs.uploadFileMaxSize
......
projects/app/src/pages/api/core/dataset/file/presignDatasetFilePostUrl.ts
......@@ -8,15 +8,16 @@ import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'
import { getTeamPlanStatus } from '@fastgpt/service/support/wallet/sub/utils';
import {
PresignDatasetFilePostUrlBodySchema,
type PresignDatasetFilePostUrlBody
type PresignDatasetFilePostUrlBody,
PresignDatasetFilePostUrlResponseSchema,
type PresignDatasetFilePostUrlResponse
} from '@fastgpt/global/openapi/core/dataset/file/api';
import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
async function handler(
req: ApiRequestProps<PresignDatasetFilePostUrlBody>
): Promise<CreatePostPresignedUrlResponseType> {
const { filename, datasetId } = parseApiInput({
): Promise<PresignDatasetFilePostUrlResponse> {
const { filename, datasetId, size } = parseApiInput({
req,
bodySchema: PresignDatasetFilePostUrlBodySchema
}).body;
......@@ -36,11 +37,14 @@ async function handler(
expiredTime: addSeconds(new Date(), 30) // 30s
});
return getS3DatasetSource().createUploadDatasetFileURL({
const result = await getS3DatasetSource().createUploadDatasetFileURL({
datasetId,
filename,
size,
maxFileSize: planStatus.standard?.maxUploadFileSize ?? global.feConfigs.uploadFileMaxSize
});
return PresignDatasetFilePostUrlResponseSchema.parse(result);
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/file/presignSearchTestImage.ts
......@@ -10,6 +10,7 @@ import { parseAllowedExtensions } from '@fastgpt/service/common/s3/utils/uploadC
import { getFileS3Key } from '@fastgpt/service/common/s3/utils';
import { S3PrivateBucket } from '@fastgpt/service/common/s3/buckets/private';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { createUploadConstraints } from '@fastgpt/service/common/s3/utils/uploadConstraints';
import {
PresignSearchTestImageBodySchema,
PresignSearchTestImageResponseSchema,
......@@ -20,7 +21,7 @@ import {
async function handler(
req: ApiRequestProps<PresignSearchTestImageBody>
): Promise<PresignSearchTestImageResponse> {
const { datasetId, filename } = parseApiInput({
const { datasetId, filename, size } = parseApiInput({
req,
bodySchema: PresignSearchTestImageBodySchema
}).body;
......@@ -42,14 +43,18 @@ async function handler(
const bucket = new S3PrivateBucket();
const { fileKey } = getFileS3Key.temp({ teamId, filename });
const result = await bucket.createPresignedPutUrl(
{ rawKey: fileKey, filename },
const uploadPolicy = createUploadConstraints({
filename,
uploadConstraints: {
allowedExtensions: parseAllowedExtensions(imageFileType)
}
});
const result = await bucket.createUploadAccessUrl(
{ rawKey: fileKey, filename, ...(size !== undefined ? { size } : {}) },
{
expiredHours: 3,
maxFileSize: planStatus.standard?.maxUploadFileSize ?? global.feConfigs.uploadFileMaxSize,
uploadConstraints: {
allowedExtensions: parseAllowedExtensions(imageFileType)
}
uploadPolicy
}
);
......
projects/app/src/pages/api/system/file/u/[token].ts
......@@ -6,22 +6,48 @@ import {
verifyS3UploadSessionToken
} from '@fastgpt/service/common/s3/accessLink';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { handleS3ProxyRouteError, handleS3ProxyUpload } from '@/service/common/s3/proxy';
import {
handleS3ProxyRouteError,
handleS3ProxyUpload,
handleS3ProxyUploadPart
} from '@/service/common/s3/proxy';
import { jsonRes } from '@fastgpt/service/common/response';
import {
UploadDatasetFileMultipartPartQuerySchema,
UploadDatasetFileMultipartPartResponseSchema
} from '@fastgpt/global/openapi/core/dataset/file/api';
const S3UploadAccessRouteWithPartQuerySchema = S3UploadAccessRouteQuerySchema.extend({
partNumber: UploadDatasetFileMultipartPartQuerySchema.shape.partNumber.optional()
});
async function handler(req: ApiRequestProps, res: NextApiResponse) {
if (req.method !== 'PUT') {
return jsonRes(res, { code: 405, error: 'Method not allowed' });
}
const { token } = parseApiInput({
const { token, partNumber } = parseApiInput({
req,
querySchema: S3UploadAccessRouteQuerySchema
querySchema: S3UploadAccessRouteWithPartQuerySchema
}).query;
try {
const payload = await verifyS3UploadSessionToken(token);
if (partNumber !== undefined) {
if (!payload.multipart) {
throw new Error('Not a multipart upload session');
}
return UploadDatasetFileMultipartPartResponseSchema.parse(
await handleS3ProxyUploadPart({ req, token, payload, partNumber })
);
}
if (payload.multipart) {
throw new Error('Multipart upload requires partNumber');
}
return await handleS3ProxyUpload({ req, payload });
} catch (error) {
return handleS3ProxyRouteError({ res, error });
......
projects/app/src/pages/api/system/file/u/[token]/abort.ts 0 → 100644
import type { ApiRequestProps } from '@fastgpt/next/type';
import type { NextApiResponse } from 'next';
import { NextAPI } from '@/service/middleware/entry';
import { S3UploadAccessRouteQuerySchema } from '@fastgpt/service/common/s3/accessLink';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { AbortDatasetFileMultipartUploadResponseSchema } from '@fastgpt/global/openapi/core/dataset/file/api';
import { handleS3AbortMultipartUpload, handleS3ProxyRouteError } from '@/service/common/s3/proxy';
import { jsonRes } from '@fastgpt/service/common/response';
async function handler(req: ApiRequestProps, res: NextApiResponse): Promise<void> {
if (req.method !== 'POST') {
return jsonRes(res, { code: 405, error: 'Method not allowed' });
}
const { token } = parseApiInput({
req,
querySchema: S3UploadAccessRouteQuerySchema
}).query;
try {
await handleS3AbortMultipartUpload(token);
return AbortDatasetFileMultipartUploadResponseSchema.parse(undefined);
} catch (error) {
return handleS3ProxyRouteError({ res, error });
}
}
export default NextAPI(handler);
projects/app/src/pages/api/system/file/u/[token]/complete.ts 0 → 100644
import type { ApiRequestProps } from '@fastgpt/next/type';
import type { NextApiResponse } from 'next';
import { NextAPI } from '@/service/middleware/entry';
import { S3UploadAccessRouteQuerySchema } from '@fastgpt/service/common/s3/accessLink';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import {
CompleteDatasetFileMultipartUploadBodySchema,
CompleteDatasetFileMultipartUploadResponseSchema,
type CompleteDatasetFileMultipartUploadResponse
} from '@fastgpt/global/openapi/core/dataset/file/api';
import {
handleS3CompleteMultipartUpload,
handleS3ProxyRouteError
} from '@/service/common/s3/proxy';
import { jsonRes } from '@fastgpt/service/common/response';
async function handler(
req: ApiRequestProps,
res: NextApiResponse
): Promise<CompleteDatasetFileMultipartUploadResponse | void> {
if (req.method !== 'POST') {
return jsonRes(res, { code: 405, error: 'Method not allowed' });
}
const { query, body } = parseApiInput({
req,
querySchema: S3UploadAccessRouteQuerySchema,
bodySchema: CompleteDatasetFileMultipartUploadBodySchema
});
const { token } = query;
const { parts } = body;
try {
const result = await handleS3CompleteMultipartUpload({ token, parts });
return CompleteDatasetFileMultipartUploadResponseSchema.parse(result);
} catch (error) {
return handleS3ProxyRouteError({ res, error });
}
}
export default NextAPI(handler);
projects/app/src/pages/api/system/file/upload/[token].ts
......@@ -31,7 +31,7 @@ async function handler(req: ApiRequestProps, res: NextApiResponse) {
objectKey,
bucketName,
maxSize,
uploadConstraints,
uploadPolicy: uploadConstraints,
metadata
}
});
......
projects/app/src/service/common/s3/proxy.ts
......@@ -8,11 +8,12 @@ import { ERROR_RESPONSE } from '@fastgpt/global/common/error/errorCode';
import { jsonRes } from '@fastgpt/service/common/response';
import { getLogger, LogCategories } from '@fastgpt/service/common/logger';
import { isS3AccessLinkError } from '@fastgpt/service/common/s3/accessLink';
import type { MultipartUploadPart } from '@fastgpt-sdk/storage';
import type {
S3ProxyDownloadPayload,
S3ProxyUploadPayload
} from '@fastgpt/service/common/s3/accessLink';
import type { UploadConstraints } from '@fastgpt/service/common/s3/contracts/type';
import { verifyS3MultipartUploadSessionToken } from '@fastgpt/service/common/s3/accessLink';
import type { UploadFileHint, UploadPolicy } from '@fastgpt/service/common/s3/uploadPolicy/type';
import {
DEFAULT_CONTENT_TYPE,
......@@ -32,8 +33,7 @@ const logger = getLogger(LogCategories.INFRA.FILE);
type GuardStreamOptions = {
maxSize: number;
uploadConstraints: UploadConstraints;
uploadPolicy?: UploadPolicy;
uploadPolicy: UploadPolicy;
fileHint: UploadFileHint;
};
......@@ -108,14 +108,8 @@ export const parseS3ProxyContentLength = (value: string | string[] | undefined)
return parsed;
};
const createUploadGuardStream = ({
maxSize,
uploadConstraints,
uploadPolicy,
fileHint
}: GuardStreamOptions) => {
const policy = uploadPolicy || uploadConstraints;
const inspectBytes = getUploadInspectBytes({ hint: fileHint, policy });
const createUploadGuardStream = ({ maxSize, uploadPolicy, fileHint }: GuardStreamOptions) => {
const inspectBytes = getUploadInspectBytes({ hint: fileHint, policy: uploadPolicy });
let uploadedBytes = 0;
let bufferedBytes = 0;
const chunks: Buffer[] = [];
......@@ -155,8 +149,7 @@ const createUploadGuardStream = ({
const result = await validateUploadFile({
buffer,
filename: fileHint.filename,
uploadConstraints,
uploadPolicy: policy,
uploadPolicy,
fileHint
});
......@@ -421,8 +414,7 @@ export const handleS3ProxyUpload = async ({
req: NextApiRequest;
payload: S3ProxyUploadPayload;
}) => {
const { objectKey, bucketName, maxSize, uploadConstraints, uploadPolicy, fileHint, metadata } =
payload;
const { objectKey, bucketName, maxSize, uploadPolicy, fileHint, metadata } = payload;
const bucket = global.s3BucketMap[bucketName];
if (!bucket) {
......@@ -437,7 +429,6 @@ export const handleS3ProxyUpload = async ({
const resolvedFileHint = resolveProxyUploadFileHint({ objectKey, metadata, fileHint });
const { stream: guardStream, validatedUpload } = createUploadGuardStream({
maxSize,
uploadConstraints,
uploadPolicy,
fileHint: resolvedFileHint
});
......@@ -459,6 +450,178 @@ export const handleS3ProxyUpload = async ({
return { success: true };
};
const resolveExpectedMultipartPartLength = ({
payload,
partNumber
}: {
payload: S3ProxyUploadPayload;
partNumber: number;
}) => {
const multipart = payload.multipart;
if (!multipart) {
throw new Error('Not a multipart upload session');
}
const partCount = Math.ceil(multipart.totalSize / multipart.partSize);
if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > partCount) {
throw new Error('Multipart part number is out of range');
}
return partNumber === partCount
? multipart.totalSize - multipart.partSize * (partCount - 1)
: multipart.partSize;
};
/**
* 限制单个 Multipart 分片的实际字节数，防止客户端伪造 Content-Length 或发送超长 body。
* 只有最后一个分片允许小于 partSize，具体期望长度由 session 推导。
*/
const createMultipartPartLengthGuard = (expectedLength: number) => {
let receivedLength = 0;
return new Transform({
transform(chunk, _encoding, callback) {
receivedLength += chunk.length;
if (receivedLength > expectedLength) {
callback(new Error('Multipart part length does not match session'));
return;
}
callback(null, chunk);
},
flush(callback) {
if (receivedLength !== expectedLength) {
callback(new Error('Multipart part length does not match session'));
return;
}
callback();
}
});
};
/**
* 代理 Multipart 分片上传。请求 body 始终以 stream 传给 storage，part 1 额外复用现有文件
* 内容校验；客户端断开时主动销毁中间 stream，使底层 SDK 尽快停止读取该分片。
*/
export const handleS3ProxyUploadPart = async ({
req,
token,
payload,
partNumber
}: {
req: NextApiRequest;
token: string;
payload: S3ProxyUploadPayload;
partNumber: number;
}): Promise<{ etag: string }> => {
const { objectKey, bucketName, maxSize, uploadPolicy, fileHint, metadata } = payload;
const bucket = global.s3BucketMap[bucketName];
if (!bucket) {
throw new Error('S3 bucket not found');
}
const contentLength = parseS3ProxyContentLength(req.headers['content-length']);
if (contentLength === undefined) {
throw new Error('Multipart part content-length is required');
}
const expectedLength = resolveExpectedMultipartPartLength({ payload, partNumber });
if (contentLength !== expectedLength) {
throw new Error('Multipart part length does not match session');
}
const lengthGuard = createMultipartPartLengthGuard(expectedLength);
const resolvedFileHint = resolveProxyUploadFileHint({ objectKey, metadata, fileHint });
const contentGuard =
partNumber === 1
? createUploadGuardStream({
maxSize,
uploadPolicy,
fileHint: resolvedFileHint
})
: undefined;
const uploadStream = contentGuard?.stream ?? lengthGuard;
const validatedUpload = contentGuard?.validatedUpload;
const destroyUploadStreams = (error: Error) => {
if (!lengthGuard.destroyed) lengthGuard.destroy(error);
if (!uploadStream.destroyed) uploadStream.destroy(error);
};
const abortUpload = () => {
// 正常请求结束也可能触发 close；只有未完整读取的请求需要销毁上传流。
if (req.complete || req.readableEnded) return;
destroyUploadStreams(new Error('Multipart upload request aborted'));
};
if (req.aborted) {
abortUpload();
throw new Error('Multipart upload request aborted');
}
req.once('aborted', abortUpload);
req.once('error', abortUpload);
req.once('close', abortUpload);
try {
const uploadPromise = bucket.uploadMultipartPart({
token,
partNumber,
body: uploadStream,
contentLength
});
const requestPipeline = contentGuard
? pipeline(req, lengthGuard, contentGuard.stream)
: pipeline(req, lengthGuard);
const result = validatedUpload
? await Promise.all([validatedUpload, uploadPromise, requestPipeline]).then(
([, uploadResult]) => uploadResult
)
: await Promise.all([uploadPromise, requestPipeline]).then(([uploadResult]) => uploadResult);
return { etag: result.etag };
} catch (error) {
destroyUploadStreams(error instanceof Error ? error : new Error(String(error)));
throw error;
} finally {
req.off('aborted', abortUpload);
req.off('error', abortUpload);
req.off('close', abortUpload);
}
};
/** 通过 opaque token 定位 bucket，并完成 Multipart 对象合并。 */
export const handleS3CompleteMultipartUpload = async ({
token,
parts
}: {
token: string;
parts: MultipartUploadPart[];
}) => {
const payload = await verifyS3MultipartUploadSessionToken(token);
const bucket = global.s3BucketMap[payload.bucketName];
if (!bucket) {
throw new Error('S3 bucket not found');
}
return bucket.completeMultipartUpload({ token, parts });
};
/** 通过 opaque token 定位 bucket，并取消未完成的 Multipart 对象。 */
export const handleS3AbortMultipartUpload = async (token: string) => {
const payload = await verifyS3MultipartUploadSessionToken(token);
const bucket = global.s3BucketMap[payload.bucketName];
if (!bucket) {
throw new Error('S3 bucket not found');
}
await bucket.abortMultipartUpload({ token });
};
const getProxyErrorKey = (error: unknown) => {
if (typeof error === 'string') return error;
if (error instanceof Error) return error.message;
......@@ -512,6 +675,23 @@ export function resolveS3ProxyErrorResponse(error: unknown): {
};
}
if (
errorKey === 'Not a multipart upload session' ||
errorKey === 'Multipart part content-length is required' ||
errorKey === 'Multipart part length does not match session' ||
errorKey === 'Multipart part number is out of range' ||
errorKey === 'Multipart upload request aborted' ||
errorKey === 'Multipart upload requires partNumber' ||
errorKey === 'Multipart parts count does not match total size' ||
errorKey?.startsWith('Multipart parts must be') ||
errorKey?.startsWith('Multipart upload session is')
) {
return {
httpStatus: 400,
publicError: error
};
}
const errorResponse = errorKey ? ERROR_RESPONSE[errorKey] : undefined;
if (
typeof errorResponse?.code === 'number' &&
......
projects/app/src/service/core/dataset/queues/datasetParse.ts
......@@ -34,6 +34,7 @@ import { pushLLMTrainingUsage } from '@fastgpt/service/support/wallet/usage/cont
import { UsageItemTypeEnum } from '@fastgpt/global/support/wallet/usage/constants';
import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
import { i18nT } from '@fastgpt/global/common/i18n/utils';
import { createParseTaskLease, PARSE_QUEUE_LEASE_TIMEOUT_MINUTES } from './parseLease';
const logger = getLogger(LogCategories.MODULE.DATASET.FILE_PARSE);
......@@ -113,23 +114,27 @@ export const datasetParseQueue = async (): Promise<any> => {
while (true) {
const startTime = Date.now();
// 1. Get task and lock 20 minutes ago
// 1. Get task and lock 10 minutes ago
const {
data,
done = false,
error = false
} = await (async () => {
try {
const claimedLockTime = new Date();
const data = await MongoDatasetTraining.findOneAndUpdate(
{
mode: TrainingModeEnum.parse,
retryCount: { $gt: 0 },
lockTime: { $lte: addMinutes(new Date(), -10) }
lockTime: {
$lte: addMinutes(new Date(), -PARSE_QUEUE_LEASE_TIMEOUT_MINUTES)
}
},
{
lockTime: new Date(),
lockTime: claimedLockTime,
$inc: { retryCount: -1 }
}
},
{ new: true }
)
.populate<{
dataset: DatasetSchemaType;
......@@ -183,7 +188,15 @@ export const datasetParseQueue = async (): Promise<any> => {
collectionId: data.collectionId,
trainingId: data._id
});
await MongoDatasetTraining.deleteOne({ _id: data._id });
const deleteResult = await MongoDatasetTraining.deleteOne({
_id: data._id,
lockTime: data.lockTime
});
if (deleteResult.deletedCount !== 1) {
logger.warn('Parse queue task lease lost before deleting incomplete task', {
trainingId: data._id
});
}
continue;
}
......@@ -197,6 +210,31 @@ export const datasetParseQueue = async (): Promise<any> => {
trainingType: collection.trainingType
});
const taskLease = createParseTaskLease({
taskId: data._id,
lockTime: data.lockTime,
updateLock: async (filter, nextLockTime) => {
const result = await MongoDatasetTraining.updateOne(filter, {
lockTime: nextLockTime
});
return result.matchedCount === 1;
},
onLost: () => {
logger.warn('Parse queue task lease lost', {
trainingId: data._id,
datasetId: data.datasetId,
collectionId: data.collectionId
});
},
onError: (error) => {
logger.warn('Parse queue task lease heartbeat failed', {
trainingId: data._id,
error
});
}
});
taskLease.start();
try {
const trainingMode = getTrainingModeByCollection({
trainingType: collection.trainingType ?? DatasetCollectionDataProcessModeEnum.chunk,
......@@ -252,9 +290,13 @@ export const datasetParseQueue = async (): Promise<any> => {
collectionId: data.collectionId,
collectionType: collection.type
});
await MongoDatasetTraining.deleteOne({
_id: data._id
});
await taskLease.stop();
const deleteResult = await MongoDatasetTraining.deleteOne(taskLease.getFilter());
if (deleteResult.deletedCount !== 1) {
logger.warn('Parse queue task lease lost before deleting invalid task', {
trainingId: data._id
});
}
continue;
}
......@@ -315,6 +357,8 @@ export const datasetParseQueue = async (): Promise<any> => {
chunkIndex: index
}));
// 成功写入前先停止续租，并等待正在进行的 heartbeat 完成，保证下面使用最新 lease。
await taskLease.stop();
await mongoSessionRun(async (session) => {
// 5. Update collection title(Link)
await MongoDatasetCollection.updateOne(
......@@ -344,14 +388,12 @@ export const datasetParseQueue = async (): Promise<any> => {
});
// 7. Delete task
await MongoDatasetTraining.deleteOne(
{
_id: data._id
},
{
session
}
);
const deleteResult = await MongoDatasetTraining.deleteOne(taskLease.getFilter(), {
session
});
if (deleteResult.deletedCount !== 1) {
throw new Error('Parse queue task lease lost before completion');
}
});
logger.debug('Parse queue task finished', {
......@@ -361,21 +403,17 @@ export const datasetParseQueue = async (): Promise<any> => {
collectionId: data.collectionId
});
} catch (err) {
await taskLease.stop();
if (err === TeamErrEnum.datasetSizeNotEnough) {
logger.info('Parse queue dataset limit exceeded, locking task', {
trainingId: data._id,
datasetId: data.datasetId,
collectionId: data.collectionId
});
await MongoDatasetTraining.updateOne(
{
_id: data._id
},
{
errorMsg: i18nT('common:code_error.team_error.dataset_size_not_enough'),
lockTime: new Date('2999/5/5')
}
);
await MongoDatasetTraining.updateOne(taskLease.getFilter(), {
errorMsg: i18nT('common:code_error.team_error.dataset_size_not_enough'),
lockTime: new Date('2999/5/5')
});
continue;
}
......@@ -387,17 +425,14 @@ export const datasetParseQueue = async (): Promise<any> => {
collectionId: data.collectionId
});
await MongoDatasetTraining.updateOne(
{
_id: data._id
},
{
errorMsg: getErrText(err, 'unknown error'),
lockTime: addMinutes(new Date(), -10)
}
);
await MongoDatasetTraining.updateOne(taskLease.getFilter(), {
errorMsg: getErrText(err, 'unknown error'),
lockTime: addMinutes(new Date(), -PARSE_QUEUE_LEASE_TIMEOUT_MINUTES)
});
await delay(100);
} finally {
await taskLease.stop();
}
}
} catch (error) {
......
projects/app/src/service/core/dataset/queues/parseLease.ts 0 → 100644
export const PARSE_QUEUE_LEASE_TIMEOUT_MINUTES = 10;
export const PARSE_QUEUE_LEASE_HEARTBEAT_INTERVAL_MS = 60 * 1000;
export type ParseTaskLeaseFilter<TTaskId = unknown> = {
_id: TTaskId;
lockTime: Date;
};
type CreateParseTaskLeaseParams<TTaskId> = {
taskId: TTaskId;
lockTime: Date;
updateLock: (filter: ParseTaskLeaseFilter<TTaskId>, nextLockTime: Date) => Promise<boolean>;
intervalMs?: number;
onLost?: () => void;
onError?: (error: unknown) => void;
};
export type ParseTaskLease = {
getFilter: () => ParseTaskLeaseFilter;
isLost: () => boolean;
heartbeat: () => Promise<void>;
start: () => void;
stop: () => Promise<void>;
};
/**
* 创建基于 lockTime 的乐观 lease。
*
* heartbeat 只在当前 lockTime 仍匹配时续租；匹配失败代表任务已经被其他 worker
* 领取，后续写操作继续使用最后一个 lease 条件，从而不会覆盖新 worker 的状态。
*/
export const createParseTaskLease = <TTaskId>({
taskId,
lockTime,
updateLock,
intervalMs = PARSE_QUEUE_LEASE_HEARTBEAT_INTERVAL_MS,
onLost,
onError
}: CreateParseTaskLeaseParams<TTaskId>): ParseTaskLease => {
let expectedLockTime = lockTime;
let timer: ReturnType<typeof setInterval> | undefined;
let pendingHeartbeat: Promise<void> | undefined;
let stopped = false;
let lost = false;
const handleLost = () => {
lost = true;
if (timer) {
clearInterval(timer);
timer = undefined;
}
try {
onLost?.();
} catch {
// lease 状态已经丢失，通知日志失败不应影响 heartbeat 的收尾。
}
};
const heartbeat = async () => {
if (stopped || lost) return;
const nextLockTime = new Date();
try {
const renewed = await updateLock(
{
_id: taskId,
lockTime: expectedLockTime
},
nextLockTime
);
if (!renewed) {
handleLost();
return;
}
expectedLockTime = nextLockTime;
} catch (error) {
try {
onError?.(error);
} catch {
// heartbeat 错误只影响本轮续租，不能留下未处理 rejection。
}
}
};
const start = () => {
if (stopped || lost || timer) return;
timer = setInterval(() => {
if (pendingHeartbeat) return;
const currentHeartbeat = heartbeat();
pendingHeartbeat = currentHeartbeat;
const clearPendingHeartbeat = () => {
if (pendingHeartbeat === currentHeartbeat) {
pendingHeartbeat = undefined;
}
};
void currentHeartbeat.then(clearPendingHeartbeat, clearPendingHeartbeat);
}, intervalMs);
};
const stop = async () => {
stopped = true;
if (timer) {
clearInterval(timer);
timer = undefined;
}
await pendingHeartbeat;
};
return {
getFilter: () => ({
_id: taskId,
lockTime: expectedLockTime
}),
isLost: () => lost,
heartbeat,
start,
stop
};
};
projects/app/src/web/common/file/api.ts
......@@ -4,12 +4,16 @@ import type {
PresignChatFilePostUrlParams,
PresignDraftChatFilePostUrlParams
} from '@fastgpt/global/openapi/core/chat/file/api';
import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type';
import type {
CreatePostPresignedUrlResponseType,
PresignFileUploadParams
} from '@fastgpt/global/common/file/s3/type';
export const getUploadAvatarPresignedUrl = (params: {
filename: string;
autoExpired?: boolean;
}) => {
export const getUploadAvatarPresignedUrl = (
params: PresignFileUploadParams & {
autoExpired?: boolean;
}
) => {
return POST<CreatePostPresignedUrlResponseType>('/common/file/presignAvatarPostUrl', params);
};
......@@ -39,6 +43,13 @@ export const getPresignedChatFileGetUrl = (params: PresignChatFileGetUrlParams)
return POST<string>('/core/chat/file/presignChatFileGetUrl', params);
};
export const getUploadTempFilePresignedUrl = (params: { filename: string }) => {
return POST<CreatePostPresignedUrlResponseType>('/common/file/presignTempFilePostUrl', params);
export const getUploadTempFilePresignedUrl = (
params: PresignFileUploadParams,
config?: Parameters<typeof POST>[2]
) => {
return POST<CreatePostPresignedUrlResponseType>(
'/common/file/presignTempFilePostUrl',
params,
config
);
};
projects/app/src/web/core/dataset/api/file.ts
......@@ -7,13 +7,20 @@ import type {
GetRawTextPreviewChunksBody,
GetRawTextPreviewChunksResponse,
PresignDatasetFilePostUrlBody,
PresignDatasetFilePostUrlResponse,
PresignSearchTestImageBody,
PresignSearchTestImageResponse
} from '@fastgpt/global/openapi/core/dataset/file/api';
import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type';
export const getUploadDatasetFilePresignedUrl = (params: PresignDatasetFilePostUrlBody) =>
POST<CreatePostPresignedUrlResponseType>('/core/dataset/file/presignDatasetFilePostUrl', params);
export const getUploadDatasetFilePresignedUrl = (
params: PresignDatasetFilePostUrlBody,
config?: Parameters<typeof POST>[2]
) =>
POST<PresignDatasetFilePostUrlResponse>(
'/core/dataset/file/presignDatasetFilePostUrl',
params,
config
);
export const getPreviewChunks = (data: GetPreviewChunksBody) =>
POST<GetPreviewChunksResponse>('/core/dataset/file/getPreviewChunks', data, {
......
projects/app/test/service/common/s3/proxy.test.ts
import { EventEmitter } from 'node:events';
import { PassThrough, Readable, Writable } from 'node:stream';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { handleS3ProxyDownload } from '@/service/common/s3/proxy';
import { handleS3ProxyDownload, handleS3ProxyUploadPart } from '@/service/common/s3/proxy';
const createRequest = (method = 'GET') =>
Object.assign(new EventEmitter(), {
......@@ -180,3 +180,52 @@ describe('handleS3ProxyDownload', () => {
expect(req.listenerCount('aborted')).toBe(0);
});
});
describe('handleS3ProxyUploadPart', () => {
it('destroys the storage stream when an incomplete request closes', async () => {
const req = new PassThrough() as any;
Object.assign(req, {
headers: { 'content-length': '4' },
aborted: false,
complete: false
});
let uploadStream: Readable | undefined;
const uploadMultipartPart = vi.fn(async ({ body }: { body: Readable }) => {
uploadStream = body;
return { etag: 'etag-2' };
});
global.s3BucketMap = {
'fastgpt-private': {
uploadMultipartPart
}
} as any;
const uploadPromise = handleS3ProxyUploadPart({
req,
token: 'multipart-token',
partNumber: 2,
payload: {
bucketName: 'fastgpt-private',
objectKey: 'dataset/team/file.bin',
maxSize: 1024,
uploadPolicy: {
defaultContentType: 'application/octet-stream'
},
multipart: {
uploadId: 'upload-1',
partSize: 4,
totalSize: 8,
status: 'active'
}
}
});
await vi.waitFor(() => expect(uploadMultipartPart).toHaveBeenCalled());
req.emit('close');
req.destroy();
await expect(uploadPromise).rejects.toBeTruthy();
expect(uploadStream?.destroyed).toBe(true);
});
});
projects/app/test/service/core/dataset/queues/parseLease.test.ts 0 → 100644
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
createParseTaskLease,
PARSE_QUEUE_LEASE_HEARTBEAT_INTERVAL_MS
} from '@/service/core/dataset/queues/parseLease';
describe('createParseTaskLease', () => {
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
it('按当前 lockTime 条件续租，并将成功后的时间用于下一次续租', async () => {
const initialLockTime = new Date('2026-07-30T00:00:00.000Z');
const updateLock = vi.fn().mockResolvedValue(true);
const lease = createParseTaskLease({
taskId: 'training-1',
lockTime: initialLockTime,
updateLock
});
lease.start();
await vi.advanceTimersByTimeAsync(PARSE_QUEUE_LEASE_HEARTBEAT_INTERVAL_MS);
expect(updateLock).toHaveBeenCalledTimes(1);
const firstCall = updateLock.mock.calls[0];
expect(firstCall[0]).toEqual({
_id: 'training-1',
lockTime: initialLockTime
});
expect(firstCall[1]).toBeInstanceOf(Date);
await vi.advanceTimersByTimeAsync(PARSE_QUEUE_LEASE_HEARTBEAT_INTERVAL_MS);
expect(updateLock).toHaveBeenCalledTimes(2);
expect(updateLock.mock.calls[1][0]).toEqual({
_id: 'training-1',
lockTime: firstCall[1]
});
expect(lease.getFilter()).toEqual({
_id: 'training-1',
lockTime: updateLock.mock.calls[1][1]
});
await lease.stop();
expect(vi.getTimerCount()).toBe(0);
});
it('续租条件不匹配时标记 lease 丢失并停止 heartbeat', async () => {
const initialLockTime = new Date('2026-07-30T00:00:00.000Z');
const onLost = vi.fn();
const updateLock = vi.fn().mockResolvedValue(false);
const lease = createParseTaskLease({
taskId: 'training-1',
lockTime: initialLockTime,
updateLock,
onLost
});
lease.start();
await vi.advanceTimersByTimeAsync(PARSE_QUEUE_LEASE_HEARTBEAT_INTERVAL_MS);
await vi.advanceTimersByTimeAsync(PARSE_QUEUE_LEASE_HEARTBEAT_INTERVAL_MS * 2);
expect(updateLock).toHaveBeenCalledTimes(1);
expect(onLost).toHaveBeenCalledTimes(1);
expect(lease.isLost()).toBe(true);
expect(lease.getFilter()).toEqual({
_id: 'training-1',
lockTime: initialLockTime
});
await lease.stop();
});
it('heartbeat 更新异常时通知调用方，并在下一轮继续尝试', async () => {
const error = new Error('temporary mongo error');
const onError = vi.fn();
const updateLock = vi.fn().mockRejectedValueOnce(error).mockResolvedValueOnce(true);
const lease = createParseTaskLease({
taskId: 'training-1',
lockTime: new Date('2026-07-30T00:00:00.000Z'),
updateLock,
onError
});
lease.start();
await vi.advanceTimersByTimeAsync(PARSE_QUEUE_LEASE_HEARTBEAT_INTERVAL_MS);
await vi.advanceTimersByTimeAsync(PARSE_QUEUE_LEASE_HEARTBEAT_INTERVAL_MS);
expect(onError).toHaveBeenCalledWith(error);
expect(updateLock).toHaveBeenCalledTimes(2);
expect(lease.isLost()).toBe(false);
await lease.stop();
});
it('停止后不会启动或执行新的 heartbeat', async () => {
const updateLock = vi.fn().mockResolvedValue(true);
const lease = createParseTaskLease({
taskId: 'training-1',
lockTime: new Date(),
updateLock,
intervalMs: 1000
});
await lease.heartbeat();
await lease.stop();
lease.start();
await vi.advanceTimersByTimeAsync(2000);
expect(updateLock).toHaveBeenCalledTimes(1);
});
it('不会并发执行 heartbeat，并在 stop 时等待当前续租完成', async () => {
let resolveUpdate: ((renewed: boolean) => void) | undefined;
const updateLock = vi.fn(
() =>
new Promise<boolean>((resolve) => {
resolveUpdate = resolve;
})
);
const lease = createParseTaskLease({
taskId: 'training-1',
lockTime: new Date('2026-07-30T00:00:00.000Z'),
updateLock,
intervalMs: 1000
});
lease.start();
await vi.advanceTimersByTimeAsync(1000);
await vi.advanceTimersByTimeAsync(5000);
expect(updateLock).toHaveBeenCalledTimes(1);
const stopPromise = lease.stop();
let stopped = false;
void stopPromise.then(() => {
stopped = true;
});
await Promise.resolve();
expect(stopped).toBe(false);
resolveUpdate?.(true);
await stopPromise;
expect(stopped).toBe(true);
});
});
sdk/storage/.env.example
# Integration tests are opt-in. Copy the required provider block to .env.test.local.
# BUCKET must start with fastgpt-sdk-. Its existing contents are deleted before every suite.
# Opt-in Multipart benchmark. It only uses STORAGE_PRIVATE_BUCKET by default and cleans its test keys.
STORAGE_MULTIPART_BENCHMARK_ENV_FILE=
STORAGE_MULTIPART_BENCHMARK_BUCKET=
STORAGE_MULTIPART_BENCHMARK_SIZE_BYTES=33554432
STORAGE_MULTIPART_BENCHMARK_PART_SIZE_BYTES=8388608
STORAGE_MULTIPART_BENCHMARK_CONCURRENCY=3
STORAGE_MULTIPART_BENCHMARK_MAX_RETRY=3
STORAGE_MULTIPART_BENCHMARK_RETRY_BASE_DELAY_MS=300
STORAGE_MULTIPART_BENCHMARK_ROUNDS=1
STORAGE_MULTIPART_BENCHMARK_MAX_DURATION_MS=900000
STORAGE_MULTIPART_BENCHMARK_ABORT_PROBE=true
# Local MinIO
STORAGE_TEST_MINIO_ENABLED=false
STORAGE_TEST_MINIO_BUCKET=fastgpt-sdk.integration-bucket-1
STORAGE_TEST_MINIO_ENDPOINT=http://127.0.0.1:9000
STORAGE_TEST_MINIO_REGION=us-east-1
STORAGE_TEST_MINIO_ACCESS_KEY_ID=minioadmin
STORAGE_TEST_MINIO_SECRET_ACCESS_KEY=minioadmin
STORAGE_TEST_MINIO_PUBLIC_BUCKET=
STORAGE_TEST_MINIO_PRIVATE_BUCKET=
STORAGE_TEST_MINIO_PUBLIC_ENDPOINT=
STORAGE_TEST_MINIO_PRIVATE_BUCKET=fastgpt-sdk.integration-bucket-1-private
STORAGE_TEST_MINIO_PUBLIC_BUCKET=fastgpt-sdk.integration-bucket-1-public
STORAGE_TEST_MINIO_PUBLIC_ENDPOINT=http://127.0.0.1:9000
# AWS S3 or another S3-compatible endpoint
STORAGE_TEST_AWS_S3_ENABLED=false
STORAGE_TEST_AWS_S3_BUCKET=
STORAGE_TEST_AWS_S3_ENDPOINT=https://s3.amazonaws.com
STORAGE_TEST_AWS_S3_REGION=us-east-1
STORAGE_TEST_AWS_S3_ACCESS_KEY_ID=
STORAGE_TEST_AWS_S3_SECRET_ACCESS_KEY=
STORAGE_TEST_AWS_S3_FORCE_PATH_STYLE=false
STORAGE_TEST_AWS_S3_PRIVATE_BUCKET=
STORAGE_TEST_AWS_S3_PUBLIC_BUCKET=
STORAGE_TEST_AWS_S3_PUBLIC_ENDPOINT=
# Cloudflare R2
STORAGE_TEST_R2_ENABLED=false
......@@ -33,7 +46,6 @@ STORAGE_TEST_R2_PUBLIC_ENDPOINT=
# Alibaba Cloud OSS
STORAGE_TEST_OSS_ENABLED=false
STORAGE_TEST_OSS_BUCKET=
STORAGE_TEST_OSS_ENDPOINT=
STORAGE_TEST_OSS_REGION=
STORAGE_TEST_OSS_ACCESS_KEY_ID=
......@@ -45,11 +57,10 @@ STORAGE_TEST_OSS_PUBLIC_ENDPOINT=
# Tencent Cloud COS
STORAGE_TEST_COS_ENABLED=false
# Full COS bucket name; it must end with -<STORAGE_TEST_COS_APP_ID>.
STORAGE_TEST_COS_BUCKET=
STORAGE_TEST_COS_REGION=
STORAGE_TEST_COS_APP_ID=
STORAGE_TEST_COS_ACCESS_KEY_ID=
STORAGE_TEST_COS_SECRET_ACCESS_KEY=
STORAGE_TEST_COS_PUBLIC_BUCKET=
STORAGE_TEST_COS_PRIVATE_BUCKET=
STORAGE_TEST_COS_PUBLIC_BUCKET=
STORAGE_TEST_COS_PUBLIC_ENDPOINT=
sdk/storage/package.json
{
"name": "@fastgpt-sdk/storage",
"private": false,
"version": "0.8.0",
"version": "0.9.0",
"type": "module",
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
......@@ -67,7 +67,7 @@
"vitest": "catalog:"
},
"devDependencies": {
"@types/ali-oss": "^6.16.13",
"@types/ali-oss": "^6.23.3",
"@types/node": "^20",
"tsdown": "catalog:",
"typescript": "catalog:"
......
sdk/storage/src/access-link/constants.ts
......@@ -2,6 +2,7 @@ export const S3_DOWNLOAD_ALIAS_SIGN_VERSION = 'v1';
export const S3_DOWNLOAD_ALIAS_ID_LENGTH = 16;
export const S3_DOWNLOAD_SIGNATURE_LENGTH = 22;
export const S3_UPLOAD_TOKEN_LENGTH = 22;
export const S3_MULTIPART_MAX_PART_COUNT = 10_000;
export const S3_DOWNLOAD_URL_BATCH_MAX_SIZE = 500;
export const S3_ACCESS_LINK_PURGE_GRACE_HOURS = 24;
// 在 alias 距离新链接过期时间只剩 1 小时时续租，避免每次签发都写 store。
......
sdk/storage/src/access-link/crypto.ts
......@@ -15,7 +15,8 @@ const randomUrlSafeToken = (length: number) => {
export const createDefaultIdGenerator = () => ({
aliasId: () => randomUrlSafeToken(S3_DOWNLOAD_ALIAS_ID_LENGTH),
uploadToken: () => randomUrlSafeToken(S3_UPLOAD_TOKEN_LENGTH)
uploadToken: () => randomUrlSafeToken(S3_UPLOAD_TOKEN_LENGTH),
multipartCompletionAttemptId: () => randomUrlSafeToken(32)
});
export const createS3AccessLinkCrypto = ({
......
sdk/storage/src/access-link/guards.ts
import { S3_SIGNED_DOWNLOAD_ALIAS_PATTERN, S3_UPLOAD_TOKEN_PATTERN } from './constants';
import {
S3_MULTIPART_MAX_PART_COUNT,
S3_SIGNED_DOWNLOAD_ALIAS_PATTERN,
S3_UPLOAD_TOKEN_PATTERN
} from './constants';
import { S3AccessLinkErrCode, S3AccessLinkError, type S3AccessLinkErrorCode } from './errors';
import type {
CreateS3DownloadAccessUrlParams,
......@@ -8,6 +12,7 @@ import type {
S3UploadConstraints,
S3UploadExtensionRule,
S3UploadFileHint,
S3MultipartUploadSession,
S3UploadPolicy
} from './types';
......@@ -48,6 +53,63 @@ const assertNonNegativeNumber = (value: unknown, code: S3AccessLinkErrorCode): n
return value;
};
const assertPositiveInteger = (value: unknown, code: S3AccessLinkErrorCode): number => {
const result = assertPositiveNumber(value, code);
if (!Number.isSafeInteger(result)) throw new S3AccessLinkError(code);
return result;
};
const assertMultipartSession = (
value: unknown,
code: S3AccessLinkErrorCode,
mode: 'create' | 'read' = 'read'
): S3MultipartUploadSession | undefined => {
if (value === undefined) return undefined;
if (!isRecord(value)) throw new S3AccessLinkError(code);
const status = value.status;
if (mode === 'create' && status !== 'active') {
throw new S3AccessLinkError(code);
}
if (
status !== 'active' &&
status !== 'completing' &&
status !== 'completed' &&
status !== 'aborted'
) {
throw new S3AccessLinkError(code);
}
const totalSize = assertPositiveInteger(value.totalSize, code);
const partSize = assertPositiveInteger(value.partSize, code);
if (Math.ceil(totalSize / partSize) > S3_MULTIPART_MAX_PART_COUNT) {
throw new S3AccessLinkError(code);
}
if (
mode === 'create' &&
['completionAttemptId', 'completingAt', 'completedAt', 'abortedAt'].some((key) => key in value)
) {
throw new S3AccessLinkError(code);
}
return {
uploadId: assertNonEmptyString(value.uploadId, code),
partSize,
totalSize,
status,
...(value.completionAttemptId !== undefined
? { completionAttemptId: assertNonEmptyString(value.completionAttemptId, code) }
: {}),
...(value.completingAt !== undefined
? { completingAt: assertDate(value.completingAt, code) }
: {}),
...(value.completedAt !== undefined
? { completedAt: assertDate(value.completedAt, code) }
: {}),
...(value.abortedAt !== undefined ? { abortedAt: assertDate(value.abortedAt, code) } : {})
};
};
const assertStringRecord = (
value: unknown,
code: S3AccessLinkErrorCode
......@@ -97,11 +159,7 @@ const assertStringArray = (value: unknown, code: S3AccessLinkErrorCode): string[
return value;
};
const assertUploadPolicy = (
value: unknown,
code: S3AccessLinkErrorCode
): S3UploadPolicy | undefined => {
if (value === undefined) return undefined;
const assertUploadPolicy = (value: unknown, code: S3AccessLinkErrorCode): S3UploadPolicy => {
if (!isRecord(value)) throw new S3AccessLinkError(code);
const base = assertUploadConstraints(value, code);
......@@ -236,28 +294,34 @@ export const assertCreateUploadParams = (
): CreateS3UploadAccessUrlParams => {
if (!isRecord(params)) throw new S3AccessLinkError(S3AccessLinkErrCode.uploadSessionNotFound);
const maxSize = assertPositiveInteger(params.maxSize, S3AccessLinkErrCode.uploadSessionNotFound);
const multipart =
params.multipart === undefined
? undefined
: assertMultipartSession(
params.multipart,
S3AccessLinkErrCode.uploadSessionNotFound,
'create'
);
if (multipart && multipart.totalSize > maxSize) {
throw new S3AccessLinkError(S3AccessLinkErrCode.uploadSessionNotFound);
}
return {
bucketName: assertNonEmptyString(params.bucketName, S3AccessLinkErrCode.uploadSessionNotFound),
objectKey: assertNonEmptyString(params.objectKey, S3AccessLinkErrCode.uploadSessionNotFound),
expiredTime: assertDate(params.expiredTime, S3AccessLinkErrCode.uploadSessionNotFound),
maxSize: assertPositiveNumber(params.maxSize, S3AccessLinkErrCode.uploadSessionNotFound),
uploadConstraints: assertUploadConstraints(
params.uploadConstraints,
maxSize,
uploadPolicy: assertUploadPolicy(
params.uploadPolicy,
S3AccessLinkErrCode.uploadSessionNotFound
),
...(params.uploadPolicy !== undefined
? {
uploadPolicy: assertUploadPolicy(
params.uploadPolicy,
S3AccessLinkErrCode.uploadSessionNotFound
)
}
: {}),
...(params.fileHint !== undefined
? {
fileHint: assertUploadFileHint(params.fileHint, S3AccessLinkErrCode.uploadSessionNotFound)
}
: {}),
...(multipart !== undefined ? { multipart } : {}),
...(params.metadata !== undefined
? {
metadata: assertStringRecord(params.metadata, S3AccessLinkErrCode.uploadSessionNotFound)
......@@ -275,22 +339,23 @@ export const assertUploadTokenFormat = (token: string) => {
export const assertUploadPayload = (payload: S3ProxyUploadPayload): S3ProxyUploadPayload => {
if (!isRecord(payload)) throw new S3AccessLinkError(S3AccessLinkErrCode.uploadSessionNotFound);
const maxSize = assertPositiveInteger(payload.maxSize, S3AccessLinkErrCode.uploadSessionNotFound);
const multipart =
payload.multipart === undefined
? undefined
: assertMultipartSession(payload.multipart, S3AccessLinkErrCode.uploadSessionNotFound);
if (multipart && multipart.totalSize > maxSize) {
throw new S3AccessLinkError(S3AccessLinkErrCode.uploadSessionNotFound);
}
return {
bucketName: assertNonEmptyString(payload.bucketName, S3AccessLinkErrCode.uploadSessionNotFound),
objectKey: assertNonEmptyString(payload.objectKey, S3AccessLinkErrCode.uploadSessionNotFound),
maxSize: assertPositiveNumber(payload.maxSize, S3AccessLinkErrCode.uploadSessionNotFound),
uploadConstraints: assertUploadConstraints(
payload.uploadConstraints,
maxSize,
uploadPolicy: assertUploadPolicy(
payload.uploadPolicy,
S3AccessLinkErrCode.uploadSessionNotFound
),
...(payload.uploadPolicy !== undefined
? {
uploadPolicy: assertUploadPolicy(
payload.uploadPolicy,
S3AccessLinkErrCode.uploadSessionNotFound
)
}
: {}),
...(payload.fileHint !== undefined
? {
fileHint: assertUploadFileHint(
......@@ -299,6 +364,7 @@ export const assertUploadPayload = (payload: S3ProxyUploadPayload): S3ProxyUploa
)
}
: {}),
...(multipart !== undefined ? { multipart } : {}),
...(payload.metadata !== undefined
? {
metadata: assertStringRecord(payload.metadata, S3AccessLinkErrCode.uploadSessionNotFound)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment