Commit 9aa9709a by Xianquan Committed by GitHub

fix(storage): avoid MinIO XML entity expansion limit (#7356)

* fix(storage): avoid MinIO XML entity expansion limit

* fix(storage): add MinIO delete request timeout

* fix(storage): handle MinIO delete failures and timeouts

* test(storage): add shared adapter integration suite

* test(storage): expand adapter boundary coverage

* test(storage): recreate stable integration buckets

* fix(storage): align object key and delete contracts

* docs(storage): remove integration test design

* fix(storage): stabilize integration test setup

* fix(storage): keep legacy test mock import path

---------

Co-authored-by: Archer <545436317@qq.com>
parent 43bd4f9c
...@@ -26,3 +26,4 @@ FE_DOMAIN=https://fastgpt.example.com ...@@ -26,3 +26,4 @@ FE_DOMAIN=https://fastgpt.example.com
1. Fixed an issue where Chatbox displayed system tool errors during streaming responses. 1. Fixed an issue where Chatbox displayed system tool errors during streaming responses.
2. Fixed an issue where plain-text tool responses in full run details could be incorrectly parsed as Markdown, causing formatting issues. 2. Fixed an issue where plain-text tool responses in full run details could be incorrectly parsed as Markdown, causing formatting issues.
3. Fixed an issue where switching the embedding model triggered training but did not rebuild vectors for existing data. 3. Fixed an issue where switching the embedding model triggered training but did not rebuild vectors for existing data.
4. Fixed MinIO prefix-based bulk deletion failures caused by the XML entity expansion limit and added request timeout protection.
...@@ -24,3 +24,4 @@ FE_DOMAIN=https://fastgpt.example.com ...@@ -24,3 +24,4 @@ FE_DOMAIN=https://fastgpt.example.com
1. chatbox 流输出时候,不应该展示系统工具的错误。 1. chatbox 流输出时候,不应该展示系统工具的错误。
2. 完整运行详情,纯文本的工具响应 UI 可能会被 Markdown 错误解析,格式错乱。 2. 完整运行详情,纯文本的工具响应 UI 可能会被 Markdown 错误解析,格式错乱。
3. 切换向量模型后,训练任务会触发但已有数据的向量未重建。 3. 切换向量模型后,训练任务会触发但已有数据的向量未重建。
4. 修复 MinIO 按前缀批量删除大量对象时,可能因 XML 实体展开限制失败的问题,并增加请求超时保护。
import z from 'zod'; import z from 'zod';
import { UploadConstraintsSchema } from '../contracts/type'; import { StorageObjectKeySchema, UploadConstraintsSchema } from '../contracts/type';
import { UploadFileHintSchema, UploadPolicySchema } from '../uploadPolicy/type'; import { UploadFileHintSchema, UploadPolicySchema } from '../uploadPolicy/type';
import { S3_DOWNLOAD_URL_BATCH_MAX_SIZE } from '@fastgpt-sdk/storage/access-link'; import { S3_DOWNLOAD_URL_BATCH_MAX_SIZE } from '@fastgpt-sdk/storage/access-link';
...@@ -10,7 +10,7 @@ const HexSha256Schema = z ...@@ -10,7 +10,7 @@ const HexSha256Schema = z
.regex(/^[a-f0-9]+$/); .regex(/^[a-f0-9]+$/);
export const S3AccessBucketNameSchema = z.string().min(1); export const S3AccessBucketNameSchema = z.string().min(1);
export const S3AccessObjectKeySchema = z.string().min(1); export const S3AccessObjectKeySchema = StorageObjectKeySchema;
export const S3DownloadAliasIdSchema = UrlSafeTokenSchema.min(12).max(32); export const S3DownloadAliasIdSchema = UrlSafeTokenSchema.min(12).max(32);
export const S3DownloadAliasKeySchema = HexSha256Schema; export const S3DownloadAliasKeySchema = HexSha256Schema;
......
...@@ -5,6 +5,19 @@ import { ...@@ -5,6 +5,19 @@ import {
UploadFileHintSchema, UploadFileHintSchema,
UploadPolicySchema UploadPolicySchema
} from '../uploadPolicy/type'; } from '../uploadPolicy/type';
import { assertStorageObjectKey } from '@fastgpt-sdk/storage';
/** FastGPT 入口与底层 Storage SDK 共用同一套对象 key 规范。 */
export const StorageObjectKeySchema = z.string().superRefine((key, context) => {
try {
assertStorageObjectKey(key);
} catch (error) {
context.addIssue({
code: z.ZodIssueCode.custom,
message: error instanceof Error ? error.message : 'Invalid storage object key'
});
}
});
export const S3MetadataSchema = z.object({ export const S3MetadataSchema = z.object({
filename: z.string(), filename: z.string(),
...@@ -38,7 +51,7 @@ export type StorageDownloadUrlMode = z.infer<typeof StorageDownloadUrlModeSchema ...@@ -38,7 +51,7 @@ export type StorageDownloadUrlMode = z.infer<typeof StorageDownloadUrlModeSchema
export const CreatePostPresignedUrlParamsSchema = z.object({ export const CreatePostPresignedUrlParamsSchema = z.object({
filename: z.string().min(1), filename: z.string().min(1),
rawKey: z.string().min(1), rawKey: StorageObjectKeySchema,
contentType: UploadFileHintSchema.shape.contentType, contentType: UploadFileHintSchema.shape.contentType,
declaredExtension: UploadFileHintSchema.shape.declaredExtension, declaredExtension: UploadFileHintSchema.shape.declaredExtension,
declaredFilename: UploadFileHintSchema.shape.declaredFilename, declaredFilename: UploadFileHintSchema.shape.declaredFilename,
...@@ -64,7 +77,7 @@ export const CreatePostPresignedUrlResultSchema = z.object({ ...@@ -64,7 +77,7 @@ export const CreatePostPresignedUrlResultSchema = z.object({
}); });
export type CreatePostPresignedUrlResult = z.infer<typeof CreatePostPresignedUrlResultSchema>; export type CreatePostPresignedUrlResult = z.infer<typeof CreatePostPresignedUrlResultSchema>;
export const CreateGetPresignedUrlParamsSchema = z.object({ export const CreateGetPresignedUrlParamsSchema = z.object({
key: z.string().nonempty(), key: StorageObjectKeySchema,
expiredHours: z.number().positive().optional(), expiredHours: z.number().positive().optional(),
responseContentType: z.string().nonempty().optional() responseContentType: z.string().nonempty().optional()
}); });
...@@ -74,7 +87,7 @@ export const UploadImage2S3BucketParamsSchema = z ...@@ -74,7 +87,7 @@ export const UploadImage2S3BucketParamsSchema = z
.object({ .object({
base64Img: z.string().nonempty().optional(), base64Img: z.string().nonempty().optional(),
buffer: z.instanceof(Buffer).optional(), buffer: z.instanceof(Buffer).optional(),
uploadKey: z.string().nonempty(), uploadKey: StorageObjectKeySchema,
mimetype: z.string().nonempty(), mimetype: z.string().nonempty(),
filename: z.string().nonempty(), filename: z.string().nonempty(),
expiredTime: z.coerce.date().optional() expiredTime: z.coerce.date().optional()
...@@ -87,7 +100,7 @@ export type UploadImage2S3BucketParams = z.infer<typeof UploadImage2S3BucketPara ...@@ -87,7 +100,7 @@ export type UploadImage2S3BucketParams = z.infer<typeof UploadImage2S3BucketPara
export const UploadFileByBodySchema = z.object({ export const UploadFileByBodySchema = z.object({
body: z.union([z.instanceof(Buffer), z.string(), z.instanceof(Readable)]), body: z.union([z.instanceof(Buffer), z.string(), z.instanceof(Readable)]),
contentType: z.string().optional(), contentType: z.string().optional(),
key: z.string().nonempty(), key: StorageObjectKeySchema,
filename: z.string().nonempty(), filename: z.string().nonempty(),
expiredTime: z.coerce.date().optional() expiredTime: z.coerce.date().optional()
}); });
......
...@@ -5,7 +5,11 @@ import type { Readable } from 'node:stream'; ...@@ -5,7 +5,11 @@ import type { Readable } from 'node:stream';
import { MongoS3TTL } from './models/ttl'; import { MongoS3TTL } from './models/ttl';
import { S3Buckets } from './config/constants'; import { S3Buckets } from './config/constants';
import { S3PrivateBucket } from './buckets/private'; import { S3PrivateBucket } from './buckets/private';
import { S3Sources, type UploadImage2S3BucketParams } from './contracts/type'; import {
S3Sources,
type UploadImage2S3BucketParams,
UploadImage2S3BucketParamsSchema
} from './contracts/type';
import { S3PublicBucket } from './buckets/public'; import { S3PublicBucket } from './buckets/public';
import { getNanoid } from '@fastgpt/global/common/string/tools'; import { getNanoid } from '@fastgpt/global/common/string/tools';
import path from 'node:path'; import path from 'node:path';
...@@ -123,7 +127,14 @@ export async function uploadImage2S3Bucket( ...@@ -123,7 +127,14 @@ export async function uploadImage2S3Bucket(
bucketName: keyof typeof S3Buckets, bucketName: keyof typeof S3Buckets,
params: UploadImage2S3BucketParams params: UploadImage2S3BucketParams
) { ) {
const { base64Img, buffer: inputBuffer, filename, mimetype, uploadKey, expiredTime } = params; const {
base64Img,
buffer: inputBuffer,
filename,
mimetype,
uploadKey,
expiredTime
} = UploadImage2S3BucketParamsSchema.parse(params);
const bucket = bucketName === 'private' ? new S3PrivateBucket() : new S3PublicBucket(); const bucket = bucketName === 'private' ? new S3PrivateBucket() : new S3PublicBucket();
......
...@@ -59,4 +59,21 @@ describe('executeS3DeleteJob', () => { ...@@ -59,4 +59,21 @@ describe('executeS3DeleteJob', () => {
}) })
).rejects.toThrow('Failed to delete 1 S3 object'); ).rejects.toThrow('Failed to delete 1 S3 object');
}); });
it('throws when multi-key deletion reports failed keys so BullMQ can retry', async () => {
global.s3BucketMap = {
'fastgpt-private': {
client: {
deleteObjectsByMultiKeys: vi.fn(async () => ({ keys: ['dataset/team/failed.txt'] }))
}
}
} as any;
await expect(
executeS3DeleteJob({
bucketName: 'fastgpt-private',
keys: ['dataset/team/deleted.txt', 'dataset/team/failed.txt']
})
).rejects.toThrow('Failed to delete 1 S3 object');
});
}); });
import { describe, expect, it } from 'vitest';
import {
CreateGetPresignedUrlParamsSchema,
CreatePostPresignedUrlParamsSchema,
StorageObjectKeySchema,
UploadFileByBodySchema,
UploadImage2S3BucketParamsSchema
} from '@fastgpt/service/common/s3/contracts/type';
import { S3AccessObjectKeySchema } from '@fastgpt/service/common/s3/accessLink/type';
type KeyParseResult =
| { success: true }
| { success: false; error: { issues: Array<{ message: string }> } };
const keySchemas: ReadonlyArray<readonly [string, (key: string) => KeyParseResult]> = [
['shared key', (key) => StorageObjectKeySchema.safeParse(key)],
[
'presigned PUT',
(key) => CreatePostPresignedUrlParamsSchema.safeParse({ filename: 'file.txt', rawKey: key })
],
['presigned GET', (key) => CreateGetPresignedUrlParamsSchema.safeParse({ key })],
[
'image upload',
(key) =>
UploadImage2S3BucketParamsSchema.safeParse({
uploadKey: key,
mimetype: 'image/png',
filename: 'file.png',
buffer: Buffer.from('image')
})
],
[
'body upload',
(key) => UploadFileByBodySchema.safeParse({ key, body: 'body', filename: 'file.txt' })
],
['access link', (key) => S3AccessObjectKeySchema.safeParse(key)]
];
describe('FastGPT storage object key schemas', () => {
it.each(keySchemas)('%s accepts portable URL-sensitive characters', (_name, parseKey) => {
expect(parseKey('team/folder # & + % ?/\u6587\u4ef6-\ud83d\ude00.txt').success).toBe(true);
});
it.each(keySchemas)('%s rejects a path the storage SDK would reject', (_name, parseKey) => {
const result = parseKey('team//file.txt');
expect(result.success).toBe(false);
if (!result.success) {
expect(result.error.issues[0]?.message).toContain('consecutive slashes');
}
});
it.each(keySchemas)('%s rejects a key beyond 850 UTF-8 bytes', (_name, parseKey) => {
expect(parseKey('a'.repeat(851)).success).toBe(false);
});
});
...@@ -5,10 +5,24 @@ import { ...@@ -5,10 +5,24 @@ import {
truncateFilename, truncateFilename,
S3_FILENAME_MAX_LENGTH, S3_FILENAME_MAX_LENGTH,
isS3ObjectKey, isS3ObjectKey,
getFileS3Key getFileS3Key,
uploadImage2S3Bucket
} from '@fastgpt/service/common/s3/utils'; } from '@fastgpt/service/common/s3/utils';
import * as stringTools from '@fastgpt/global/common/string/tools'; import * as stringTools from '@fastgpt/global/common/string/tools';
describe('uploadImage2S3Bucket', () => {
it('rejects an invalid key at the FastGPT boundary before uploading', async () => {
await expect(
uploadImage2S3Bucket('private', {
buffer: Buffer.from('image'),
filename: 'image.png',
mimetype: 'image/png',
uploadKey: 'dataset//image.png'
})
).rejects.toThrow('consecutive slashes');
});
});
describe('truncateFilename', () => { describe('truncateFilename', () => {
it('should return filename as-is if within max length', () => { it('should return filename as-is if within max length', () => {
const filename = 'short.pdf'; const filename = 'short.pdf';
......
...@@ -38,6 +38,11 @@ const sdkMocks = vi.hoisted(() => ({ ...@@ -38,6 +38,11 @@ const sdkMocks = vi.hoisted(() => ({
parsePkg: vi.fn() parsePkg: vi.fn()
})); }));
const loggerMocks = vi.hoisted(() => ({
warning: vi.fn(),
error: vi.fn()
}));
vi.mock('../../../src/service/mongo/models/tool', () => ({ vi.mock('../../../src/service/mongo/models/tool', () => ({
MarketplaceToolIndexZodSchema: { MarketplaceToolIndexZodSchema: {
parse: (value: any) => ({ parse: (value: any) => ({
...@@ -66,10 +71,7 @@ vi.mock('../../../src/service/logger', () => ({ ...@@ -66,10 +71,7 @@ vi.mock('../../../src/service/logger', () => ({
API: 'api' API: 'api'
} }
}, },
getLogger: () => ({ getLogger: () => loggerMocks
warning: vi.fn(),
error: vi.fn()
})
})); }));
const createIndex = ({ const createIndex = ({
...@@ -127,6 +129,8 @@ describe('PluginRepo', () => { ...@@ -127,6 +129,8 @@ describe('PluginRepo', () => {
} }
}); });
sdkMocks.parsePkg.mockReset(); sdkMocks.parsePkg.mockReset();
loggerMocks.warning.mockReset();
loggerMocks.error.mockReset();
Reflect.deleteProperty(globalThis, 'marketplacePluginManifestCache'); Reflect.deleteProperty(globalThis, 'marketplacePluginManifestCache');
}); });
...@@ -273,6 +277,29 @@ describe('PluginRepo', () => { ...@@ -273,6 +277,29 @@ describe('PluginRepo', () => {
); );
}); });
it('reports failed asset keys without failing a completed manifest publish', async () => {
const existing = createIndex({ pluginId: 'tool-a', version: '1.0.0', etag: 'old-etag' });
const record = createManifest(
createIndex({ pluginId: 'tool-a', version: '1.0.0', etag: 'new-etag' })
);
modelMocks.findOne.mockReturnValue(mockLean(existing));
modelMocks.updateOne.mockResolvedValue({ acknowledged: true });
s3Mocks.uploadJsonToS3.mockResolvedValue(undefined);
s3Mocks.deleteObjectsByPrefixFromS3.mockResolvedValue({
keys: ['assets/tool-a/1.0.0/old-etag/failed.svg']
});
const { PluginRepo } = await import('../../../src/service/plugin/repo');
await expect(new PluginRepo().publishToolManifest(record)).resolves.toBeUndefined();
expect(loggerMocks.warning).toHaveBeenCalledWith(
'Delete old marketplace tool assets partially failed',
expect.objectContaining({
failedKeys: ['assets/tool-a/1.0.0/old-etag/failed.svg']
})
);
});
it('rejects non-official manifests when the toolId already exists under another source', async () => { it('rejects non-official manifests when the toolId already exists under another source', async () => {
const record = createManifest( const record = createManifest(
createIndex({ createIndex({
......
# Integration tests are opt-in. Copy the required provider block to .env.test.local.
# BUCKET must start with fastgpt-sdk-. Its existing contents are deleted before every suite.
# Local MinIO
STORAGE_TEST_MINIO_ENABLED=false
STORAGE_TEST_MINIO_BUCKET=fastgpt-sdk.integration-bucket-1
STORAGE_TEST_MINIO_ENDPOINT=http://127.0.0.1:9000
STORAGE_TEST_MINIO_REGION=us-east-1
STORAGE_TEST_MINIO_ACCESS_KEY_ID=minioadmin
STORAGE_TEST_MINIO_SECRET_ACCESS_KEY=minioadmin
# AWS S3 or another S3-compatible endpoint
STORAGE_TEST_AWS_S3_ENABLED=false
STORAGE_TEST_AWS_S3_BUCKET=
STORAGE_TEST_AWS_S3_ENDPOINT=https://s3.amazonaws.com
STORAGE_TEST_AWS_S3_REGION=us-east-1
STORAGE_TEST_AWS_S3_ACCESS_KEY_ID=
STORAGE_TEST_AWS_S3_SECRET_ACCESS_KEY=
STORAGE_TEST_AWS_S3_FORCE_PATH_STYLE=false
# Alibaba Cloud OSS
STORAGE_TEST_OSS_ENABLED=false
STORAGE_TEST_OSS_BUCKET=
STORAGE_TEST_OSS_ENDPOINT=
STORAGE_TEST_OSS_REGION=
STORAGE_TEST_OSS_ACCESS_KEY_ID=
STORAGE_TEST_OSS_SECRET_ACCESS_KEY=
# Tencent Cloud COS
STORAGE_TEST_COS_ENABLED=false
# Full COS bucket name; it must end with -<STORAGE_TEST_COS_APP_ID>.
STORAGE_TEST_COS_BUCKET=
STORAGE_TEST_COS_REGION=
STORAGE_TEST_COS_APP_ID=
STORAGE_TEST_COS_ACCESS_KEY_ID=
STORAGE_TEST_COS_SECRET_ACCESS_KEY=
.env
.env.local
.env.test
...@@ -148,14 +148,14 @@ const storage = createStorage({ ...@@ -148,14 +148,14 @@ const storage = createStorage({
> 重要:当前实现状态(以代码为准): > 重要:当前实现状态(以代码为准):
> - `generatePresignedPutUrl`:**AWS S3 / MinIO / COS / OSS 已实现**。 > - `generatePresignedPutUrl`:**AWS S3 / MinIO / COS / OSS 已实现**。
> - `generatePresignedGetUrl`:目前各 adapter 仍为 **未实现**(会抛 `Error('Method not implemented.')`)。 > - `generatePresignedGetUrl`:**AWS S3 / MinIO / COS / OSS 已实现**。
### 预签名 PUT 直传示例(浏览器 / 前端) ### 预签名 PUT 直传示例(浏览器 / 前端)
`generatePresignedPutUrl` 返回的 `metadata` 字段语义更接近“需要带上的 headers”(不同厂商前缀不同,如 `x-oss-meta-*` / `x-cos-meta-*`)。 `generatePresignedPutUrl` 返回的 `metadata` 字段语义更接近“需要带上的 headers”(不同厂商前缀不同,如 `x-oss-meta-*` / `x-cos-meta-*`)。
```ts ```ts
const { putUrl, metadata } = await storage.generatePresignedPutUrl({ const { url: putUrl, metadata } = await storage.generatePresignedPutUrl({
key: 'demo/hello.txt', key: 'demo/hello.txt',
expiredSeconds: 600, expiredSeconds: 600,
metadata: { app: 'fastgpt', purpose: 'direct-upload' } metadata: { app: 'fastgpt', purpose: 'direct-upload' }
...@@ -179,11 +179,13 @@ await fetch(putUrl, { ...@@ -179,11 +179,13 @@ await fetch(putUrl, {
- **`NoSuchBucketError`**: bucket 不存在(部分 adapter 会用它包装底层错误)。 - **`NoSuchBucketError`**: bucket 不存在(部分 adapter 会用它包装底层错误)。
- **`NoBucketReadPermissionError`**: bucket 无读取权限(部分 adapter 会用它包装底层错误)。 - **`NoBucketReadPermissionError`**: bucket 无读取权限(部分 adapter 会用它包装底层错误)。
- **`EmptyObjectError`**: 下载时对象为空(例如底层 SDK 返回 `Body` 为空)。 - **`EmptyObjectError`**: 下载时对象为空(例如底层 SDK 返回 `Body` 为空)。
- **`InvalidStorageObjectKeyError`**: key/prefix 未通过 SDK 统一预检;`reason`、`field`、`actualBytes` 和 `maxBytes` 可用于结构化处理。
建议你在业务层做分层处理:可恢复错误(重试/提示权限)与不可恢复错误(配置错误/接口未实现)。 建议你在业务层做分层处理:可恢复错误(重试/提示权限)与不可恢复错误(配置错误/接口未实现)。
## 注意事项 ## 注意事项
- **key 使用统一规范**:所有 adapter 都在远端请求前要求 1 - 850 UTF-8 bytes,拒绝前导 `/`、反斜线、连续 `//`、控制字符和 `.`/`..` 路径段;空格及 `+ # & % ?`、中文、emoji 可正常使用。
- **按前缀删除是高危操作**:`prefix` 必须是非空字符串;强烈建议使用业务隔离前缀(例如 `team/{teamId}/`),避免误删整桶。 - **按前缀删除是高危操作**:`prefix` 必须是非空字符串;强烈建议使用业务隔离前缀(例如 `team/{teamId}/`),避免误删整桶。
- **metadata 厂商差异**:不同厂商对元数据 key 前缀/大小写/可用字符/大小限制不同,建议使用简单 ASCII key,并控制总体大小。 - **metadata 厂商差异**:不同厂商对元数据 key 前缀/大小写/可用字符/大小限制不同,建议使用简单 ASCII key,并控制总体大小。
- **流式下载/上传**:大文件建议使用 `Readable`,减少内存峰值。 - **流式下载/上传**:大文件建议使用 `Readable`,减少内存峰值。
...@@ -191,10 +193,29 @@ await fetch(putUrl, { ...@@ -191,10 +193,29 @@ await fetch(putUrl, {
## 开发与构建 ## 开发与构建
```bash ```bash
pnpm -C FastGPT/packages/storage dev pnpm --filter @fastgpt-sdk/storage dev
pnpm -C FastGPT/packages/storage build pnpm --filter @fastgpt-sdk/storage build
pnpm --filter @fastgpt-sdk/storage test:unit
pnpm --filter @fastgpt-sdk/storage typecheck:test
``` ```
发布前会执行 `prepublishOnly` 自动构建产物到 `dist/`。 真实对象存储的统一契约测试位于 `sdk/storage/test/integration`。复制
`sdk/storage/.env.test.example` 为 `sdk/storage/.env.test.local`,填写凭证并将对应
`STORAGE_TEST_<PROVIDER>_ENABLED` 设置为 `true`,并配置对应的
`STORAGE_TEST_<PROVIDER>_BUCKET` 后运行。测试桶名必须以 `fastgpt-sdk-` 开头:
```bash
pnpm --filter @fastgpt-sdk/storage test:integration
pnpm --filter @fastgpt-sdk/storage test:integration:common
pnpm --filter @fastgpt-sdk/storage test:integration:minio
```
集成测试分为两层:
- `test/integration/common`:18 个 `IStorage` 通用契约,每个启用的 provider 都运行完全相同的用例。
- `test/integration/minio`:11 个 MinIO 专项用例,覆盖中断运行后的桶重建、400/1000 条分页边界、URL 编码、公共策略、真实 HTTP socket 超时,以及等待响应头和读取响应体时的下载取消。
- `test/integration/transport`:2 个无需云凭证的 OSS/COS 真实 socket 取消用例。
每个 provider 使用配置中的固定专用测试桶。每次 suite 启动时,harness 会先清空并删除可能由上次失败运行遗留的同名桶,再重新创建;结束时也会清理。不要对同一组测试配置并发运行集成测试。未启用的 provider 会被跳过。
发布前会执行 `prepublishOnly` 自动构建产物到 `dist/`。
...@@ -48,6 +48,12 @@ ...@@ -48,6 +48,12 @@
"scripts": { "scripts": {
"build": "tsdown", "build": "tsdown",
"dev": "tsdown --watch", "dev": "tsdown --watch",
"test": "vitest run --config vitest.config.ts",
"test:unit": "vitest run --config vitest.config.ts test/unit",
"test:integration": "vitest run --config vitest.config.ts test/integration",
"test:integration:common": "vitest run --config vitest.config.ts test/integration/common",
"test:integration:minio": "vitest run --config vitest.config.ts test/integration/minio",
"typecheck:test": "tsc --noEmit -p tsconfig.test.json",
"prepublishOnly": "pnpm build" "prepublishOnly": "pnpm build"
}, },
"dependencies": { "dependencies": {
......
...@@ -45,6 +45,17 @@ import type { Readable } from 'node:stream'; ...@@ -45,6 +45,17 @@ import type { Readable } from 'node:stream';
import { camelCase, chunk, isNotNil, kebabCase, trim } from 'es-toolkit'; import { camelCase, chunk, isNotNil, kebabCase, trim } from 'es-toolkit';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS } from '../constants'; import { DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS } from '../constants';
import {
bindAbortSignalToReadable,
encodeObjectKeyPath,
throwIfStorageDownloadAborted
} from '../utils';
import {
assertStorageObjectKey,
assertStorageObjectKeys,
assertStorageObjectPrefix,
assertRequiredStorageObjectPrefix
} from '../assert';
export class AwsS3StorageAdapter implements IStorage { export class AwsS3StorageAdapter implements IStorage {
protected readonly client: S3Client; protected readonly client: S3Client;
...@@ -69,6 +80,7 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -69,6 +80,7 @@ export class AwsS3StorageAdapter implements IStorage {
async checkObjectExists(params: ExistsObjectParams): Promise<ExistsObjectResult> { async checkObjectExists(params: ExistsObjectParams): Promise<ExistsObjectResult> {
const { key } = params; const { key } = params;
assertStorageObjectKey(key);
let exists = false; let exists = false;
...@@ -97,6 +109,7 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -97,6 +109,7 @@ export class AwsS3StorageAdapter implements IStorage {
async getObjectMetadata(params: GetObjectMetadataParams): Promise<GetObjectMetadataResult> { async getObjectMetadata(params: GetObjectMetadataParams): Promise<GetObjectMetadataResult> {
const { key } = params; const { key } = params;
assertStorageObjectKey(key);
const result = await this.client.send( const result = await this.client.send(
new HeadObjectCommand({ new HeadObjectCommand({
...@@ -135,6 +148,7 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -135,6 +148,7 @@ export class AwsS3StorageAdapter implements IStorage {
async uploadObject(params: UploadObjectParams): Promise<UploadObjectResult> { async uploadObject(params: UploadObjectParams): Promise<UploadObjectResult> {
const { key, body, contentType, contentLength, contentDisposition, metadata } = params; const { key, body, contentType, contentLength, contentDisposition, metadata } = params;
assertStorageObjectKey(key);
const meta: StorageObjectMetadata = {}; const meta: StorageObjectMetadata = {};
if (metadata) { if (metadata) {
...@@ -167,6 +181,8 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -167,6 +181,8 @@ export class AwsS3StorageAdapter implements IStorage {
async downloadObject(params: DownloadObjectParams): Promise<DownloadObjectResult> { async downloadObject(params: DownloadObjectParams): Promise<DownloadObjectResult> {
const { key, abortSignal } = params; const { key, abortSignal } = params;
assertStorageObjectKey(key);
throwIfStorageDownloadAborted(abortSignal);
const result = await this.client.send( const result = await this.client.send(
new GetObjectCommand({ new GetObjectCommand({
...@@ -179,16 +195,19 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -179,16 +195,19 @@ export class AwsS3StorageAdapter implements IStorage {
if (!result.Body) { if (!result.Body) {
throw new EmptyObjectError('Object is undefined'); throw new EmptyObjectError('Object is undefined');
} }
const body = result.Body as Readable;
bindAbortSignalToReadable({ readable: body, abortSignal });
return { return {
key, key,
bucket: this.options.bucket, bucket: this.options.bucket,
body: result.Body as Readable body
}; };
} }
async deleteObject(params: DeleteObjectParams): Promise<DeleteObjectResult> { async deleteObject(params: DeleteObjectParams): Promise<DeleteObjectResult> {
const { key } = params; const { key } = params;
assertStorageObjectKey(key);
await this.client.send( await this.client.send(
new DeleteObjectCommand({ new DeleteObjectCommand({
...@@ -205,6 +224,7 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -205,6 +224,7 @@ export class AwsS3StorageAdapter implements IStorage {
async deleteObjectsByMultiKeys(params: DeleteObjectsParams): Promise<DeleteObjectsResult> { async deleteObjectsByMultiKeys(params: DeleteObjectsParams): Promise<DeleteObjectsResult> {
const { keys } = params; const { keys } = params;
assertStorageObjectKeys(keys);
if (keys.length === 0) { if (keys.length === 0) {
return { return {
...@@ -237,9 +257,7 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -237,9 +257,7 @@ export class AwsS3StorageAdapter implements IStorage {
async deleteObjectsByPrefix(params: DeleteObjectsByPrefixParams): Promise<DeleteObjectsResult> { async deleteObjectsByPrefix(params: DeleteObjectsByPrefixParams): Promise<DeleteObjectsResult> {
const { prefix } = params; const { prefix } = params;
if (!prefix) { assertRequiredStorageObjectPrefix(prefix);
throw new Error('Prefix is required');
}
const fails: StorageObjectKey[] = []; const fails: StorageObjectKey[] = [];
let isTruncated = false; let isTruncated = false;
...@@ -258,7 +276,7 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -258,7 +276,7 @@ export class AwsS3StorageAdapter implements IStorage {
if (!listResponse.Contents || listResponse.Contents.length === 0) { if (!listResponse.Contents || listResponse.Contents.length === 0) {
return { return {
bucket: this.options.bucket, bucket: this.options.bucket,
keys: [] keys: fails
}; };
} }
...@@ -287,6 +305,7 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -287,6 +305,7 @@ export class AwsS3StorageAdapter implements IStorage {
async generatePresignedPutUrl(params: PresignedPutUrlParams): Promise<PresignedPutUrlResult> { async generatePresignedPutUrl(params: PresignedPutUrlParams): Promise<PresignedPutUrlResult> {
const { key, expiredSeconds, metadata, contentType } = params; const { key, expiredSeconds, metadata, contentType } = params;
assertStorageObjectKey(key);
const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS; const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS;
...@@ -341,6 +360,7 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -341,6 +360,7 @@ export class AwsS3StorageAdapter implements IStorage {
async generatePresignedGetUrl(params: PresignedGetUrlParams): Promise<PresignedGetUrlResult> { async generatePresignedGetUrl(params: PresignedGetUrlParams): Promise<PresignedGetUrlResult> {
const { key, expiredSeconds, responseContentType } = params; const { key, expiredSeconds, responseContentType } = params;
assertStorageObjectKey(key);
const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS; const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS;
...@@ -365,13 +385,15 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -365,13 +385,15 @@ export class AwsS3StorageAdapter implements IStorage {
generatePublicGetUrl(params: GeneratePublicGetUrlParams): GeneratePublicGetUrlResult { generatePublicGetUrl(params: GeneratePublicGetUrlParams): GeneratePublicGetUrlResult {
const { key } = params; const { key } = params;
assertStorageObjectKey(key);
const encodedKey = encodeObjectKeyPath(key);
let url: string; let url: string;
if (this.options.forcePathStyle) { if (this.options.forcePathStyle) {
if (this.options.publicAccessExtraSubPath) { if (this.options.publicAccessExtraSubPath) {
url = `${this.options.endpoint}/${trim(this.options.publicAccessExtraSubPath, '/')}/${this.options.bucket}/${key}`; url = `${this.options.endpoint}/${trim(this.options.publicAccessExtraSubPath, '/')}/${this.options.bucket}/${encodedKey}`;
} else { } else {
url = `${this.options.endpoint}/${this.options.bucket}/${key}`; url = `${this.options.endpoint}/${this.options.bucket}/${encodedKey}`;
} }
} else { } else {
const endpoint = new URL(this.options.endpoint); const endpoint = new URL(this.options.endpoint);
...@@ -379,9 +401,9 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -379,9 +401,9 @@ export class AwsS3StorageAdapter implements IStorage {
const host = endpoint.host; const host = endpoint.host;
if (this.options.publicAccessExtraSubPath) { if (this.options.publicAccessExtraSubPath) {
url = `${protocol}//${this.options.bucket}.${host}/${trim(this.options.publicAccessExtraSubPath, '/')}/${key}`; url = `${protocol}//${this.options.bucket}.${host}/${trim(this.options.publicAccessExtraSubPath, '/')}/${encodedKey}`;
} else { } else {
url = `${protocol}//${this.options.bucket}.${host}/${key}`; url = `${protocol}//${this.options.bucket}.${host}/${encodedKey}`;
} }
} }
...@@ -394,6 +416,7 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -394,6 +416,7 @@ export class AwsS3StorageAdapter implements IStorage {
async listObjects(params: ListObjectsParams): Promise<ListObjectsResult> { async listObjects(params: ListObjectsParams): Promise<ListObjectsResult> {
const { prefix } = params; const { prefix } = params;
assertStorageObjectPrefix(prefix);
let keys: StorageObjectKey[] = []; let keys: StorageObjectKey[] = [];
let isTruncated = false; let isTruncated = false;
...@@ -430,6 +453,8 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -430,6 +453,8 @@ export class AwsS3StorageAdapter implements IStorage {
async copyObjectInSelfBucket(params: CopyObjectParams): Promise<CopyObjectResult> { async copyObjectInSelfBucket(params: CopyObjectParams): Promise<CopyObjectResult> {
const { sourceKey, targetKey } = params; const { sourceKey, targetKey } = params;
assertStorageObjectKey(sourceKey, 'sourceKey');
assertStorageObjectKey(targetKey, 'targetKey');
const encodedSourceKey = sourceKey const encodedSourceKey = sourceKey
.split('/') .split('/')
......
...@@ -31,6 +31,17 @@ import type { ...@@ -31,6 +31,17 @@ import type {
import { PassThrough } from 'node:stream'; import { PassThrough } from 'node:stream';
import { camelCase, isError, isNotNil, kebabCase } from 'es-toolkit'; import { camelCase, isError, isNotNil, kebabCase } from 'es-toolkit';
import { DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS } from '../constants'; import { DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS } from '../constants';
import {
bindAbortSignalToReadable,
encodeObjectKeyPath,
throwIfStorageDownloadAborted
} from '../utils';
import {
assertStorageObjectKey,
assertStorageObjectKeys,
assertStorageObjectPrefix,
assertRequiredStorageObjectPrefix
} from '../assert';
export class CosStorageAdapter implements IStorage { export class CosStorageAdapter implements IStorage {
protected readonly client: COS; protected readonly client: COS;
...@@ -62,6 +73,7 @@ export class CosStorageAdapter implements IStorage { ...@@ -62,6 +73,7 @@ export class CosStorageAdapter implements IStorage {
async checkObjectExists(params: ExistsObjectParams): Promise<ExistsObjectResult> { async checkObjectExists(params: ExistsObjectParams): Promise<ExistsObjectResult> {
const { key } = params; const { key } = params;
assertStorageObjectKey(key);
let exists = false; let exists = false;
await new Promise<void>((resolve, reject) => { await new Promise<void>((resolve, reject) => {
...@@ -96,6 +108,7 @@ export class CosStorageAdapter implements IStorage { ...@@ -96,6 +108,7 @@ export class CosStorageAdapter implements IStorage {
async getObjectMetadata(params: GetObjectMetadataParams): Promise<GetObjectMetadataResult> { async getObjectMetadata(params: GetObjectMetadataParams): Promise<GetObjectMetadataResult> {
const { key } = params; const { key } = params;
assertStorageObjectKey(key);
const result = await new Promise<COS.HeadObjectResult>((resolve, reject) => { const result = await new Promise<COS.HeadObjectResult>((resolve, reject) => {
this.client.headObject( this.client.headObject(
...@@ -161,6 +174,7 @@ export class CosStorageAdapter implements IStorage { ...@@ -161,6 +174,7 @@ export class CosStorageAdapter implements IStorage {
async uploadObject(params: UploadObjectParams): Promise<UploadObjectResult> { async uploadObject(params: UploadObjectParams): Promise<UploadObjectResult> {
const { key, body, contentType, contentLength, contentDisposition, metadata } = params; const { key, body, contentType, contentLength, contentDisposition, metadata } = params;
assertStorageObjectKey(key);
const headers: Record<string, string> = {}; const headers: Record<string, string> = {};
if (contentDisposition) headers['Content-Disposition'] = contentDisposition; if (contentDisposition) headers['Content-Disposition'] = contentDisposition;
...@@ -199,17 +213,11 @@ export class CosStorageAdapter implements IStorage { ...@@ -199,17 +213,11 @@ export class CosStorageAdapter implements IStorage {
} }
async downloadObject(params: DownloadObjectParams): Promise<DownloadObjectResult> { async downloadObject(params: DownloadObjectParams): Promise<DownloadObjectResult> {
params.abortSignal?.throwIfAborted(); assertStorageObjectKey(params.key);
throwIfStorageDownloadAborted(params.abortSignal);
const passThrough = new PassThrough(); const passThrough = new PassThrough();
const abortDownload = () => { bindAbortSignalToReadable({ readable: passThrough, abortSignal: params.abortSignal });
passThrough.destroy();
};
params.abortSignal?.addEventListener('abort', abortDownload, { once: true });
passThrough.once('close', () => {
params.abortSignal?.removeEventListener('abort', abortDownload);
});
this.client.getObject( this.client.getObject(
{ {
...@@ -234,6 +242,7 @@ export class CosStorageAdapter implements IStorage { ...@@ -234,6 +242,7 @@ export class CosStorageAdapter implements IStorage {
async deleteObject(params: DeleteObjectParams): Promise<DeleteObjectResult> { async deleteObject(params: DeleteObjectParams): Promise<DeleteObjectResult> {
const { key } = params; const { key } = params;
assertStorageObjectKey(key);
await new Promise<COS.DeleteObjectResult>((resolve, reject) => { await new Promise<COS.DeleteObjectResult>((resolve, reject) => {
this.client.deleteObject( this.client.deleteObject(
...@@ -259,6 +268,14 @@ export class CosStorageAdapter implements IStorage { ...@@ -259,6 +268,14 @@ export class CosStorageAdapter implements IStorage {
async deleteObjectsByMultiKeys(params: DeleteObjectsParams): Promise<DeleteObjectsResult> { async deleteObjectsByMultiKeys(params: DeleteObjectsParams): Promise<DeleteObjectsResult> {
const { keys } = params; const { keys } = params;
assertStorageObjectKeys(keys);
if (keys.length === 0) {
return {
bucket: this.options.bucket,
keys: []
};
}
const result = await new Promise<COS.DeleteMultipleObjectResult>((resolve, reject) => { const result = await new Promise<COS.DeleteMultipleObjectResult>((resolve, reject) => {
this.client.deleteMultipleObject( this.client.deleteMultipleObject(
...@@ -284,9 +301,7 @@ export class CosStorageAdapter implements IStorage { ...@@ -284,9 +301,7 @@ export class CosStorageAdapter implements IStorage {
async deleteObjectsByPrefix(params: DeleteObjectsByPrefixParams): Promise<DeleteObjectsResult> { async deleteObjectsByPrefix(params: DeleteObjectsByPrefixParams): Promise<DeleteObjectsResult> {
const { prefix } = params; const { prefix } = params;
if (!prefix) { assertRequiredStorageObjectPrefix(prefix);
throw new Error('Prefix is required');
}
const fails: StorageObjectKey[] = []; const fails: StorageObjectKey[] = [];
let marker: string | undefined = undefined; let marker: string | undefined = undefined;
...@@ -354,6 +369,7 @@ export class CosStorageAdapter implements IStorage { ...@@ -354,6 +369,7 @@ export class CosStorageAdapter implements IStorage {
async generatePresignedPutUrl(params: PresignedPutUrlParams): Promise<PresignedPutUrlResult> { async generatePresignedPutUrl(params: PresignedPutUrlParams): Promise<PresignedPutUrlResult> {
const { key, expiredSeconds, metadata, contentType } = params; const { key, expiredSeconds, metadata, contentType } = params;
assertStorageObjectKey(key);
const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS; const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS;
...@@ -398,6 +414,7 @@ export class CosStorageAdapter implements IStorage { ...@@ -398,6 +414,7 @@ export class CosStorageAdapter implements IStorage {
async generatePresignedGetUrl(params: PresignedGetUrlParams): Promise<PresignedGetUrlResult> { async generatePresignedGetUrl(params: PresignedGetUrlParams): Promise<PresignedGetUrlResult> {
const { key, expiredSeconds, responseContentType } = params; const { key, expiredSeconds, responseContentType } = params;
assertStorageObjectKey(key);
const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS; const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS;
const url = await new Promise<string>((resolve, reject) => { const url = await new Promise<string>((resolve, reject) => {
...@@ -431,12 +448,14 @@ export class CosStorageAdapter implements IStorage { ...@@ -431,12 +448,14 @@ export class CosStorageAdapter implements IStorage {
generatePublicGetUrl(params: GeneratePublicGetUrlParams): GeneratePublicGetUrlResult { generatePublicGetUrl(params: GeneratePublicGetUrlParams): GeneratePublicGetUrlResult {
const { key } = params; const { key } = params;
assertStorageObjectKey(key);
const encodedKey = encodeObjectKeyPath(key);
let url: string; let url: string;
if (this.options.domain) { if (this.options.domain) {
url = `${this.options.protocol}//${this.options.domain}/${key}`; url = `${this.options.protocol}//${this.options.domain}/${encodedKey}`;
} else { } else {
url = `${this.options.protocol}//${this.options.bucket}.cos.${this.options.region}.myqcloud.com/${key}`; url = `${this.options.protocol}//${this.options.bucket}.cos.${this.options.region}.myqcloud.com/${encodedKey}`;
} }
return { return {
...@@ -448,6 +467,7 @@ export class CosStorageAdapter implements IStorage { ...@@ -448,6 +467,7 @@ export class CosStorageAdapter implements IStorage {
async listObjects(params: ListObjectsParams): Promise<ListObjectsResult> { async listObjects(params: ListObjectsParams): Promise<ListObjectsResult> {
const { prefix } = params; const { prefix } = params;
assertStorageObjectPrefix(prefix);
let keys: StorageObjectKey[] = []; let keys: StorageObjectKey[] = [];
let marker: string | undefined = undefined; let marker: string | undefined = undefined;
...@@ -490,6 +510,8 @@ export class CosStorageAdapter implements IStorage { ...@@ -490,6 +510,8 @@ export class CosStorageAdapter implements IStorage {
async copyObjectInSelfBucket(params: CopyObjectParams): Promise<CopyObjectResult> { async copyObjectInSelfBucket(params: CopyObjectParams): Promise<CopyObjectResult> {
const { sourceKey, targetKey } = params; const { sourceKey, targetKey } = params;
assertStorageObjectKey(sourceKey, 'sourceKey');
assertStorageObjectKey(targetKey, 'targetKey');
const encodedSourceKey = sourceKey const encodedSourceKey = sourceKey
.split('/') .split('/')
......
import * as Minio from 'minio'; import * as Minio from 'minio';
import * as http from 'node:http';
import * as https from 'node:https';
import type { IAwsS3CompatibleStorageOptions, IStorage } from '../interface'; import type { IAwsS3CompatibleStorageOptions, IStorage } from '../interface';
import type { import type {
DeleteObjectParams, DeleteObjectParams,
...@@ -12,10 +14,77 @@ import { AwsS3StorageAdapter } from './aws-s3.adapter'; ...@@ -12,10 +14,77 @@ import { AwsS3StorageAdapter } from './aws-s3.adapter';
import { import {
CreateBucketCommand, CreateBucketCommand,
DeleteBucketLifecycleCommand, DeleteBucketLifecycleCommand,
ListObjectsV2Command,
NotFound, NotFound,
PutBucketPolicyCommand PutBucketPolicyCommand
} from '@aws-sdk/client-s3'; } from '@aws-sdk/client-s3';
import { chunk } from 'es-toolkit'; import { chunk } from 'es-toolkit';
import {
assertStorageObjectKey,
assertStorageObjectKeys,
assertRequiredStorageObjectPrefix
} from '../assert';
export { NotFound as MinioS3NotFound };
const minioRequestTimeoutMs = 60_000;
/**
* 为 MinIO 删除请求注入总请求超时。Promise 超时无法取消底层 HTTP 请求,
* transport 层主动 destroy 才能避免队列重试时积累悬挂连接。
*/
export const createMinioTimeoutTransport = ({
transport,
timeoutMs
}: {
transport: NonNullable<Minio.ClientOptions['transport']>;
timeoutMs: number;
}): NonNullable<Minio.ClientOptions['transport']> => ({
request: ((...args: unknown[]) => {
const request = Reflect.apply(transport.request, transport, args) as http.ClientRequest;
const timeout = setTimeout(() => {
request.destroy(new Error(`MinIO request timeout after ${timeoutMs}ms`));
}, timeoutMs);
timeout.unref();
request.once('close', () => clearTimeout(timeout));
return request;
}) as NonNullable<Minio.ClientOptions['transport']>['request']
});
/**
* 提取 MinIO 批量删除响应中的失败 key。
*
* MinIO 8.x 运行时直接返回错误对象数组,但类型声明仍保留了 Error 嵌套,
* 因此同时兼容两种结构。响应结构异常或无法定位失败对象时,将当前批次
* 全部标记为失败,避免上层把不完整响应误判为删除成功。
*/
const getFailedKeysFromMinioRemove = ({
result,
requestedKeys
}: {
result: unknown;
requestedKeys: string[];
}): string[] => {
if (!Array.isArray(result)) return requestedKeys;
const requestedKeySet = new Set(requestedKeys);
const failedKeys: string[] = [];
for (const item of result) {
const key = (() => {
if (!item || typeof item !== 'object') return;
if ('Key' in item && typeof item.Key === 'string') return item.Key;
if (!('Error' in item) || !item.Error || typeof item.Error !== 'object') return;
if ('Key' in item.Error && typeof item.Error.Key === 'string') return item.Error.Key;
})();
if (!key || !requestedKeySet.has(key)) return requestedKeys;
failedKeys.push(key);
}
return failedKeys;
};
/** /**
* @description MinIO 存储适配器(基于 minio SDK 和 AWS S3 SDK) * @description MinIO 存储适配器(基于 minio SDK 和 AWS S3 SDK)
...@@ -53,6 +122,10 @@ export class MinioStorageAdapter extends AwsS3StorageAdapter implements IStorage ...@@ -53,6 +122,10 @@ export class MinioStorageAdapter extends AwsS3StorageAdapter implements IStorage
const endpointUrl = new URL(options.endpoint); const endpointUrl = new URL(options.endpoint);
const useSSL = endpointUrl.protocol === 'https:'; const useSSL = endpointUrl.protocol === 'https:';
const port = endpointUrl.port ? parseInt(endpointUrl.port, 10) : useSSL ? 443 : 80; const port = endpointUrl.port ? parseInt(endpointUrl.port, 10) : useSSL ? 443 : 80;
const transport = createMinioTimeoutTransport({
transport: useSSL ? https : http,
timeoutMs: minioRequestTimeoutMs
});
this.minioClient = new Minio.Client({ this.minioClient = new Minio.Client({
endPoint: endpointUrl.hostname, endPoint: endpointUrl.hostname,
...@@ -61,12 +134,14 @@ export class MinioStorageAdapter extends AwsS3StorageAdapter implements IStorage ...@@ -61,12 +134,14 @@ export class MinioStorageAdapter extends AwsS3StorageAdapter implements IStorage
accessKey: options.credentials.accessKeyId, accessKey: options.credentials.accessKeyId,
secretKey: options.credentials.secretAccessKey, secretKey: options.credentials.secretAccessKey,
region: options.region, region: options.region,
pathStyle: options.forcePathStyle pathStyle: options.forcePathStyle,
transport
}); });
} }
async deleteObject(params: DeleteObjectParams): Promise<DeleteObjectResult> { async deleteObject(params: DeleteObjectParams): Promise<DeleteObjectResult> {
const { key } = params; const { key } = params;
assertStorageObjectKey(key);
await this.minioClient.removeObject(this.options.bucket, key); await this.minioClient.removeObject(this.options.bucket, key);
...@@ -78,6 +153,7 @@ export class MinioStorageAdapter extends AwsS3StorageAdapter implements IStorage ...@@ -78,6 +153,7 @@ export class MinioStorageAdapter extends AwsS3StorageAdapter implements IStorage
async deleteObjectsByMultiKeys(params: DeleteObjectsParams): Promise<DeleteObjectsResult> { async deleteObjectsByMultiKeys(params: DeleteObjectsParams): Promise<DeleteObjectsResult> {
const { keys } = params; const { keys } = params;
assertStorageObjectKeys(keys);
if (keys.length === 0) { if (keys.length === 0) {
return { return {
...@@ -89,109 +165,115 @@ export class MinioStorageAdapter extends AwsS3StorageAdapter implements IStorage ...@@ -89,109 +165,115 @@ export class MinioStorageAdapter extends AwsS3StorageAdapter implements IStorage
// 每次 removeObjects 最多删除 1000 个对象 // 每次 removeObjects 最多删除 1000 个对象
// 因此需要将对象列表分块 // 因此需要将对象列表分块
const chunks = chunk(keys, 1000); const chunks = chunk(keys, 1000);
const failedKeys: string[] = [];
for (const chunk of chunks) { for (const chunk of chunks) {
await this.minioClient.removeObjects(this.options.bucket, chunk); const result = await this.minioClient.removeObjects(this.options.bucket, chunk);
failedKeys.push(...getFailedKeysFromMinioRemove({ result, requestedKeys: chunk }));
} }
// Minio Client 的 removeObjects 不返回失败列表,假设全部成功
return { return {
bucket: this.options.bucket, bucket: this.options.bucket,
keys: [] keys: failedKeys
}; };
} }
/** /**
* @note 这里的实现可以使用 `@aws-sdk/client-s3` 来列出对象,然后使用 `minio` 来删除对象,但是这里直接使用 `minio` 的 `listObjectsV2` 方法来列出对象了。 * 按前缀分页列举并删除 MinIO 对象。
*
* MinIO SDK 的 listObjectsV2 固定每页返回 1000 条,且其 XML 解析器默认最多展开
* 1000 个实体。部分 S3 兼容服务会将每个 ETag 的引号编码为两个 XML 实体,
* 导致大批量列举时误触安全上限。因此使用 AWS SDK 控制分页大小和 URL 编码,
* 再使用 MinIO SDK 删除,以保留现有 MinIO 校验和兼容性逻辑。
*
* 列举和删除按页串行执行,避免对象数量较大时积累无上限的并发删除任务。
* 每个列举和删除请求最多等待 60 秒,避免异常连接长期占用删除队列 worker。
*/ */
async deleteObjectsByPrefix(params: DeleteObjectsByPrefixParams): Promise<DeleteObjectsResult> { async deleteObjectsByPrefix(params: DeleteObjectsByPrefixParams): Promise<DeleteObjectsResult> {
const { prefix } = params; const { prefix } = params;
const batchSize = 1000; const listBatchSize = 400;
const timeoutMs = 60000;
if (!prefix?.trim()) { assertRequiredStorageObjectPrefix(prefix);
throw new Error('Prefix is required');
}
const bucket = this.options.bucket; const bucket = this.options.bucket;
const failedKeys: string[] = []; const failedKeys: string[] = [];
let deleteTasks: Promise<void>[] = []; let continuationToken: string | undefined;
const stream = this.minioClient.listObjectsV2(bucket, prefix, true); do {
const abortController = new AbortController();
return await new Promise<DeleteObjectsResult>((resolve, reject) => { const listResponse = await this._withListRequestTimeout({
let settled = false; prefix,
let timer: NodeJS.Timeout; promise: this.client.send(
new ListObjectsV2Command({
const finish = (error?: any) => { Bucket: bucket,
if (settled) return; Prefix: prefix,
settled = true; ContinuationToken: continuationToken,
EncodingType: 'url',
if (timer) clearTimeout(timer); MaxKeys: listBatchSize
}),
stream.removeAllListeners(); { abortSignal: abortController.signal }
try { ),
stream.destroy(); onTimeout: () => abortController.abort()
} catch {}
if (error) {
reject(error);
} else {
resolve({ bucket, keys: failedKeys });
}
};
timer = setTimeout(() => {
finish(new Error(`Delete by prefix timeout: ${prefix}`));
}, timeoutMs);
const flushBatch = async (keys: string[]) => {
if (keys.length === 0) return;
try {
await this.minioClient.removeObjects(bucket, keys);
} catch {
failedKeys.push(...keys);
}
};
let batch: string[] = [];
stream.on('data', (obj) => {
if (!obj.name) return;
batch.push(obj.name);
if (batch.length >= batchSize) {
const toDelete = batch;
batch = [];
deleteTasks.push(flushBatch(toDelete));
}
}); });
const keys = (listResponse.Contents ?? []).flatMap(({ Key }) => {
if (!Key) return [];
stream.on('error', (err) => { // MinIO 的 encoding-type=url 响应会用 + 表示空格,需先转换再解码。
finish(err); return [decodeURIComponent(Key.replace(/\+/g, ' '))];
}); });
stream.on('end', async () => { if (keys.length > 0) {
if (timer) clearTimeout(timer); await this.minioClient
.removeObjects(bucket, keys)
.then((result) => {
failedKeys.push(...getFailedKeysFromMinioRemove({ result, requestedKeys: keys }));
})
.catch(() => {
failedKeys.push(...keys);
});
}
if (!listResponse.IsTruncated) break;
if (!listResponse.NextContinuationToken) {
throw new Error('Invalid MinIO list response: missing continuation token');
}
continuationToken = listResponse.NextContinuationToken;
} while (true);
try { return {
if (batch.length > 0) { bucket,
deleteTasks.push(flushBatch(batch)); keys: failedKeys
} };
}
await Promise.all(deleteTasks); private async _withListRequestTimeout<T>({
finish(); prefix,
} catch (e) { promise,
finish(e); onTimeout
} }: {
}); prefix: string;
promise: Promise<T>;
onTimeout: () => void;
}): Promise<T> {
let timer: ReturnType<typeof setTimeout> | undefined;
stream.on('pause', () => { try {
stream.resume(); return await Promise.race([
}); promise,
}); new Promise<never>((_, reject) => {
timer = setTimeout(() => {
const timeoutError = new Error(
`Delete by prefix list timeout after ${minioRequestTimeoutMs}ms: ${prefix}`
);
// 先固定 Promise.race 的结果,再 abort 底层请求,避免 AbortError 抢先返回。
reject(timeoutError);
onTimeout();
}, minioRequestTimeoutMs);
})
]);
} finally {
if (timer) clearTimeout(timer);
}
} }
async ensureBucket(): Promise<EnsureBucketResult> { async ensureBucket(): Promise<EnsureBucketResult> {
......
...@@ -31,6 +31,17 @@ import type { ...@@ -31,6 +31,17 @@ import type {
import type { Readable } from 'node:stream'; import type { Readable } from 'node:stream';
import { camelCase, difference, kebabCase } from 'es-toolkit'; import { camelCase, difference, kebabCase } from 'es-toolkit';
import { DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS } from '../constants'; import { DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS } from '../constants';
import {
bindAbortSignalToReadable,
encodeObjectKeyPath,
throwIfStorageDownloadAborted
} from '../utils';
import {
assertStorageObjectKey,
assertStorageObjectKeys,
assertStorageObjectPrefix,
assertRequiredStorageObjectPrefix
} from '../assert';
export class OssStorageAdapter implements IStorage { export class OssStorageAdapter implements IStorage {
protected readonly client: OSS; protected readonly client: OSS;
...@@ -61,6 +72,7 @@ export class OssStorageAdapter implements IStorage { ...@@ -61,6 +72,7 @@ export class OssStorageAdapter implements IStorage {
async checkObjectExists(params: ExistsObjectParams): Promise<ExistsObjectResult> { async checkObjectExists(params: ExistsObjectParams): Promise<ExistsObjectResult> {
const { key } = params; const { key } = params;
assertStorageObjectKey(key);
let exists = false; let exists = false;
try { try {
...@@ -83,6 +95,7 @@ export class OssStorageAdapter implements IStorage { ...@@ -83,6 +95,7 @@ export class OssStorageAdapter implements IStorage {
async getObjectMetadata(params: GetObjectMetadataParams): Promise<GetObjectMetadataResult> { async getObjectMetadata(params: GetObjectMetadataParams): Promise<GetObjectMetadataResult> {
const { key } = params; const { key } = params;
assertStorageObjectKey(key);
const result = await this.client.head(key); const result = await this.client.head(key);
...@@ -122,6 +135,7 @@ export class OssStorageAdapter implements IStorage { ...@@ -122,6 +135,7 @@ export class OssStorageAdapter implements IStorage {
async uploadObject(params: UploadObjectParams): Promise<UploadObjectResult> { async uploadObject(params: UploadObjectParams): Promise<UploadObjectResult> {
const { key, body, contentType, contentLength, contentDisposition, metadata } = params; const { key, body, contentType, contentLength, contentDisposition, metadata } = params;
assertStorageObjectKey(key);
const headers: Record<string, any> = { const headers: Record<string, any> = {
'x-oss-storage-class': 'Standard', 'x-oss-storage-class': 'Standard',
...@@ -153,24 +167,12 @@ export class OssStorageAdapter implements IStorage { ...@@ -153,24 +167,12 @@ export class OssStorageAdapter implements IStorage {
async downloadObject(params: DownloadObjectParams): Promise<DownloadObjectResult> { async downloadObject(params: DownloadObjectParams): Promise<DownloadObjectResult> {
const { key, abortSignal } = params; const { key, abortSignal } = params;
assertStorageObjectKey(key);
abortSignal?.throwIfAborted(); throwIfStorageDownloadAborted(abortSignal);
const result = await this.client.getStream(key); const result = await this.client.getStream(key);
const stream = result.stream as Readable; const stream = result.stream as Readable;
const abortDownload = () => { bindAbortSignalToReadable({ readable: stream, abortSignal });
stream.destroy();
};
if (abortSignal?.aborted) {
abortDownload();
abortSignal.throwIfAborted();
}
abortSignal?.addEventListener('abort', abortDownload, { once: true });
stream.once('close', () => {
abortSignal?.removeEventListener('abort', abortDownload);
});
return { return {
key, key,
...@@ -181,6 +183,7 @@ export class OssStorageAdapter implements IStorage { ...@@ -181,6 +183,7 @@ export class OssStorageAdapter implements IStorage {
async deleteObject(params: DeleteObjectParams): Promise<DeleteObjectResult> { async deleteObject(params: DeleteObjectParams): Promise<DeleteObjectResult> {
const { key } = params; const { key } = params;
assertStorageObjectKey(key);
await this.client.delete(key); await this.client.delete(key);
...@@ -192,20 +195,46 @@ export class OssStorageAdapter implements IStorage { ...@@ -192,20 +195,46 @@ export class OssStorageAdapter implements IStorage {
async deleteObjectsByMultiKeys(params: DeleteObjectsParams): Promise<DeleteObjectsResult> { async deleteObjectsByMultiKeys(params: DeleteObjectsParams): Promise<DeleteObjectsResult> {
const { keys } = params; const { keys } = params;
assertStorageObjectKeys(keys);
const result = await this.client.deleteMulti(keys, { quiet: true }); if (keys.length === 0) {
return {
bucket: this.options.bucket,
keys: []
};
}
// verbose 模式会返回成功删除的 key;quiet 全成功时响应为空,无法与失败区分。
const result = await this.client.deleteMulti(keys, { quiet: false });
const deletedKeys = (() => {
const deletedItems: unknown = result.deleted;
if (!Array.isArray(deletedItems)) return [];
const normalizedKeys: string[] = [];
for (const item of deletedItems) {
if (typeof item === 'string') {
normalizedKeys.push(item);
continue;
}
// ali-oss 的类型声明是 string[],但标准 OSS XML 在运行时解析为 { Key }[]。
if (item && typeof item === 'object' && 'Key' in item && typeof item.Key === 'string') {
normalizedKeys.push(item.Key);
continue;
}
return [];
}
return normalizedKeys;
})();
return { return {
bucket: this.options.bucket, bucket: this.options.bucket,
keys: difference(keys, result.deleted ?? []) keys: difference(keys, deletedKeys)
}; };
} }
async deleteObjectsByPrefix(params: DeleteObjectsByPrefixParams): Promise<DeleteObjectsResult> { async deleteObjectsByPrefix(params: DeleteObjectsByPrefixParams): Promise<DeleteObjectsResult> {
const { prefix } = params; const { prefix } = params;
if (!prefix) { assertRequiredStorageObjectPrefix(prefix);
throw new Error('Prefix is required');
}
const fails: StorageObjectKey[] = []; const fails: StorageObjectKey[] = [];
let marker: string | undefined = undefined; let marker: string | undefined = undefined;
...@@ -226,7 +255,7 @@ export class OssStorageAdapter implements IStorage { ...@@ -226,7 +255,7 @@ export class OssStorageAdapter implements IStorage {
if (!listResponse.objects || listResponse.objects.length === 0) { if (!listResponse.objects || listResponse.objects.length === 0) {
return { return {
bucket: this.options.bucket, bucket: this.options.bucket,
keys: [] keys: fails
}; };
} }
...@@ -247,6 +276,7 @@ export class OssStorageAdapter implements IStorage { ...@@ -247,6 +276,7 @@ export class OssStorageAdapter implements IStorage {
async generatePresignedPutUrl(params: PresignedPutUrlParams): Promise<PresignedPutUrlResult> { async generatePresignedPutUrl(params: PresignedPutUrlParams): Promise<PresignedPutUrlResult> {
const { key, expiredSeconds, metadata, contentType } = params; const { key, expiredSeconds, metadata, contentType } = params;
assertStorageObjectKey(key);
const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS; const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS;
...@@ -285,6 +315,7 @@ export class OssStorageAdapter implements IStorage { ...@@ -285,6 +315,7 @@ export class OssStorageAdapter implements IStorage {
async generatePresignedGetUrl(params: PresignedGetUrlParams): Promise<PresignedGetUrlResult> { async generatePresignedGetUrl(params: PresignedGetUrlParams): Promise<PresignedGetUrlResult> {
const { key, expiredSeconds, responseContentType } = params; const { key, expiredSeconds, responseContentType } = params;
assertStorageObjectKey(key);
const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS; const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS;
const url = this.client.signatureUrl(key, { const url = this.client.signatureUrl(key, {
...@@ -308,6 +339,8 @@ export class OssStorageAdapter implements IStorage { ...@@ -308,6 +339,8 @@ export class OssStorageAdapter implements IStorage {
generatePublicGetUrl(params: GeneratePublicGetUrlParams): GeneratePublicGetUrlResult { generatePublicGetUrl(params: GeneratePublicGetUrlParams): GeneratePublicGetUrlResult {
const { key } = params; const { key } = params;
assertStorageObjectKey(key);
const encodedKey = encodeObjectKeyPath(key);
let protocol = 'https:'; let protocol = 'https:';
if (!this.options.secure) { if (!this.options.secure) {
...@@ -316,9 +349,9 @@ export class OssStorageAdapter implements IStorage { ...@@ -316,9 +349,9 @@ export class OssStorageAdapter implements IStorage {
let url: string; let url: string;
if (this.options.cname) { if (this.options.cname) {
url = `${protocol}//${this.options.endpoint}/${key}`; url = `${protocol}//${this.options.endpoint}/${encodedKey}`;
} else { } else {
url = `${protocol}//${this.options.bucket}.${this.options.region}.aliyuncs.com/${key}`; url = `${protocol}//${this.options.bucket}.${this.options.region}.aliyuncs.com/${encodedKey}`;
} }
return { return {
...@@ -330,6 +363,7 @@ export class OssStorageAdapter implements IStorage { ...@@ -330,6 +363,7 @@ export class OssStorageAdapter implements IStorage {
async listObjects(params: ListObjectsParams): Promise<ListObjectsResult> { async listObjects(params: ListObjectsParams): Promise<ListObjectsResult> {
const { prefix } = params; const { prefix } = params;
assertStorageObjectPrefix(prefix);
let keys: StorageObjectKey[] = []; let keys: StorageObjectKey[] = [];
let marker: string | undefined = undefined; let marker: string | undefined = undefined;
...@@ -367,6 +401,8 @@ export class OssStorageAdapter implements IStorage { ...@@ -367,6 +401,8 @@ export class OssStorageAdapter implements IStorage {
async copyObjectInSelfBucket(params: CopyObjectParams): Promise<CopyObjectResult> { async copyObjectInSelfBucket(params: CopyObjectParams): Promise<CopyObjectResult> {
const { sourceKey, targetKey } = params; const { sourceKey, targetKey } = params;
assertStorageObjectKey(sourceKey, 'sourceKey');
assertStorageObjectKey(targetKey, 'targetKey');
await this.client.copy(targetKey, sourceKey); await this.client.copy(targetKey, sourceKey);
......
import { InvalidStorageObjectKeyError, type InvalidStorageObjectKeyReason } from './errors';
/** 四个 adapter 都可移植的对象 key 最大 UTF-8 字节数。 */
export const MAX_STORAGE_OBJECT_KEY_UTF8_BYTES = 850;
function throwInvalidStorageObjectKey({
field,
reason,
actualBytes
}: {
field: string;
reason: InvalidStorageObjectKeyReason;
actualBytes?: number;
}): never {
throw new InvalidStorageObjectKeyError({
field,
reason,
actualBytes,
maxBytes: actualBytes === undefined ? undefined : MAX_STORAGE_OBJECT_KEY_UTF8_BYTES
});
}
/**
* 检查字符串是否不存在未配对的 UTF-16 surrogate。
* Buffer 会把非法 surrogate 静默替换成 U+FFFD,因此必须在计算 UTF-8 长度前显式检查。
*/
function isWellFormedUnicode(value: string): boolean {
for (let index = 0; index < value.length; index += 1) {
const codeUnit = value.charCodeAt(index);
if (codeUnit >= 0xd800 && codeUnit <= 0xdbff) {
const nextCodeUnit = value.charCodeAt(index + 1);
if (nextCodeUnit < 0xdc00 || nextCodeUnit > 0xdfff) return false;
index += 1;
continue;
}
if (codeUnit >= 0xdc00 && codeUnit <= 0xdfff) return false;
}
return true;
}
/**
* 按 SDK 统一规范预检对象 key;失败时不会把原始 key 写入错误消息。
* 该规范取 AWS S3、MinIO、OSS、COS 可稳定处理范围的交集。
*/
export function assertStorageObjectKey(value: unknown, field = 'key'): asserts value is string {
if (typeof value !== 'string') {
throwInvalidStorageObjectKey({ field, reason: 'invalid_type' });
}
if (value.length === 0) {
throwInvalidStorageObjectKey({ field, reason: 'empty' });
}
if (!isWellFormedUnicode(value)) {
throwInvalidStorageObjectKey({ field, reason: 'invalid_unicode' });
}
const actualBytes = Buffer.byteLength(value, 'utf8');
if (actualBytes > MAX_STORAGE_OBJECT_KEY_UTF8_BYTES) {
throwInvalidStorageObjectKey({ field, reason: 'too_long', actualBytes });
}
if (value.startsWith('/')) {
throwInvalidStorageObjectKey({ field, reason: 'leading_slash' });
}
if (value.includes('\\')) {
throwInvalidStorageObjectKey({ field, reason: 'backslash' });
}
if (value.includes('//')) {
throwInvalidStorageObjectKey({ field, reason: 'empty_path_segment' });
}
if (/[\u0000-\u001f\u007f]/u.test(value)) {
throwInvalidStorageObjectKey({ field, reason: 'control_character' });
}
if (
value.split('/').some((segment) => {
const trimmedSegment = segment.trim();
return trimmedSegment === '.' || trimmedSegment === '..';
})
) {
throwInvalidStorageObjectKey({ field, reason: 'dot_path_segment' });
}
}
/** 批量方法必须完整预检数组后,调用方才能开始分块或产生远端副作用。 */
export function assertStorageObjectKeys(keys: unknown): asserts keys is string[] {
if (!Array.isArray(keys)) {
throwInvalidStorageObjectKey({ field: 'keys', reason: 'invalid_type' });
}
for (let index = 0; index < keys.length; index += 1) {
assertStorageObjectKey(keys[index], `keys[${index}]`);
}
}
/** listObjects 允许省略或传空 prefix;非空 prefix 与对象 key 使用同一规范。 */
export function assertStorageObjectPrefix(prefix: unknown): asserts prefix is string | undefined {
if (prefix === undefined || prefix === '') return;
assertStorageObjectKey(prefix, 'prefix');
}
/** 删除前缀必须非空;其余字符和长度限制与对象 key 完全一致。 */
export function assertRequiredStorageObjectPrefix(prefix: unknown): asserts prefix is string {
if (typeof prefix === 'string' && prefix.trim().length === 0) {
throw new Error('Prefix is required');
}
assertStorageObjectKey(prefix, 'prefix');
}
...@@ -18,3 +18,59 @@ export class EmptyObjectError extends Error { ...@@ -18,3 +18,59 @@ export class EmptyObjectError extends Error {
this.name = 'EmptyObjectError'; this.name = 'EmptyObjectError';
} }
} }
export type InvalidStorageObjectKeyReason =
| 'invalid_type'
| 'empty'
| 'invalid_unicode'
| 'too_long'
| 'leading_slash'
| 'backslash'
| 'empty_path_segment'
| 'dot_path_segment'
| 'control_character';
const invalidStorageObjectKeyReasonMessages: Record<InvalidStorageObjectKeyReason, string> = {
invalid_type: 'must be a string',
empty: 'must not be empty',
invalid_unicode: 'must contain well-formed Unicode',
too_long: 'exceeds the UTF-8 byte limit',
leading_slash: 'must not start with a slash',
backslash: 'must not contain a backslash',
empty_path_segment: 'must not contain consecutive slashes',
dot_path_segment: 'must not contain dot path segments',
control_character: 'must not contain ASCII control characters'
};
/** SDK 在远端请求前发现对象 key 或 prefix 不符合统一可移植规范。 */
export class InvalidStorageObjectKeyError extends Error {
readonly field: string;
readonly reason: InvalidStorageObjectKeyReason;
readonly actualBytes?: number;
readonly maxBytes?: number;
constructor({
field,
reason,
actualBytes,
maxBytes
}: {
field: string;
reason: InvalidStorageObjectKeyReason;
actualBytes?: number;
maxBytes?: number;
}) {
const byteDetails =
actualBytes !== undefined && maxBytes !== undefined
? ` (${actualBytes} bytes, maximum ${maxBytes})`
: '';
super(
`Invalid storage object ${field}: ${invalidStorageObjectKeyReasonMessages[reason]}${byteDetails}`
);
this.name = 'InvalidStorageObjectKeyError';
this.field = field;
this.reason = reason;
this.actualBytes = actualBytes;
this.maxBytes = maxBytes;
}
}
import type { Readable } from 'node:stream';
import { Readable as NodeReadable } from 'node:stream';
import type { Mock } from 'vitest';
import type { IStorage } from '../interface';
import type {
CopyObjectParams,
CopyObjectResult,
DeleteObjectParams,
DeleteObjectResult,
DeleteObjectsByPrefixParams,
DeleteObjectsParams,
DeleteObjectsResult,
DownloadObjectParams,
DownloadObjectResult,
EnsureBucketResult,
ExistsObjectParams,
ExistsObjectResult,
GeneratePublicGetUrlParams,
GeneratePublicGetUrlResult,
GetObjectMetadataParams,
GetObjectMetadataResult,
ListObjectsParams,
ListObjectsResult,
PresignedGetUrlParams,
PresignedGetUrlResult,
PresignedPutUrlParams,
PresignedPutUrlResult,
StorageObjectKey,
StorageObjectMetadata,
StorageUploadBody,
UploadObjectParams,
UploadObjectResult
} from '../types';
import {
assertStorageObjectKey,
assertStorageObjectKeys,
assertStorageObjectPrefix,
assertRequiredStorageObjectPrefix
} from '../assert';
import { bindAbortSignalToReadable, throwIfStorageDownloadAborted } from '../utils';
type VitestLike = {
fn: <T extends (...args: any[]) => any>(impl?: T) => Mock<T>;
};
type StoredObject = {
body: Buffer;
metadata: StorageObjectMetadata;
contentType?: string;
contentLength?: number;
contentDisposition?: string;
etag?: string;
};
export type VitestStorageMock = IStorage & {
/** 便于在测试中直接读写内存对象(key -> object)。 */
__objects: Map<StorageObjectKey, StoredObject>;
/** 清空内存对象。 */
__reset: () => void;
/** 直接写入一个对象(绕过 uploadObject)。 */
__putObject: (key: StorageObjectKey, obj: Partial<StoredObject> & { body: Buffer }) => void;
};
export type CreateVitestStorageMockParams = {
vi: VitestLike;
bucketName?: string;
/**
* 用于构造 presigned/public URL 的 base(仅 mock 用)。
* 例如:`https://mock-storage.local`
*/
baseUrl?: string;
};
async function bodyToBuffer(body: StorageUploadBody): Promise<Buffer> {
if (Buffer.isBuffer(body)) return body;
if (typeof body === 'string') return Buffer.from(body);
return await readableToBuffer(body);
}
async function readableToBuffer(readable: Readable): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const chunk of readable) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks);
}
function bufferToReadable(buf: Buffer): Readable {
return NodeReadable.from(buf);
}
function getEtag(buf: Buffer) {
// mock: 非加密 hash,只是为了在测试里有稳定值可断言
return `etag_${buf.length}_${buf.subarray(0, 8).toString('hex')}`;
}
export function createVitestStorageMock(params: CreateVitestStorageMockParams): VitestStorageMock {
const { vi, bucketName = 'mock-bucket', baseUrl = 'https://mock-storage.local' } = params;
const objects = new Map<StorageObjectKey, StoredObject>();
let bucketEnsured = false;
const ensureBucket = vi.fn(async (): Promise<EnsureBucketResult> => {
const exists = bucketEnsured;
bucketEnsured = true;
return { exists, created: !exists, bucket: bucketName };
});
const checkObjectExists = vi.fn(
async ({ key }: ExistsObjectParams): Promise<ExistsObjectResult> => {
assertStorageObjectKey(key);
return { bucket: bucketName, key, exists: objects.has(key) };
}
);
const uploadObject = vi.fn(async (p: UploadObjectParams): Promise<UploadObjectResult> => {
assertStorageObjectKey(p.key);
const buf = await bodyToBuffer(p.body);
const contentLength = p.contentLength ?? buf.length;
objects.set(p.key, {
body: buf,
metadata: p.metadata ?? {},
contentType: p.contentType,
contentDisposition: p.contentDisposition,
contentLength,
etag: getEtag(buf)
});
return { bucket: bucketName, key: p.key };
});
const downloadObject = vi.fn(async (p: DownloadObjectParams): Promise<DownloadObjectResult> => {
assertStorageObjectKey(p.key);
throwIfStorageDownloadAborted(p.abortSignal);
const obj = objects.get(p.key);
if (!obj) {
throw new Error(`Object not found: ${p.key}`);
}
const body = bufferToReadable(obj.body);
bindAbortSignalToReadable({ readable: body, abortSignal: p.abortSignal });
return { bucket: bucketName, key: p.key, body };
});
const deleteObject = vi.fn(async (p: DeleteObjectParams): Promise<DeleteObjectResult> => {
assertStorageObjectKey(p.key);
objects.delete(p.key);
return { bucket: bucketName, key: p.key };
});
const deleteObjectsByMultiKeys = vi.fn(
async (p: DeleteObjectsParams): Promise<DeleteObjectsResult> => {
assertStorageObjectKeys(p.keys);
for (const key of p.keys) objects.delete(key);
return { bucket: bucketName, keys: [] };
}
);
const deleteObjectsByPrefix = vi.fn(
async (p: DeleteObjectsByPrefixParams): Promise<DeleteObjectsResult> => {
assertRequiredStorageObjectPrefix(p.prefix);
const keys: string[] = [];
for (const key of objects.keys()) {
if (key.startsWith(p.prefix)) keys.push(key);
}
for (const key of keys) objects.delete(key);
return { bucket: bucketName, keys: [] };
}
);
const generatePresignedPutUrl = vi.fn(
async (p: PresignedPutUrlParams): Promise<PresignedPutUrlResult> => {
assertStorageObjectKey(p.key);
const putUrl = `${baseUrl}/put/${encodeURIComponent(bucketName)}/${encodeURIComponent(p.key)}`;
// mock: 直接透传 metadata 作为“headers”
const metadata: Record<string, string> = p.metadata ? { ...p.metadata } : {};
return { bucket: bucketName, key: p.key, url: putUrl, metadata };
}
);
const generatePresignedGetUrl = vi.fn(
async (p: PresignedGetUrlParams): Promise<PresignedGetUrlResult> => {
assertStorageObjectKey(p.key);
const query = p.responseContentType
? `?response-content-type=${encodeURIComponent(p.responseContentType)}`
: '';
const getUrl = `${baseUrl}/get/${encodeURIComponent(bucketName)}/${encodeURIComponent(p.key)}${query}`;
return { bucket: bucketName, key: p.key, url: getUrl };
}
);
const generatePublicGetUrl = vi.fn(
({ key }: GeneratePublicGetUrlParams): GeneratePublicGetUrlResult => {
assertStorageObjectKey(key);
const publicGetUrl = `${baseUrl}/public/${encodeURIComponent(bucketName)}/${encodeURIComponent(key)}`;
return { url: publicGetUrl, bucket: bucketName, key };
}
);
const listObjects = vi.fn(async (p: ListObjectsParams): Promise<ListObjectsResult> => {
assertStorageObjectPrefix(p.prefix);
const keys = Array.from(objects.keys()).filter((k) =>
p.prefix ? k.startsWith(p.prefix) : true
);
keys.sort();
return { bucket: bucketName, keys };
});
const copyObjectInSelfBucket = vi.fn(async (p: CopyObjectParams): Promise<CopyObjectResult> => {
assertStorageObjectKey(p.sourceKey, 'sourceKey');
assertStorageObjectKey(p.targetKey, 'targetKey');
const src = objects.get(p.sourceKey);
if (!src) {
throw new Error(`Source object not found: ${p.sourceKey}`);
}
objects.set(p.targetKey, { ...src, body: Buffer.from(src.body) });
return { bucket: bucketName, sourceKey: p.sourceKey, targetKey: p.targetKey };
});
const getObjectMetadata = vi.fn(
async (p: GetObjectMetadataParams): Promise<GetObjectMetadataResult> => {
assertStorageObjectKey(p.key);
const obj = objects.get(p.key);
if (!obj) {
throw new Error(`Object not found: ${p.key}`);
}
return {
bucket: bucketName,
key: p.key,
metadata: obj.metadata ?? {},
contentType: obj.contentType,
contentLength: obj.contentLength,
etag: obj.etag
};
}
);
const destroy = vi.fn(async (): Promise<void> => {});
const mock: VitestStorageMock = {
bucketName,
ensureBucket,
checkObjectExists,
uploadObject,
downloadObject,
deleteObject,
deleteObjectsByMultiKeys,
deleteObjectsByPrefix,
generatePresignedPutUrl,
generatePresignedGetUrl,
generatePublicGetUrl,
listObjects,
copyObjectInSelfBucket,
getObjectMetadata,
destroy,
__objects: objects,
__reset: () => objects.clear(),
__putObject: (key, obj) => {
objects.set(key, {
body: obj.body,
metadata: obj.metadata ?? {},
contentType: obj.contentType,
contentLength: obj.contentLength ?? obj.body.length,
contentDisposition: obj.contentDisposition,
etag: obj.etag ?? getEtag(obj.body)
});
}
};
return mock;
}
export { createStorage } from './factory'; export { createStorage } from './factory';
export { createVitestStorageMock } from './testing/vitestMock'; export { createVitestStorageMock } from './helper/mock';
export type { VitestStorageMock, CreateVitestStorageMockParams } from './testing/vitestMock'; export type { VitestStorageMock, CreateVitestStorageMockParams } from './helper/mock';
export type { export type {
IStorage, IStorage,
IStorageOptions, IStorageOptions,
...@@ -35,7 +35,19 @@ export type { ...@@ -35,7 +35,19 @@ export type {
GetObjectMetadataParams, GetObjectMetadataParams,
GetObjectMetadataResult GetObjectMetadataResult
} from './types'; } from './types';
export { NoSuchBucketError, NoBucketReadPermissionError, EmptyObjectError } from './errors'; export {
NoSuchBucketError,
NoBucketReadPermissionError,
EmptyObjectError,
InvalidStorageObjectKeyError
} from './errors';
export type { InvalidStorageObjectKeyReason } from './errors';
export {
MAX_STORAGE_OBJECT_KEY_UTF8_BYTES,
assertStorageObjectKey,
assertStorageObjectKeys,
assertStorageObjectPrefix
} from './assert';
export { AwsS3StorageAdapter } from './adapters/aws-s3.adapter'; export { AwsS3StorageAdapter } from './adapters/aws-s3.adapter';
export { CosStorageAdapter } from './adapters/cos.adapter'; export { CosStorageAdapter } from './adapters/cos.adapter';
export { MinioStorageAdapter } from './adapters/minio.adapter'; export { MinioStorageAdapter } from './adapters/minio.adapter';
......
...@@ -309,7 +309,7 @@ export interface IStorage { ...@@ -309,7 +309,7 @@ export interface IStorage {
* *
* 注意: * 注意:
* - 各厂商对单次批量删除的最大数量限制不同,adapter 可能需要分批处理。 * - 各厂商对单次批量删除的最大数量限制不同,adapter 可能需要分批处理。
* - 返回的 `deleted` 通常只包含实际删除/确认删除的 key。 * - 返回的 `keys` 只包含删除失败、需要上层重试的 key;空数组表示全部成功。
*/ */
deleteObjectsByMultiKeys(params: DeleteObjectsParams): Promise<DeleteObjectsResult>; deleteObjectsByMultiKeys(params: DeleteObjectsParams): Promise<DeleteObjectsResult>;
......
import type { Readable } from 'node:stream'; // Keep the former test helper path available to external workspace consumers.
import { Readable as NodeReadable } from 'node:stream'; export * from '../helper/mock';
import type { MockedFunction } from 'vitest';
import type { IStorage } from '../interface';
import type {
CopyObjectParams,
CopyObjectResult,
DeleteObjectParams,
DeleteObjectResult,
DeleteObjectsByPrefixParams,
DeleteObjectsParams,
DeleteObjectsResult,
DownloadObjectParams,
DownloadObjectResult,
EnsureBucketResult,
ExistsObjectParams,
ExistsObjectResult,
GeneratePublicGetUrlParams,
GeneratePublicGetUrlResult,
GetObjectMetadataParams,
GetObjectMetadataResult,
ListObjectsParams,
ListObjectsResult,
PresignedGetUrlParams,
PresignedGetUrlResult,
PresignedPutUrlParams,
PresignedPutUrlResult,
StorageObjectKey,
StorageObjectMetadata,
StorageUploadBody,
UploadObjectParams,
UploadObjectResult
} from '../types';
type VitestLike = {
fn: <T extends (...args: any[]) => any>(impl?: T) => MockedFunction<T>;
};
type StoredObject = {
body: Buffer;
metadata: StorageObjectMetadata;
contentType?: string;
contentLength?: number;
contentDisposition?: string;
etag?: string;
};
export type VitestStorageMock = IStorage & {
/** 便于在测试中直接读写内存对象(key -> object)。 */
__objects: Map<StorageObjectKey, StoredObject>;
/** 清空内存对象。 */
__reset: () => void;
/** 直接写入一个对象(绕过 uploadObject)。 */
__putObject: (key: StorageObjectKey, obj: Partial<StoredObject> & { body: Buffer }) => void;
};
export type CreateVitestStorageMockParams = {
vi: VitestLike;
bucketName?: string;
/**
* 用于构造 presigned/public URL 的 base(仅 mock 用)。
* 例如:`https://mock-storage.local`
*/
baseUrl?: string;
};
async function bodyToBuffer(body: StorageUploadBody): Promise<Buffer> {
if (Buffer.isBuffer(body)) return body;
if (typeof body === 'string') return Buffer.from(body);
return await readableToBuffer(body);
}
async function readableToBuffer(readable: Readable): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const chunk of readable) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks);
}
function bufferToReadable(buf: Buffer): Readable {
return NodeReadable.from(buf);
}
function getEtag(buf: Buffer) {
// mock: 非加密 hash,只是为了在测试里有稳定值可断言
return `etag_${buf.length}_${buf.subarray(0, 8).toString('hex')}`;
}
export function createVitestStorageMock(params: CreateVitestStorageMockParams): VitestStorageMock {
const { vi, bucketName = 'mock-bucket', baseUrl = 'https://mock-storage.local' } = params;
const objects = new Map<StorageObjectKey, StoredObject>();
let bucketEnsured = false;
const ensureBucket = vi.fn(async (): Promise<EnsureBucketResult> => {
const exists = bucketEnsured;
bucketEnsured = true;
return { exists, created: !exists, bucket: bucketName };
});
const checkObjectExists = vi.fn(
async ({ key }: ExistsObjectParams): Promise<ExistsObjectResult> => {
return { bucket: bucketName, key, exists: objects.has(key) };
}
);
const uploadObject = vi.fn(async (p: UploadObjectParams): Promise<UploadObjectResult> => {
const buf = await bodyToBuffer(p.body);
const contentLength = p.contentLength ?? buf.length;
objects.set(p.key, {
body: buf,
metadata: p.metadata ?? {},
contentType: p.contentType,
contentDisposition: p.contentDisposition,
contentLength,
etag: getEtag(buf)
});
return { bucket: bucketName, key: p.key };
});
const downloadObject = vi.fn(async (p: DownloadObjectParams): Promise<DownloadObjectResult> => {
p.abortSignal?.throwIfAborted();
const obj = objects.get(p.key);
if (!obj) {
throw new Error(`Object not found: ${p.key}`);
}
const body = bufferToReadable(obj.body);
const abortDownload = () => {
body.destroy();
};
p.abortSignal?.addEventListener('abort', abortDownload, { once: true });
body.once('close', () => {
p.abortSignal?.removeEventListener('abort', abortDownload);
});
return { bucket: bucketName, key: p.key, body };
});
const deleteObject = vi.fn(async (p: DeleteObjectParams): Promise<DeleteObjectResult> => {
objects.delete(p.key);
return { bucket: bucketName, key: p.key };
});
const deleteObjectsByMultiKeys = vi.fn(
async (p: DeleteObjectsParams): Promise<DeleteObjectsResult> => {
for (const key of p.keys) objects.delete(key);
return { bucket: bucketName, keys: p.keys };
}
);
const deleteObjectsByPrefix = vi.fn(
async (p: DeleteObjectsByPrefixParams): Promise<DeleteObjectsResult> => {
if (!p.prefix) {
throw new Error('prefix must be a non-empty string');
}
const keys: string[] = [];
for (const key of objects.keys()) {
if (key.startsWith(p.prefix)) keys.push(key);
}
for (const key of keys) objects.delete(key);
return { bucket: bucketName, keys };
}
);
const generatePresignedPutUrl = vi.fn(
async (p: PresignedPutUrlParams): Promise<PresignedPutUrlResult> => {
const putUrl = `${baseUrl}/put/${encodeURIComponent(bucketName)}/${encodeURIComponent(p.key)}`;
// mock: 直接透传 metadata 作为“headers”
const metadata: Record<string, string> = p.metadata ? { ...p.metadata } : {};
return { bucket: bucketName, key: p.key, url: putUrl, metadata };
}
);
const generatePresignedGetUrl = vi.fn(
async (p: PresignedGetUrlParams): Promise<PresignedGetUrlResult> => {
const query = p.responseContentType
? `?response-content-type=${encodeURIComponent(p.responseContentType)}`
: '';
const getUrl = `${baseUrl}/get/${encodeURIComponent(bucketName)}/${encodeURIComponent(p.key)}${query}`;
return { bucket: bucketName, key: p.key, url: getUrl };
}
);
const generatePublicGetUrl = vi.fn(
({ key }: GeneratePublicGetUrlParams): GeneratePublicGetUrlResult => {
const publicGetUrl = `${baseUrl}/public/${encodeURIComponent(bucketName)}/${encodeURIComponent(key)}`;
return { url: publicGetUrl, bucket: bucketName, key };
}
);
const listObjects = vi.fn(async (p: ListObjectsParams): Promise<ListObjectsResult> => {
const keys = Array.from(objects.keys()).filter((k) =>
p.prefix ? k.startsWith(p.prefix) : true
);
keys.sort();
return { bucket: bucketName, keys };
});
const copyObjectInSelfBucket = vi.fn(async (p: CopyObjectParams): Promise<CopyObjectResult> => {
const src = objects.get(p.sourceKey);
if (!src) {
throw new Error(`Source object not found: ${p.sourceKey}`);
}
objects.set(p.targetKey, { ...src, body: Buffer.from(src.body) });
return { bucket: bucketName, sourceKey: p.sourceKey, targetKey: p.targetKey };
});
const getObjectMetadata = vi.fn(
async (p: GetObjectMetadataParams): Promise<GetObjectMetadataResult> => {
const obj = objects.get(p.key);
if (!obj) {
throw new Error(`Object not found: ${p.key}`);
}
return {
bucket: bucketName,
key: p.key,
metadata: obj.metadata ?? {},
contentType: obj.contentType,
contentLength: obj.contentLength,
etag: obj.etag
};
}
);
const destroy = vi.fn(async (): Promise<void> => {});
const mock: VitestStorageMock = {
bucketName,
ensureBucket,
checkObjectExists,
uploadObject,
downloadObject,
deleteObject,
deleteObjectsByMultiKeys,
deleteObjectsByPrefix,
generatePresignedPutUrl,
generatePresignedGetUrl,
generatePublicGetUrl,
listObjects,
copyObjectInSelfBucket,
getObjectMetadata,
destroy,
__objects: objects,
__reset: () => objects.clear(),
__putObject: (key, obj) => {
objects.set(key, {
body: obj.body,
metadata: obj.metadata ?? {},
contentType: obj.contentType,
contentLength: obj.contentLength ?? obj.body.length,
contentDisposition: obj.contentDisposition,
etag: obj.etag ?? getEtag(obj.body)
});
}
};
return mock;
}
...@@ -24,6 +24,8 @@ export type StorageBucketName = string; ...@@ -24,6 +24,8 @@ export type StorageBucketName = string;
* 说明: * 说明:
* - 在同一个 bucket 内唯一标识一个对象。 * - 在同一个 bucket 内唯一标识一个对象。
* - 通常形如:`a/b/c.txt`(用 `/` 形成“目录”层级,但对象存储并不是真正的目录结构)。 * - 通常形如:`a/b/c.txt`(用 `/` 形成“目录”层级,但对象存储并不是真正的目录结构)。
* - SDK 统一限制为 1 - 850 UTF-8 bytes,并拒绝不可移植的控制字符、反斜线、空路径段和 `.`/`..` 路径段。
* - 空格、`+`、`#`、`&`、`%`、`?`、中文和 emoji 均为合法字符,由 adapter 在 URL 层编码。
*/ */
export type StorageObjectKey = string; export type StorageObjectKey = string;
......
import type { Readable } from 'node:stream';
/** 将对象 key 编码为 URL path,同时保留对象存储使用的 `/` 层级分隔符。 */
export const encodeObjectKeyPath = (key: string): string =>
key
.split('/')
.map((segment) => encodeURIComponent(segment))
.join('/');
/** 将任意 AbortSignal.reason 归一为可用于 Readable.destroy 的 Error。 */
export const getAbortSignalError = (abortSignal: AbortSignal): Error => {
if (abortSignal.reason instanceof Error) return abortSignal.reason;
const error = new Error(
abortSignal.reason === undefined ? 'The operation was aborted' : String(abortSignal.reason)
);
error.name = 'AbortError';
return error;
};
/** 在发起远端下载前检查取消,确保预取消请求不会进入厂商 SDK。 */
export const throwIfStorageDownloadAborted = (abortSignal?: AbortSignal): void => {
if (abortSignal?.aborted) throw getAbortSignalError(abortSignal);
};
/**
* 将下载取消绑定到返回流,并在流关闭后解除监听。
* 二次检查覆盖等待厂商返回流期间发生的取消竞态。
*/
export const bindAbortSignalToReadable = ({
readable,
abortSignal
}: {
readable: Readable;
abortSignal?: AbortSignal;
}): void => {
if (!abortSignal) return;
if (abortSignal.aborted) {
readable.destroy();
throw getAbortSignalError(abortSignal);
}
const abortDownload = () => readable.destroy(getAbortSignalError(abortSignal));
abortSignal.addEventListener('abort', abortDownload, { once: true });
readable.once('close', () => {
abortSignal.removeEventListener('abort', abortDownload);
});
};
import fs from 'node:fs';
/**
* Priority: .env.test.local > .env.test > .env.local > .env
*/
function getEnvFilePath(): URL | undefined {
const files = ['.env.test.local', '.env.test', '.env.local', '.env'];
return files.map((f) => new URL(f, import.meta.url)).find((p) => fs.existsSync(p));
}
export function setup() {
process.loadEnvFile(getEnvFilePath());
}
export function teardown() {
// no-op
}
import { Readable } from 'node:stream';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import {
ValidTestBucketNamePrefixPattern,
type StorageIntegrationContext,
type StorageIntegrationProvider
} from '../providers';
import { createAsciiKeyAtLength } from '../helpers';
const readBody = async (body: Readable): Promise<Buffer> => {
const chunks: Buffer[] = [];
for await (const chunk of body) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks);
};
/**
* 对任意 IStorage 实现执行相同的外部行为契约。
* Provider 只负责环境和 bucket 生命周期,断言不依赖厂商 SDK。
*/
export const runStorageAdapterContract = (provider: StorageIntegrationProvider) => {
describe
.skipIf(!provider.enabled)
.sequential(`${provider.name} IStorage integration contract`, () => {
let context: StorageIntegrationContext;
beforeAll(async () => {
context = await provider.createContext();
});
afterAll(async () => {
await context?.cleanup();
});
it('creates a dedicated bucket and reports it through the interface', async () => {
expect(context.bucket).toMatch(ValidTestBucketNamePrefixPattern);
expect(context.storage.bucketName).toBe(context.bucket);
expect(context.initialEnsureResult).toMatchObject({ bucket: context.bucket });
expect(context.initialEnsureResult.created || context.initialEnsureResult.exists).toBe(
true
);
await expect(context.storage.ensureBucket()).resolves.toEqual({
bucket: context.bucket,
exists: true,
created: false
});
});
it('uploads, checks, downloads and reads metadata for an object', async () => {
const key = `${context.rootPrefix}object/basic.txt`;
const content = Buffer.from('FastGPT storage integration');
await expect(
context.storage.uploadObject({
key,
body: content,
contentType: 'text/plain',
contentLength: content.length,
contentDisposition: 'attachment; filename="basic.txt"',
metadata: { traceId: 'contract-basic' }
})
).resolves.toEqual({ bucket: context.bucket, key });
await expect(context.storage.checkObjectExists({ key })).resolves.toEqual({
bucket: context.bucket,
key,
exists: true
});
const metadata = await context.storage.getObjectMetadata({ key });
expect(metadata).toMatchObject({
bucket: context.bucket,
key,
contentType: 'text/plain',
contentLength: content.length,
metadata: { traceId: 'contract-basic' }
});
expect(metadata.etag).toBeTruthy();
const download = await context.storage.downloadObject({ key });
expect(download).toMatchObject({ bucket: context.bucket, key });
await expect(readBody(download.body)).resolves.toEqual(content);
});
it('accepts Readable and string upload bodies', async () => {
const streamKey = `${context.rootPrefix}upload/stream.txt`;
const stringKey = `${context.rootPrefix}upload/string.txt`;
await context.storage.uploadObject({
key: streamKey,
body: Readable.from(['stream-', 'body']),
contentType: 'text/plain'
});
await context.storage.uploadObject({
key: stringKey,
body: 'string-body',
contentType: 'text/plain',
contentLength: 11
});
const streamDownload = await context.storage.downloadObject({ key: streamKey });
const stringDownload = await context.storage.downloadObject({ key: stringKey });
await expect(readBody(streamDownload.body)).resolves.toEqual(Buffer.from('stream-body'));
await expect(readBody(stringDownload.body)).resolves.toEqual(Buffer.from('string-body'));
});
it('round-trips zero-byte and binary objects without coercing content', async () => {
const emptyKey = `${context.rootPrefix}binary/empty.bin`;
const binaryKey = `${context.rootPrefix}binary/raw.bin`;
const binaryContent = Buffer.from([0, 255, 1, 128, 13, 10, 0]);
await context.storage.uploadObject({
key: emptyKey,
body: Buffer.alloc(0),
contentType: 'application/octet-stream',
contentLength: 0
});
await context.storage.uploadObject({
key: binaryKey,
body: binaryContent,
contentType: 'application/octet-stream',
contentLength: binaryContent.length
});
const emptyDownload = await context.storage.downloadObject({ key: emptyKey });
const binaryDownload = await context.storage.downloadObject({ key: binaryKey });
await expect(readBody(emptyDownload.body)).resolves.toEqual(Buffer.alloc(0));
await expect(readBody(binaryDownload.body)).resolves.toEqual(binaryContent);
await expect(context.storage.getObjectMetadata({ key: emptyKey })).resolves.toMatchObject({
contentLength: 0,
contentType: 'application/octet-stream'
});
});
it('atomically overwrites object content and metadata at the same key', async () => {
const key = `${context.rootPrefix}overwrite/file.txt`;
await context.storage.uploadObject({
key,
body: 'old-content',
metadata: { revision: 'old', removedAfterOverwrite: 'true' }
});
await context.storage.uploadObject({
key,
body: 'new-content',
metadata: { revision: 'new' }
});
const download = await context.storage.downloadObject({ key });
await expect(readBody(download.body)).resolves.toEqual(Buffer.from('new-content'));
const metadata = await context.storage.getObjectMetadata({ key });
expect(metadata.metadata).toMatchObject({ revision: 'new' });
expect(metadata.metadata).not.toHaveProperty('removedAfterOverwrite');
});
it('isolates concurrent uploads and downloads under one prefix', async () => {
const prefix = `${context.rootPrefix}concurrent/`;
const entries = Array.from({ length: 20 }, (_, index) => ({
key: `${prefix}${index}.txt`,
content: `content-${index}`
}));
await Promise.all(
entries.map(({ key, content }) => context.storage.uploadObject({ key, body: content }))
);
const listed = await context.storage.listObjects({ prefix });
expect(new Set(listed.keys)).toEqual(new Set(entries.map(({ key }) => key)));
const contents = await Promise.all(
entries.map(async ({ key }) => {
const download = await context.storage.downloadObject({ key });
return (await readBody(download.body)).toString();
})
);
expect(contents).toEqual(entries.map(({ content }) => content));
});
it('round-trips and deletes an object key at the portable 850-byte limit', async () => {
const keyPrefix = `${context.rootPrefix}long-key/`;
const key = createAsciiKeyAtLength({ prefix: keyPrefix, byteLength: 850 });
expect(Buffer.byteLength(key)).toBe(850);
await context.storage.uploadObject({ key, body: 'long-key-content' });
await expect(context.storage.checkObjectExists({ key })).resolves.toMatchObject({
exists: true
});
const download = await context.storage.downloadObject({ key });
await expect(readBody(download.body)).resolves.toEqual(Buffer.from('long-key-content'));
await expect(context.storage.deleteObject({ key })).resolves.toEqual({
bucket: context.bucket,
key
});
await expect(context.storage.checkObjectExists({ key })).resolves.toMatchObject({
exists: false
});
});
it('lists and copies keys containing path and URL-sensitive characters', async () => {
const sourceKey = `${context.rootPrefix}special/team # & + % ?/\u6587\u4ef6-\ud83d\ude00.txt`;
const targetKey = `${context.rootPrefix}special/copied file.txt`;
await context.storage.uploadObject({ key: sourceKey, body: 'special-content' });
const listed = await context.storage.listObjects({
prefix: `${context.rootPrefix}special/`
});
expect(listed.bucket).toBe(context.bucket);
expect(listed.keys).toContain(sourceKey);
await expect(
context.storage.copyObjectInSelfBucket({ sourceKey, targetKey })
).resolves.toEqual({ bucket: context.bucket, sourceKey, targetKey });
const copied = await context.storage.downloadObject({ key: targetKey });
await expect(readBody(copied.body)).resolves.toEqual(Buffer.from('special-content'));
});
it('uploads and downloads through presigned URLs', async () => {
const key = `${context.rootPrefix}presigned/file.txt`;
const content = 'presigned-content';
const put = await context.storage.generatePresignedPutUrl({
key,
expiredSeconds: 300,
contentType: 'text/plain',
metadata: { uploadSource: 'contract' }
});
expect(put).toMatchObject({ bucket: context.bucket, key });
expect(() => new URL(put.url)).not.toThrow();
const putResponse = await fetch(put.url, {
method: 'PUT',
headers: put.metadata,
body: content
});
expect(putResponse.ok).toBe(true);
const get = await context.storage.generatePresignedGetUrl({
key,
expiredSeconds: 300,
responseContentType: 'text/plain'
});
expect(get).toMatchObject({ bucket: context.bucket, key });
const getResponse = await fetch(get.url);
expect(getResponse.ok).toBe(true);
expect(getResponse.headers.get('content-type')).toContain('text/plain');
await expect(getResponse.text()).resolves.toBe(content);
});
it('generates a public URL that preserves reserved characters inside the key path', () => {
const key = `${context.rootPrefix}public/folder name/file #+&%?.txt`;
const result = context.storage.generatePublicGetUrl({ key });
expect(result).toMatchObject({ bucket: context.bucket, key });
const url = new URL(result.url);
expect(decodeURIComponent(url.pathname).endsWith(`/${key}`)).toBe(true);
expect(url.hash).toBe('');
expect(url.search).toBe('');
});
it('rejects a download that was aborted before dispatch', async () => {
const key = `${context.rootPrefix}abort/file.txt`;
await context.storage.uploadObject({ key, body: 'abort-content' });
const controller = new AbortController();
controller.abort();
await expect(
context.storage.downloadObject({ key, abortSignal: controller.signal })
).rejects.toMatchObject({ name: 'AbortError' });
});
it('deletes a single object idempotently', async () => {
const key = `${context.rootPrefix}delete/single.txt`;
await context.storage.uploadObject({ key, body: 'delete-me' });
await expect(context.storage.deleteObject({ key })).resolves.toEqual({
bucket: context.bucket,
key
});
await expect(context.storage.deleteObject({ key })).resolves.toEqual({
bucket: context.bucket,
key
});
await expect(context.storage.checkObjectExists({ key })).resolves.toMatchObject({
exists: false
});
});
it('deletes multiple keys and treats an empty list as a no-op', async () => {
await expect(context.storage.deleteObjectsByMultiKeys({ keys: [] })).resolves.toEqual({
bucket: context.bucket,
keys: []
});
const keys = [
`${context.rootPrefix}delete/multi-1.txt`,
`${context.rootPrefix}delete/multi-2.txt`,
`${context.rootPrefix}delete/multi-3.txt`
];
await Promise.all(keys.map((key) => context.storage.uploadObject({ key, body: key })));
await expect(context.storage.deleteObjectsByMultiKeys({ keys })).resolves.toEqual({
bucket: context.bucket,
keys: []
});
const checks = await Promise.all(
keys.map((key) => context.storage.checkObjectExists({ key }))
);
expect(checks.every(({ exists }) => !exists)).toBe(true);
});
it('treats missing keys in batch and prefix deletion as successful no-ops', async () => {
const existingKey = `${context.rootPrefix}delete-missing/existing.txt`;
const missingKey = `${context.rootPrefix}delete-missing/missing.txt`;
await context.storage.uploadObject({ key: existingKey, body: 'existing' });
await expect(
context.storage.deleteObjectsByMultiKeys({ keys: [existingKey, missingKey] })
).resolves.toEqual({ bucket: context.bucket, keys: [] });
await expect(
context.storage.deleteObjectsByPrefix({
prefix: `${context.rootPrefix}delete-missing/never-created/`
})
).resolves.toEqual({ bucket: context.bucket, keys: [] });
});
it('rejects an empty prefix and deletes only matching objects', async () => {
for (const prefix of ['', ' ']) {
await expect(context.storage.deleteObjectsByPrefix({ prefix })).rejects.toThrow(
'Prefix is required'
);
}
const prefix = `${context.rootPrefix}delete-prefix/target/`;
const targetKeys = [`${prefix}first.txt`, `${prefix}second.txt`];
const siblingKey = `${context.rootPrefix}delete-prefix/sibling.txt`;
await Promise.all(
[...targetKeys, siblingKey].map((key) =>
context.storage.uploadObject({ key, body: 'prefix-delete' })
)
);
await expect(context.storage.deleteObjectsByPrefix({ prefix })).resolves.toEqual({
bucket: context.bucket,
keys: []
});
await expect(context.storage.listObjects({ prefix })).resolves.toEqual({
bucket: context.bucket,
keys: []
});
await expect(context.storage.checkObjectExists({ key: siblingKey })).resolves.toMatchObject(
{
exists: true
}
);
});
it('rejects reads for a missing object', async () => {
const key = `${context.rootPrefix}missing/not-found.txt`;
await expect(context.storage.downloadObject({ key })).rejects.toBeTruthy();
await expect(context.storage.getObjectMetadata({ key })).rejects.toBeTruthy();
await expect(context.storage.checkObjectExists({ key })).resolves.toMatchObject({
bucket: context.bucket,
key,
exists: false
});
});
it('returns an empty list for an unmatched prefix', async () => {
await expect(
context.storage.listObjects({ prefix: `${context.rootPrefix}not-present/` })
).resolves.toEqual({ bucket: context.bucket, keys: [] });
});
it('allows independently created adapters to be destroyed repeatedly', async () => {
const isolatedStorage = context.createStorage();
await expect(isolatedStorage.ensureBucket()).resolves.toMatchObject({
bucket: context.bucket,
exists: true
});
await expect(isolatedStorage.destroy()).resolves.toBeUndefined();
await expect(isolatedStorage.destroy()).resolves.toBeUndefined();
});
});
};
import { storageIntegrationProviders } from '../providers';
import { runStorageAdapterContract } from './storage.contract';
for (const provider of storageIntegrationProviders) {
runStorageAdapterContract(provider);
}
import type { IStorage } from '../../src/interface';
/** 构造指定总字节数的 ASCII key,并限制单个路径段长度以兼容文件系统型对象存储。 */
export const createAsciiKeyAtLength = ({
prefix,
byteLength,
maxSegmentLength = 200
}: {
prefix: string;
byteLength: number;
maxSegmentLength?: number;
}): string => {
let remainingLength = byteLength - Buffer.byteLength(prefix);
if (remainingLength <= 0) {
throw new Error('Target byte length must be longer than the prefix');
}
const segments: string[] = [];
while (remainingLength > 0) {
const separatorLength = segments.length > 0 ? 1 : 0;
const segmentLength = Math.min(maxSegmentLength, remainingLength - separatorLength);
if (segmentLength <= 0) throw new Error('Insufficient space for another path segment');
segments.push('a'.repeat(segmentLength));
remainingLength -= segmentLength + separatorLength;
}
return `${prefix}${segments.join('/')}`;
};
/**
* 删除已存在的固定集成测试桶,供下次运行重新创建干净环境。
* `DeleteObjectsResult.keys` 是失败项;只要存在失败 key,就保留桶并让测试失败。
*/
export const removeIntegrationBucketIfExists = async ({
storage,
bucketExists,
deleteBucket
}: {
storage: IStorage;
bucketExists: () => Promise<boolean>;
deleteBucket: () => Promise<void>;
}): Promise<void> => {
if (!(await bucketExists())) return;
const { keys } = await storage.listObjects({});
if (keys.length > 0) {
const { keys: failedKeys } = await storage.deleteObjectsByMultiKeys({ keys });
if (failedKeys.length > 0) {
throw new Error(`Failed to clean integration test bucket: ${failedKeys.join(', ')}`);
}
}
await deleteBucket();
};
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import type { MinioStorageAdapter } from '../../../src/adapters/minio.adapter';
import { InvalidStorageObjectKeyError } from '../../../src/errors';
import { minioIntegrationProvider, type StorageIntegrationContext } from '../providers';
import { createAsciiKeyAtLength } from '../helpers';
const uploadInBatches = async ({
context,
keys,
batchSize
}: {
context: StorageIntegrationContext;
keys: string[];
batchSize: number;
}) => {
for (let index = 0; index < keys.length; index += batchSize) {
await Promise.all(
keys.slice(index, index + batchSize).map((key) =>
context.storage.uploadObject({
key,
body: 'x',
contentType: 'text/plain',
contentLength: 1
})
)
);
}
};
describe.skipIf(!minioIntegrationProvider.enabled).sequential('MinIO-specific integration', () => {
let context: StorageIntegrationContext;
beforeAll(async () => {
context = await minioIntegrationProvider.createContext();
});
afterAll(async () => {
await context?.cleanup();
});
it('recreates the stable bucket and removes objects left by an interrupted run', async () => {
const interruptedContext = context;
const staleKey = `${interruptedContext.rootPrefix}stale/object.txt`;
await interruptedContext.storage.uploadObject({ key: staleKey, body: 'stale' });
await interruptedContext.storage.destroy();
context = await minioIntegrationProvider.createContext();
expect(context.bucket).toBe(interruptedContext.bucket);
await expect(context.storage.listObjects({ prefix: staleKey })).resolves.toEqual({
bucket: context.bucket,
keys: []
});
});
it('creates a missing bucket through MinioStorageAdapter', () => {
expect(context.initialEnsureResult).toEqual({
bucket: context.bucket,
exists: false,
created: true
});
});
it('deletes 401 URL-sensitive keys across the 400-object prefix page boundary', async () => {
const prefix = `${context.rootPrefix}prefix-page/team & +/`;
const keys = Array.from({ length: 401 }, (_, index) => `${prefix}file + ${index}.txt`);
await uploadInBatches({ context, keys, batchSize: 20 });
const beforeDelete = await context.storage.listObjects({ prefix });
expect(new Set(beforeDelete.keys)).toEqual(new Set(keys));
await expect(context.storage.deleteObjectsByPrefix({ prefix })).resolves.toEqual({
bucket: context.bucket,
keys: []
});
await expect(context.storage.listObjects({ prefix })).resolves.toEqual({
bucket: context.bucket,
keys: []
});
});
it('lists 1001 objects across pages and deletes them across 1000-key batches', async () => {
const prefix = `${context.rootPrefix}list-page/`;
const keys = Array.from({ length: 1001 }, (_, index) => `${prefix}${index}.txt`);
await uploadInBatches({ context, keys, batchSize: 25 });
const listed = await context.storage.listObjects({ prefix });
expect(new Set(listed.keys)).toEqual(new Set(keys));
await expect(context.storage.deleteObjectsByMultiKeys({ keys })).resolves.toEqual({
bucket: context.bucket,
keys: []
});
await expect(context.storage.listObjects({ prefix })).resolves.toEqual({
bucket: context.bucket,
keys: []
});
});
it('rejects an object key beyond the portable 850-byte limit without creating an object', async () => {
const prefix = `${context.rootPrefix}too-long/`;
const key = createAsciiKeyAtLength({ prefix, byteLength: 851 });
await expect(context.storage.uploadObject({ key, body: 'too-long' })).rejects.toMatchObject({
name: InvalidStorageObjectKeyError.name,
reason: 'too_long',
actualBytes: 851,
maxBytes: 850
});
await expect(context.storage.listObjects({ prefix })).resolves.toEqual({
bucket: context.bucket,
keys: []
});
});
it('grants anonymous GET without granting anonymous PUT', async () => {
const storage = context.storage as MinioStorageAdapter;
const key = `${context.rootPrefix}public/folder name/file #+.txt`;
await storage.uploadObject({ key, body: 'public-content' });
const publicUrl = storage.generatePublicGetUrl({ key }).url;
const privateResponse = await fetch(publicUrl);
expect(privateResponse.status).toBe(403);
await storage.ensurePublicBucketPolicy();
const publicResponse = await fetch(publicUrl);
expect(publicResponse.ok).toBe(true);
await expect(publicResponse.text()).resolves.toBe('public-content');
const anonymousPut = await fetch(publicUrl, { method: 'PUT', body: 'overwritten' });
expect(anonymousPut.status).toBe(403);
const authenticatedDownload = await storage.downloadObject({ key });
const chunks: Buffer[] = [];
for await (const chunk of authenticatedDownload.body) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
expect(Buffer.concat(chunks).toString()).toBe('public-content');
});
it('removes bucket lifecycle when no lifecycle configuration exists', async () => {
const storage = context.storage as MinioStorageAdapter;
await expect(storage.removeBucketLifecycle()).resolves.toBeUndefined();
});
});
import * as http from 'node:http';
import type { AddressInfo, Socket } from 'node:net';
import { afterEach, describe, expect, it } from 'vitest';
import {
createMinioTimeoutTransport,
MinioStorageAdapter
} from '../../../src/adapters/minio.adapter';
const servers = new Set<http.Server>();
const listen = async (server: http.Server): Promise<number> => {
await new Promise<void>((resolve, reject) => {
server.once('error', reject);
server.listen(0, '127.0.0.1', resolve);
});
servers.add(server);
return (server.address() as AddressInfo).port;
};
const closeServer = async (server: http.Server) => {
server.closeAllConnections();
await new Promise<void>((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
});
servers.delete(server);
};
const withTimeout = async <T>(promise: Promise<T>, message: string): Promise<T> => {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
promise,
new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error(message)), 1000);
})
]);
} finally {
if (timer) clearTimeout(timer);
}
};
afterEach(async () => {
await Promise.all([...servers].map(closeServer));
});
describe('MinIO timeout transport integration', () => {
it('destroys a real socket when the server never sends response headers', async () => {
const sockets = new Set<Socket>();
const server = http.createServer(() => {});
server.on('connection', (socket) => {
sockets.add(socket);
socket.once('close', () => sockets.delete(socket));
});
const port = await listen(server);
const transport = createMinioTimeoutTransport({ transport: http, timeoutMs: 50 });
const error = await new Promise<Error>((resolve, reject) => {
const request = transport.request({ host: '127.0.0.1', port, path: '/' });
request.once('error', resolve);
request.once('response', () => reject(new Error('Unexpected response')));
request.end();
});
expect(error.message).toBe('MinIO request timeout after 50ms');
await expect.poll(() => sockets.size, { timeout: 1000 }).toBe(0);
await closeServer(server);
});
it('destroys a real socket when the response body never completes', async () => {
const sockets = new Set<Socket>();
const server = http.createServer((_request, response) => {
response.writeHead(200, { 'Content-Length': '100' });
response.write('partial');
});
server.on('connection', (socket) => {
sockets.add(socket);
socket.once('close', () => sockets.delete(socket));
});
const port = await listen(server);
const transport = createMinioTimeoutTransport({ transport: http, timeoutMs: 50 });
let responseComplete: boolean | undefined;
await new Promise<void>((resolve, reject) => {
const request = transport.request({ host: '127.0.0.1', port, path: '/' }, (response) => {
response.resume();
response.once('end', () => reject(new Error('Unexpected complete response')));
response.once('error', () => {
responseComplete = response.complete;
resolve();
});
response.once('aborted', () => {
responseComplete = response.complete;
resolve();
});
});
request.once('error', () => {});
request.end();
});
expect(responseComplete).toBe(false);
await expect.poll(() => sockets.size, { timeout: 1000 }).toBe(0);
await closeServer(server);
});
it('closes a real download socket when the caller aborts after response headers', async () => {
const sockets = new Set<Socket>();
const server = http.createServer((_request, response) => {
response.writeHead(200, {
'Content-Length': '100',
'Content-Type': 'application/octet-stream'
});
response.write('partial');
});
server.on('connection', (socket) => {
sockets.add(socket);
socket.once('close', () => sockets.delete(socket));
});
const port = await listen(server);
const storage = new MinioStorageAdapter({
vendor: 'minio',
bucket: 'test-bucket',
endpoint: `http://127.0.0.1:${port}`,
region: 'us-east-1',
forcePathStyle: true,
maxRetries: 1,
credentials: { accessKeyId: 'access-key', secretAccessKey: 'secret-key' }
});
const controller = new AbortController();
const { body } = await storage.downloadObject({
key: 'abort/file.bin',
abortSignal: controller.signal
});
body.on('error', () => {});
const streamClosed = new Promise<void>((resolve) => body.once('close', resolve));
controller.abort(new Error('client aborted'));
await streamClosed;
expect(body.destroyed).toBe(true);
await expect.poll(() => sockets.size, { timeout: 1000 }).toBe(0);
await storage.destroy();
await closeServer(server);
});
it('aborts a real AWS-compatible request while waiting for response headers', async () => {
const sockets = new Set<Socket>();
let notifyRequest: (() => void) | undefined;
const requestReceived = new Promise<void>((resolve) => {
notifyRequest = resolve;
});
const server = http.createServer(() => notifyRequest?.());
server.on('connection', (socket) => {
sockets.add(socket);
socket.once('close', () => sockets.delete(socket));
});
const port = await listen(server);
const storage = new MinioStorageAdapter({
vendor: 'minio',
bucket: 'test-bucket',
endpoint: `http://127.0.0.1:${port}`,
region: 'us-east-1',
forcePathStyle: true,
maxRetries: 1,
credentials: { accessKeyId: 'access-key', secretAccessKey: 'secret-key' }
});
const controller = new AbortController();
const downloadPromise = storage.downloadObject({
key: 'abort/waiting-for-headers.bin',
abortSignal: controller.signal
});
await withTimeout(requestReceived, 'AWS-compatible request did not reach the local server');
controller.abort(new Error('client aborted'));
await expect(downloadPromise).rejects.toMatchObject({ name: 'AbortError' });
await expect.poll(() => sockets.size, { timeout: 1000 }).toBe(0);
await storage.destroy();
await closeServer(server);
});
});
import { randomUUID } from 'node:crypto';
import {
CreateBucketCommand,
DeleteBucketCommand,
HeadBucketCommand,
S3Client,
type BucketLocationConstraint
} from '@aws-sdk/client-s3';
import OSS from 'ali-oss';
import COS from 'cos-nodejs-sdk-v5';
import * as Minio from 'minio';
import { createStorage } from '../../src/factory';
import type { IStorage } from '../../src/interface';
import type { EnsureBucketResult } from '../../src/types';
import { removeIntegrationBucketIfExists } from './helpers';
export type StorageIntegrationProviderName = 'aws-s3' | 'minio' | 'oss' | 'cos';
export type StorageIntegrationContext = {
provider: StorageIntegrationProviderName;
storage: IStorage;
bucket: string;
rootPrefix: string;
initialEnsureResult: EnsureBucketResult;
createStorage: () => IStorage;
cleanup: () => Promise<void>;
};
export type StorageIntegrationProvider = {
name: StorageIntegrationProviderName;
enabled: boolean;
createContext: () => Promise<StorageIntegrationContext>;
};
const isEnabled = (name: string) => process.env[name]?.toLowerCase() === 'true';
const getRequiredEnv = (name: string): string => {
const value = process.env[name]?.trim();
if (!value) throw new Error(`Missing integration test environment variable: ${name}`);
return value;
};
export const ValidTestBucketNamePrefixPattern = /^fastgpt-sdk\.integration-/;
const getTestBucket = (envName: string): string => {
const bucket = getRequiredEnv(envName);
if (ValidTestBucketNamePrefixPattern.test(bucket)) {
return bucket;
}
throw new Error(`Met invalid bucket name to protect non-test buckets`);
};
const isBucketNotFoundError = (error: unknown): boolean => {
if (!error || typeof error !== 'object') return false;
const value = error as {
code?: unknown;
status?: unknown;
statusCode?: unknown;
$metadata?: { httpStatusCode?: unknown };
};
return (
value.code === 'NoSuchBucket' ||
value.code === 'NotFound' ||
value.status === 404 ||
value.statusCode === 404 ||
value.$metadata?.httpStatusCode === 404
);
};
const createContextResult = async ({
provider,
storage,
bucket,
initialEnsureResult,
createStorage,
bucketExists,
deleteBucket,
destroyProvider
}: {
provider: StorageIntegrationProviderName;
storage: IStorage;
bucket: string;
initialEnsureResult: EnsureBucketResult;
createStorage: () => IStorage;
bucketExists: () => Promise<boolean>;
deleteBucket: () => Promise<void>;
destroyProvider?: () => Promise<void> | void;
}): Promise<StorageIntegrationContext> => ({
provider,
storage,
bucket,
rootPrefix: `contract/${randomUUID()}/`,
initialEnsureResult,
createStorage,
cleanup: async () => {
try {
await removeIntegrationBucketIfExists({ storage, bucketExists, deleteBucket });
} finally {
try {
await storage.destroy();
} finally {
await destroyProvider?.();
}
}
}
});
const createMinioProvider = (): StorageIntegrationProvider => ({
name: 'minio',
enabled: isEnabled('STORAGE_TEST_MINIO_ENABLED'),
createContext: async () => {
const endpoint = getRequiredEnv('STORAGE_TEST_MINIO_ENDPOINT');
const region = getRequiredEnv('STORAGE_TEST_MINIO_REGION');
const accessKeyId = getRequiredEnv('STORAGE_TEST_MINIO_ACCESS_KEY_ID');
const secretAccessKey = getRequiredEnv('STORAGE_TEST_MINIO_SECRET_ACCESS_KEY');
const endpointUrl = new URL(endpoint);
const useSSL = endpointUrl.protocol === 'https:';
const bucket = getTestBucket('STORAGE_TEST_MINIO_BUCKET');
const adminClient = new Minio.Client({
endPoint: endpointUrl.hostname,
port: endpointUrl.port ? Number(endpointUrl.port) : useSSL ? 443 : 80,
useSSL,
accessKey: accessKeyId,
secretKey: secretAccessKey,
region
});
const createMinioStorage = () =>
createStorage({
vendor: 'minio',
bucket,
endpoint,
region,
forcePathStyle: true,
maxRetries: 1,
credentials: { accessKeyId, secretAccessKey }
});
const storage = createMinioStorage();
const bucketExists = () => adminClient.bucketExists(bucket);
const deleteBucket = () => adminClient.removeBucket(bucket);
await removeIntegrationBucketIfExists({ storage, bucketExists, deleteBucket });
const initialEnsureResult = await storage.ensureBucket();
return createContextResult({
provider: 'minio',
storage,
bucket,
initialEnsureResult,
createStorage: createMinioStorage,
bucketExists,
deleteBucket
});
}
});
const createAwsS3Provider = (): StorageIntegrationProvider => ({
name: 'aws-s3',
enabled: isEnabled('STORAGE_TEST_AWS_S3_ENABLED'),
createContext: async () => {
const endpoint = getRequiredEnv('STORAGE_TEST_AWS_S3_ENDPOINT');
const region = getRequiredEnv('STORAGE_TEST_AWS_S3_REGION');
const accessKeyId = getRequiredEnv('STORAGE_TEST_AWS_S3_ACCESS_KEY_ID');
const secretAccessKey = getRequiredEnv('STORAGE_TEST_AWS_S3_SECRET_ACCESS_KEY');
const forcePathStyle = isEnabled('STORAGE_TEST_AWS_S3_FORCE_PATH_STYLE');
const bucket = getTestBucket('STORAGE_TEST_AWS_S3_BUCKET');
const adminClient = new S3Client({
endpoint,
region,
forcePathStyle,
credentials: { accessKeyId, secretAccessKey }
});
const bucketExists = async () => {
try {
await adminClient.send(new HeadBucketCommand({ Bucket: bucket }));
return true;
} catch (error) {
if (isBucketNotFoundError(error)) return false;
throw error;
}
};
const deleteBucket = () =>
adminClient.send(new DeleteBucketCommand({ Bucket: bucket })).then(() => undefined);
const createAwsStorage = () =>
createStorage({
vendor: 'aws-s3',
bucket,
endpoint,
region,
forcePathStyle,
maxRetries: 1,
credentials: { accessKeyId, secretAccessKey }
});
const storage = createAwsStorage();
await removeIntegrationBucketIfExists({ storage, bucketExists, deleteBucket });
await adminClient.send(
new CreateBucketCommand({
Bucket: bucket,
CreateBucketConfiguration:
region === 'us-east-1'
? undefined
: { LocationConstraint: region as BucketLocationConstraint }
})
);
const initialEnsureResult = await storage.ensureBucket();
return createContextResult({
provider: 'aws-s3',
storage,
bucket,
initialEnsureResult,
createStorage: createAwsStorage,
bucketExists,
deleteBucket,
destroyProvider: () => adminClient.destroy()
});
}
});
const createOssProvider = (): StorageIntegrationProvider => ({
name: 'oss',
enabled: isEnabled('STORAGE_TEST_OSS_ENABLED'),
createContext: async () => {
const endpoint = getRequiredEnv('STORAGE_TEST_OSS_ENDPOINT');
const region = getRequiredEnv('STORAGE_TEST_OSS_REGION');
const accessKeyId = getRequiredEnv('STORAGE_TEST_OSS_ACCESS_KEY_ID');
const secretAccessKey = getRequiredEnv('STORAGE_TEST_OSS_SECRET_ACCESS_KEY');
const bucket = getTestBucket('STORAGE_TEST_OSS_BUCKET');
const adminClient = new OSS({
endpoint,
region,
accessKeyId,
accessKeySecret: secretAccessKey
});
const createOssStorage = () =>
createStorage({
vendor: 'oss',
bucket,
endpoint,
region,
secure: endpoint.startsWith('https:'),
credentials: { accessKeyId, secretAccessKey }
});
const storage = createOssStorage();
const bucketExists = async () => {
try {
await adminClient.getBucketInfo(bucket);
return true;
} catch (error) {
if (isBucketNotFoundError(error)) return false;
throw error;
}
};
const deleteBucket = () => adminClient.deleteBucket(bucket).then(() => undefined);
await removeIntegrationBucketIfExists({ storage, bucketExists, deleteBucket });
await adminClient.putBucket(bucket);
const initialEnsureResult = await storage.ensureBucket();
return createContextResult({
provider: 'oss',
storage,
bucket,
initialEnsureResult,
createStorage: createOssStorage,
bucketExists,
deleteBucket
});
}
});
const createCosProvider = (): StorageIntegrationProvider => ({
name: 'cos',
enabled: isEnabled('STORAGE_TEST_COS_ENABLED'),
createContext: async () => {
const region = getRequiredEnv('STORAGE_TEST_COS_REGION');
const appId = getRequiredEnv('STORAGE_TEST_COS_APP_ID');
const accessKeyId = getRequiredEnv('STORAGE_TEST_COS_ACCESS_KEY_ID');
const secretAccessKey = getRequiredEnv('STORAGE_TEST_COS_SECRET_ACCESS_KEY');
const bucket = getTestBucket('STORAGE_TEST_COS_BUCKET');
if (!bucket.endsWith(`-${appId}`)) {
throw new Error('STORAGE_TEST_COS_BUCKET must end with the configured COS app ID');
}
const adminClient = new COS({ SecretId: accessKeyId, SecretKey: secretAccessKey });
const createCosStorage = () =>
createStorage({
vendor: 'cos',
bucket,
region,
protocol: 'https:',
credentials: { accessKeyId, secretAccessKey }
});
const storage = createCosStorage();
const bucketExists = async () => {
try {
await adminClient.headBucket({ Bucket: bucket, Region: region });
return true;
} catch (error) {
if (isBucketNotFoundError(error)) return false;
throw error;
}
};
const deleteBucket = () =>
adminClient.deleteBucket({ Bucket: bucket, Region: region }).then(() => undefined);
await removeIntegrationBucketIfExists({ storage, bucketExists, deleteBucket });
await adminClient.putBucket({ Bucket: bucket, Region: region });
const initialEnsureResult = await storage.ensureBucket();
return createContextResult({
provider: 'cos',
storage,
bucket,
initialEnsureResult,
createStorage: createCosStorage,
bucketExists,
deleteBucket
});
}
});
export const minioIntegrationProvider = createMinioProvider();
export const storageIntegrationProviders: StorageIntegrationProvider[] = [
createAwsS3Provider(),
minioIntegrationProvider,
createOssProvider(),
createCosProvider()
];
import * as http from 'node:http';
import type { AddressInfo, Socket } from 'node:net';
import { afterEach, describe, expect, it } from 'vitest';
import { CosStorageAdapter } from '../../../src/adapters/cos.adapter';
import { OssStorageAdapter } from '../../../src/adapters/oss.adapter';
const servers = new Set<http.Server>();
const listen = async (server: http.Server): Promise<number> => {
await new Promise<void>((resolve, reject) => {
server.once('error', reject);
server.listen(0, '127.0.0.1', resolve);
});
servers.add(server);
return (server.address() as AddressInfo).port;
};
const closeServer = async (server: http.Server) => {
server.closeAllConnections();
await new Promise<void>((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
});
servers.delete(server);
};
const createStalledObjectServer = () => {
const sockets = new Set<Socket>();
let notifyRequest: (() => void) | undefined;
const requestReceived = new Promise<void>((resolve) => {
notifyRequest = resolve;
});
const server = http.createServer((_request, response) => {
response.writeHead(200, {
'Content-Length': '100',
'Content-Type': 'application/octet-stream'
});
response.write('partial');
notifyRequest?.();
});
server.on('connection', (socket) => {
sockets.add(socket);
socket.once('close', () => sockets.delete(socket));
});
return { server, sockets, requestReceived };
};
const withTimeout = async <T>(promise: Promise<T>, message: string): Promise<T> => {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
promise,
new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error(message)), 1000);
})
]);
} finally {
if (timer) clearTimeout(timer);
}
};
afterEach(async () => {
await Promise.all([...servers].map(closeServer));
});
describe('provider download abort transport integration', () => {
it('closes the OSS response socket when the caller aborts an in-flight stream', async () => {
const { server, sockets, requestReceived } = createStalledObjectServer();
const port = await listen(server);
const storage = new OssStorageAdapter({
vendor: 'oss',
bucket: 'test-bucket',
region: 'oss-cn-hangzhou',
endpoint: `http://127.0.0.1:${port}`,
cname: true,
secure: false,
credentials: { accessKeyId: 'access-key', secretAccessKey: 'secret-key' }
});
const controller = new AbortController();
const abortReason = new Error('client aborted');
const { body } = await storage.downloadObject({
key: 'abort/file.bin',
abortSignal: controller.signal
});
body.on('error', () => {});
const streamClosed = new Promise<void>((resolve) => body.once('close', resolve));
await withTimeout(requestReceived, 'OSS request did not reach the local server');
controller.abort(abortReason);
await withTimeout(streamClosed, 'OSS stream did not close after abort');
expect(body.errored).toBe(abortReason);
await expect.poll(() => sockets.size, { timeout: 1000 }).toBe(0);
await storage.destroy();
await closeServer(server);
});
it('aborts the COS request when the caller aborts its output stream', async () => {
const { server, sockets, requestReceived } = createStalledObjectServer();
const port = await listen(server);
const storage = new CosStorageAdapter({
vendor: 'cos',
bucket: 'test-bucket-1250000000',
region: 'ap-guangzhou',
protocol: 'http:',
domain: `127.0.0.1:${port}`,
credentials: { accessKeyId: 'secret-id', secretAccessKey: 'secret-key' }
});
const controller = new AbortController();
const abortReason = new Error('client aborted');
const { body } = await storage.downloadObject({
key: 'abort/file.bin',
abortSignal: controller.signal
});
body.on('error', () => {});
const streamClosed = new Promise<void>((resolve) => body.once('close', resolve));
await withTimeout(requestReceived, 'COS request did not reach the local server');
controller.abort(abortReason);
await withTimeout(streamClosed, 'COS stream did not close after abort');
expect(body.errored).toBe(abortReason);
await expect.poll(() => sockets.size, { timeout: 1000 }).toBe(0);
await storage.destroy();
await closeServer(server);
});
});
...@@ -8,7 +8,7 @@ import { ...@@ -8,7 +8,7 @@ import {
createS3AccessLinkService, createS3AccessLinkService,
encodeExpiresAtMinute, encodeExpiresAtMinute,
type S3DownloadAliasStore type S3DownloadAliasStore
} from '@fastgpt-sdk/storage/access-link'; } from '../../src/access-link';
const baseNow = new Date('2026-01-01T00:00:00.000Z'); const baseNow = new Date('2026-01-01T00:00:00.000Z');
const getFutureDate = (minutes: number) => new Date(baseNow.getTime() + minutes * 60_000); const getFutureDate = (minutes: number) => new Date(baseNow.getTime() + minutes * 60_000);
......
import { PassThrough, Readable } from 'node:stream';
import { describe, expect, it, vi } from 'vitest';
import { AwsS3StorageAdapter } from '../../../src/adapters/aws-s3.adapter';
const createAdapter = () =>
new AwsS3StorageAdapter({
vendor: 'aws-s3',
bucket: 'fastgpt-private',
endpoint: 'http://localhost:9000',
region: 'us-east-1',
forcePathStyle: true,
maxRetries: 1,
credentials: {
accessKeyId: 'access-key',
secretAccessKey: 'secret-key'
}
});
describe('AwsS3StorageAdapter.downloadObject', () => {
it('rejects a pre-aborted download without dispatching an AWS request', async () => {
const adapter = createAdapter();
const send = vi.fn();
(adapter as any).client.send = send;
const controller = new AbortController();
controller.abort();
await expect(
adapter.downloadObject({
key: 'dataset/team/file.txt',
abortSignal: controller.signal
})
).rejects.toMatchObject({ name: 'AbortError' });
expect(send).not.toHaveBeenCalled();
});
it('passes the caller abort signal to the AWS request handler', async () => {
const adapter = createAdapter();
const body = Readable.from([Buffer.from('file')]);
const send = vi.fn().mockResolvedValue({ Body: body });
(adapter as any).client.send = send;
const controller = new AbortController();
const result = await adapter.downloadObject({
key: 'dataset/team/file.txt',
abortSignal: controller.signal
});
expect(result.body).toBe(body);
expect(send).toHaveBeenCalledWith(
expect.objectContaining({
input: {
Bucket: 'fastgpt-private',
Key: 'dataset/team/file.txt'
}
}),
{ abortSignal: controller.signal }
);
});
it('destroys an in-flight body with the caller abort reason', async () => {
const adapter = createAdapter();
const body = new PassThrough();
(adapter as any).client.send = vi.fn().mockResolvedValue({ Body: body });
const controller = new AbortController();
const abortReason = new Error('client aborted');
const result = await adapter.downloadObject({
key: 'dataset/team/file.txt',
abortSignal: controller.signal
});
result.body.on('error', () => {});
controller.abort(abortReason);
expect(result.body.errored).toBe(abortReason);
expect(result.body.destroyed).toBe(true);
});
});
describe('AwsS3StorageAdapter.deleteObjectsByPrefix', () => {
it('rejects a whitespace-only prefix without calling S3', async () => {
const adapter = createAdapter();
const send = vi.fn();
(adapter as any).client.send = send;
await expect(adapter.deleteObjectsByPrefix({ prefix: ' ' })).rejects.toThrow(
'Prefix is required'
);
expect(send).not.toHaveBeenCalled();
});
it('preserves failures collected before a later listing page is empty', async () => {
const adapter = createAdapter();
const send = vi
.fn()
.mockResolvedValueOnce({
Contents: [{ Key: 'dataset/failed.txt' }],
IsTruncated: true,
NextContinuationToken: 'next-page'
})
.mockResolvedValueOnce({ Errors: [{ Key: 'dataset/failed.txt' }] })
.mockResolvedValueOnce({ Contents: [], IsTruncated: false });
(adapter as any).client.send = send;
await expect(adapter.deleteObjectsByPrefix({ prefix: 'dataset/' })).resolves.toEqual({
bucket: 'fastgpt-private',
keys: ['dataset/failed.txt']
});
});
});
describe('AwsS3StorageAdapter.generatePublicGetUrl', () => {
it.each([
[
{ forcePathStyle: true, publicAccessExtraSubPath: undefined },
'https://storage.example.com/fastgpt-private/folder%20name/file%20%23%2B.txt'
],
[
{ forcePathStyle: true, publicAccessExtraSubPath: '/proxy/' },
'https://storage.example.com/proxy/fastgpt-private/folder%20name/file%20%23%2B.txt'
],
[
{ forcePathStyle: false, publicAccessExtraSubPath: undefined },
'https://fastgpt-private.storage.example.com/folder%20name/file%20%23%2B.txt'
],
[
{ forcePathStyle: false, publicAccessExtraSubPath: '/proxy/' },
'https://fastgpt-private.storage.example.com/proxy/folder%20name/file%20%23%2B.txt'
]
])('encodes keys for options %j', (overrides, expectedUrl) => {
const adapter = new AwsS3StorageAdapter({
vendor: 'aws-s3',
bucket: 'fastgpt-private',
endpoint: 'https://storage.example.com',
region: 'us-east-1',
credentials: {
accessKeyId: 'access-key',
secretAccessKey: 'secret-key'
},
...overrides
});
expect(adapter.generatePublicGetUrl({ key: 'folder name/file #+.txt' }).url).toBe(expectedUrl);
});
});
import { beforeEach, describe, expect, it, vi } from 'vitest'; import { beforeEach, describe, expect, it, vi } from 'vitest';
import { CosStorageAdapter } from '../../../../sdk/storage/src/adapters/cos.adapter'; import { CosStorageAdapter } from '../../../src/adapters/cos.adapter';
const createAdapter = () => const createAdapter = () =>
new CosStorageAdapter({ new CosStorageAdapter({
...@@ -52,6 +52,22 @@ describe('CosStorageAdapter.generatePresignedGetUrl', () => { ...@@ -52,6 +52,22 @@ describe('CosStorageAdapter.generatePresignedGetUrl', () => {
}); });
describe('CosStorageAdapter.downloadObject', () => { describe('CosStorageAdapter.downloadObject', () => {
it('rejects a pre-aborted download without requesting the object', async () => {
const adapter = createAdapter();
const getObject = vi.fn();
(adapter as any).client.getObject = getObject;
const controller = new AbortController();
controller.abort();
await expect(
adapter.downloadObject({
key: 'dataset/team/file.txt',
abortSignal: controller.signal
})
).rejects.toMatchObject({ name: 'AbortError' });
expect(getObject).not.toHaveBeenCalled();
});
it('destroys the output stream when the caller aborts the download', async () => { it('destroys the output stream when the caller aborts the download', async () => {
const adapter = createAdapter(); const adapter = createAdapter();
(adapter as any).client.getObject = vi.fn(); (adapter as any).client.getObject = vi.fn();
...@@ -61,8 +77,57 @@ describe('CosStorageAdapter.downloadObject', () => { ...@@ -61,8 +77,57 @@ describe('CosStorageAdapter.downloadObject', () => {
key: 'dataset/team/file.txt', key: 'dataset/team/file.txt',
abortSignal: controller.signal abortSignal: controller.signal
}); });
controller.abort(new Error('client aborted')); const abortReason = new Error('client aborted');
body.on('error', () => {});
controller.abort(abortReason);
expect(body.errored).toBe(abortReason);
expect(body.destroyed).toBe(true); expect(body.destroyed).toBe(true);
}); });
}); });
describe('CosStorageAdapter deletion boundaries', () => {
it('treats an empty key list as a no-op', async () => {
const adapter = createAdapter();
const deleteMultipleObject = vi.fn();
(adapter as any).client.deleteMultipleObject = deleteMultipleObject;
await expect(adapter.deleteObjectsByMultiKeys({ keys: [] })).resolves.toEqual({
bucket: 'fastgpt-private',
keys: []
});
expect(deleteMultipleObject).not.toHaveBeenCalled();
});
it('rejects a whitespace-only prefix without listing objects', async () => {
const adapter = createAdapter();
const getBucket = vi.fn();
(adapter as any).client.getBucket = getBucket;
await expect(adapter.deleteObjectsByPrefix({ prefix: ' ' })).rejects.toThrow(
'Prefix is required'
);
expect(getBucket).not.toHaveBeenCalled();
});
});
describe('CosStorageAdapter.generatePublicGetUrl', () => {
it.each([
[undefined, 'https://fastgpt-private.cos.ap-guangzhou.myqcloud.com/folder%20%23/file%2B.txt'],
['cdn.example.com', 'https://cdn.example.com/folder%20%23/file%2B.txt']
])('encodes keys with domain %j', (domain, expectedUrl) => {
const adapter = new CosStorageAdapter({
vendor: 'cos',
bucket: 'fastgpt-private',
region: 'ap-guangzhou',
protocol: 'https:',
domain,
credentials: {
accessKeyId: 'secret-id',
secretAccessKey: 'secret-key'
}
});
expect(adapter.generatePublicGetUrl({ key: 'folder #/file+.txt' }).url).toBe(expectedUrl);
});
});
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
createMinioTimeoutTransport,
MinioS3NotFound,
MinioStorageAdapter
} from '../../../src/adapters/minio.adapter';
describe('createMinioTimeoutTransport', () => {
afterEach(() => {
vi.useRealTimers();
});
it('destroys the underlying request when the request timeout expires', () => {
vi.useFakeTimers();
const destroy = vi.fn();
const request = {
once: vi.fn(),
destroy
};
const transport = {
request: vi.fn(() => request)
};
const timeoutTransport = createMinioTimeoutTransport({
transport: transport as any,
timeoutMs: 1234
});
expect(timeoutTransport.request({} as any)).toBe(request);
vi.advanceTimersByTime(1234);
expect(destroy).toHaveBeenCalledOnce();
expect(destroy.mock.calls[0]?.[0]).toEqual(
expect.objectContaining({ message: 'MinIO request timeout after 1234ms' })
);
});
it('clears the timeout when the request closes', () => {
vi.useFakeTimers();
const destroy = vi.fn();
const request = {
once: vi.fn(),
destroy
};
const timeoutTransport = createMinioTimeoutTransport({
transport: { request: vi.fn(() => request) } as any,
timeoutMs: 1234
});
timeoutTransport.request({} as any);
expect(request.once).toHaveBeenCalledWith('close', expect.any(Function));
request.once.mock.calls[0]?.[1]();
vi.advanceTimersByTime(1234);
expect(destroy).not.toHaveBeenCalled();
});
});
const createAdapter = () =>
new MinioStorageAdapter({
vendor: 'minio',
bucket: 'fastgpt-private',
endpoint: 'http://localhost:9000',
region: 'us-east-1',
forcePathStyle: true,
maxRetries: 1,
credentials: {
accessKeyId: 'access-key',
secretAccessKey: 'secret-key'
}
});
describe('MinioStorageAdapter.constructor', () => {
it('rejects a non-MinIO vendor', () => {
expect(
() =>
new MinioStorageAdapter({
vendor: 'aws-s3',
bucket: 'fastgpt-private',
endpoint: 'http://localhost',
region: 'us-east-1',
credentials: {
accessKeyId: 'access-key',
secretAccessKey: 'secret-key'
}
})
).toThrow('Invalid storage vendor: expected "minio"');
});
it.each(['http://localhost', 'https://localhost'])(
'accepts an endpoint without an explicit port: %s',
(endpoint) => {
expect(
new MinioStorageAdapter({
vendor: 'minio',
bucket: 'fastgpt-private',
endpoint,
region: 'us-east-1',
credentials: {
accessKeyId: 'access-key',
secretAccessKey: 'secret-key'
}
})
).toBeInstanceOf(MinioStorageAdapter);
}
);
});
describe('MinioStorageAdapter.deleteObject', () => {
it('deletes one object with the MinIO client', async () => {
const adapter = createAdapter();
const removeObject = vi.fn().mockResolvedValue(undefined);
(adapter as any).minioClient.removeObject = removeObject;
await expect(adapter.deleteObject({ key: 'dataset/file.txt' })).resolves.toEqual({
bucket: 'fastgpt-private',
key: 'dataset/file.txt'
});
expect(removeObject).toHaveBeenCalledWith('fastgpt-private', 'dataset/file.txt');
});
});
describe('MinioStorageAdapter.deleteObjectsByMultiKeys', () => {
it('returns without calling MinIO for an empty key list', async () => {
const adapter = createAdapter();
const removeObjects = vi.fn().mockResolvedValue(undefined);
(adapter as any).minioClient.removeObjects = removeObjects;
await expect(adapter.deleteObjectsByMultiKeys({ keys: [] })).resolves.toEqual({
bucket: 'fastgpt-private',
keys: []
});
expect(removeObjects).not.toHaveBeenCalled();
});
it('splits deletion into batches of at most 1000 keys', async () => {
const adapter = createAdapter();
const removeObjects = vi
.fn()
.mockResolvedValueOnce([{ Key: 'dataset/file-3.txt' }])
.mockResolvedValueOnce([{ Error: { Key: 'dataset/file-1000.txt' } }]);
const keys = Array.from({ length: 1001 }, (_, index) => `dataset/file-${index}.txt`);
(adapter as any).minioClient.removeObjects = removeObjects;
await expect(adapter.deleteObjectsByMultiKeys({ keys })).resolves.toEqual({
bucket: 'fastgpt-private',
keys: ['dataset/file-3.txt', 'dataset/file-1000.txt']
});
expect(removeObjects).toHaveBeenNthCalledWith(1, 'fastgpt-private', keys.slice(0, 1000));
expect(removeObjects).toHaveBeenNthCalledWith(2, 'fastgpt-private', keys.slice(1000));
});
it.each([
['an error without a key', [{ Code: 'AccessDenied', Message: 'permission denied' }]],
['an error for an unexpected key', [{ Key: 'another-prefix/file.txt' }]],
['a non-array response', undefined]
])('marks the whole batch as failed for %s', async (_case, response) => {
const adapter = createAdapter();
const keys = ['dataset/first.txt', 'dataset/second.txt'];
(adapter as any).minioClient.removeObjects = vi.fn().mockResolvedValue(response);
await expect(adapter.deleteObjectsByMultiKeys({ keys })).resolves.toEqual({
bucket: 'fastgpt-private',
keys
});
});
});
describe('MinioStorageAdapter.deleteObjectsByPrefix', () => {
let adapter: MinioStorageAdapter;
let listObjects: ReturnType<typeof vi.fn>;
let removeObjects: ReturnType<typeof vi.fn>;
beforeEach(() => {
adapter = createAdapter();
listObjects = vi.fn();
removeObjects = vi.fn().mockResolvedValue([]);
(adapter as any).client.send = listObjects;
(adapter as any).minioClient.listObjectsV2 = vi.fn(() => {
throw new Error('Entity expansion limit exceeded: 1002 > 1000');
});
(adapter as any).minioClient.removeObjects = removeObjects;
});
afterEach(() => {
vi.useRealTimers();
});
it.each(['', ' '])('rejects an empty prefix: %j', async (prefix) => {
await expect(adapter.deleteObjectsByPrefix({ prefix })).rejects.toThrow('Prefix is required');
expect(listObjects).not.toHaveBeenCalled();
expect(removeObjects).not.toHaveBeenCalled();
});
it('lists 501 encoded objects in bounded pages and deletes each page sequentially', async () => {
const firstPageKeys = Array.from(
{ length: 400 },
(_, index) => `dataset/team%20%26%20one/file-${index}.txt`
);
const secondPageKeys = Array.from(
{ length: 101 },
(_, index) => `dataset/team%20%26%20one/file-${index + 400}.txt`
);
let resolveFirstDelete: ((value: []) => void) | undefined;
const firstDelete = new Promise<[]>((resolve) => {
resolveFirstDelete = resolve;
});
listObjects
.mockResolvedValueOnce({
Contents: firstPageKeys.map((Key) => ({ Key })),
IsTruncated: true,
NextContinuationToken: 'next-page'
})
.mockResolvedValueOnce({
Contents: secondPageKeys.map((Key) => ({ Key })),
IsTruncated: false
});
removeObjects.mockReturnValueOnce(firstDelete).mockResolvedValueOnce([]);
const resultPromise = adapter.deleteObjectsByPrefix({ prefix: 'dataset/team & one/' });
await vi.waitFor(() => expect(removeObjects).toHaveBeenCalledTimes(1));
expect(listObjects).toHaveBeenCalledTimes(1);
expect(listObjects.mock.calls[0]?.[0]).toEqual(
expect.objectContaining({
input: {
Bucket: 'fastgpt-private',
Prefix: 'dataset/team & one/',
ContinuationToken: undefined,
EncodingType: 'url',
MaxKeys: 400
}
})
);
expect(removeObjects).toHaveBeenNthCalledWith(
1,
'fastgpt-private',
firstPageKeys.map((key) => decodeURIComponent(key))
);
resolveFirstDelete?.([]);
await expect(resultPromise).resolves.toEqual({ bucket: 'fastgpt-private', keys: [] });
expect(listObjects).toHaveBeenCalledTimes(2);
expect(listObjects.mock.calls[1]?.[0]).toEqual(
expect.objectContaining({
input: expect.objectContaining({
ContinuationToken: 'next-page'
})
})
);
expect(removeObjects).toHaveBeenNthCalledWith(
2,
'fastgpt-private',
secondPageKeys.map((key) => decodeURIComponent(key))
);
expect((adapter as any).minioClient.listObjectsV2).not.toHaveBeenCalled();
});
it('collects per-object failures and continues deleting later pages', async () => {
listObjects
.mockResolvedValueOnce({
Contents: [{ Key: 'parsed/first.txt' }, { Key: 'parsed/file+with+spaces.txt' }],
IsTruncated: true,
NextContinuationToken: 'next-page'
})
.mockResolvedValueOnce({
Contents: [{ Key: 'parsed/last.txt' }, {}],
IsTruncated: false
});
removeObjects
.mockResolvedValueOnce([
{ Key: 'parsed/first.txt' },
{ Error: { Key: 'parsed/file with spaces.txt' } }
])
.mockResolvedValueOnce([]);
await expect(adapter.deleteObjectsByPrefix({ prefix: 'parsed/' })).resolves.toEqual({
bucket: 'fastgpt-private',
keys: ['parsed/first.txt', 'parsed/file with spaces.txt']
});
expect(removeObjects).toHaveBeenNthCalledWith(2, 'fastgpt-private', ['parsed/last.txt']);
});
it('marks the whole page as failed when the delete request rejects', async () => {
listObjects.mockResolvedValueOnce({
Contents: [{ Key: 'failed/first.txt' }, { Key: 'failed/second.txt' }],
IsTruncated: false
});
removeObjects.mockRejectedValueOnce(new Error('MinIO request timeout after 60000ms'));
await expect(adapter.deleteObjectsByPrefix({ prefix: 'failed/' })).resolves.toEqual({
bucket: 'fastgpt-private',
keys: ['failed/first.txt', 'failed/second.txt']
});
});
it('marks the whole page as failed when an error item has no key', async () => {
listObjects.mockResolvedValueOnce({
Contents: [{ Key: 'failed/first.txt' }, { Key: 'failed/second.txt' }],
IsTruncated: false
});
removeObjects.mockResolvedValueOnce([{ Code: 'AccessDenied', Message: 'permission denied' }]);
await expect(adapter.deleteObjectsByPrefix({ prefix: 'failed/' })).resolves.toEqual({
bucket: 'fastgpt-private',
keys: ['failed/first.txt', 'failed/second.txt']
});
});
it('skips deletion for an empty result page', async () => {
listObjects.mockResolvedValueOnce({ Contents: [], IsTruncated: false });
await expect(adapter.deleteObjectsByPrefix({ prefix: 'empty/' })).resolves.toEqual({
bucket: 'fastgpt-private',
keys: []
});
expect(removeObjects).not.toHaveBeenCalled();
});
it('rejects a truncated response without a continuation token', async () => {
listObjects.mockResolvedValueOnce({ Contents: [], IsTruncated: true });
await expect(adapter.deleteObjectsByPrefix({ prefix: 'parsed/' })).rejects.toThrow(
'Invalid MinIO list response: missing continuation token'
);
expect(removeObjects).not.toHaveBeenCalled();
});
it('aborts and rejects when listing exceeds the request timeout', async () => {
vi.useFakeTimers();
listObjects.mockReturnValue(new Promise(() => {}));
const resultPromise = adapter.deleteObjectsByPrefix({ prefix: 'stuck-list/' });
const assertion = expect(resultPromise).rejects.toThrow(
'Delete by prefix list timeout after 60000ms: stuck-list/'
);
await vi.advanceTimersByTimeAsync(60000);
await assertion;
expect(listObjects.mock.calls[0]?.[1]?.abortSignal.aborted).toBe(true);
expect(removeObjects).not.toHaveBeenCalled();
});
it('keeps the timeout error stable when abort immediately rejects the list request', async () => {
vi.useFakeTimers();
listObjects.mockImplementation((_command, { abortSignal }) => {
return new Promise((_resolve, reject) => {
abortSignal.addEventListener('abort', () => reject(abortSignal.reason), { once: true });
});
});
const resultPromise = adapter.deleteObjectsByPrefix({ prefix: 'stuck-list/' });
const assertion = expect(resultPromise).rejects.toThrow(
'Delete by prefix list timeout after 60000ms: stuck-list/'
);
await vi.advanceTimersByTimeAsync(60000);
await assertion;
expect(removeObjects).not.toHaveBeenCalled();
});
});
describe('MinioStorageAdapter.ensureBucket', () => {
it('returns the inherited result when the bucket exists', async () => {
const adapter = createAdapter();
const send = vi.fn().mockResolvedValue(undefined);
(adapter as any).client.send = send;
await expect(adapter.ensureBucket()).resolves.toEqual({
exists: true,
created: false,
bucket: 'fastgpt-private'
});
expect(send).toHaveBeenCalledTimes(1);
});
it('creates the bucket when the head request returns NotFound', async () => {
const adapter = createAdapter();
const send = vi
.fn()
.mockRejectedValueOnce(new MinioS3NotFound({ $metadata: {}, message: 'not found' }))
.mockResolvedValueOnce(undefined);
(adapter as any).client.send = send;
await expect(adapter.ensureBucket()).resolves.toEqual({
exists: false,
created: true,
bucket: 'fastgpt-private'
});
expect(send).toHaveBeenCalledTimes(2);
expect(send.mock.calls[1]?.[0]).toEqual(
expect.objectContaining({
input: { Bucket: 'fastgpt-private' }
})
);
});
it('preserves errors other than NotFound', async () => {
const adapter = createAdapter();
const error = new Error('permission denied');
(adapter as any).client.send = vi.fn().mockRejectedValue(error);
await expect(adapter.ensureBucket()).rejects.toBe(error);
});
});
describe('MinioStorageAdapter.ensurePublicBucketPolicy', () => {
it('applies a public object-read policy', async () => {
const adapter = createAdapter();
const send = vi.fn().mockResolvedValue(undefined);
(adapter as any).client.send = send;
await adapter.ensurePublicBucketPolicy();
const command = send.mock.calls[0]?.[0];
expect(command).toEqual(
expect.objectContaining({
input: expect.objectContaining({
Bucket: 'fastgpt-private'
})
})
);
expect(JSON.parse(command.input.Policy)).toEqual(
expect.objectContaining({
Statement: [
expect.objectContaining({
Action: ['s3:GetObject'],
Resource: ['arn:aws:s3:::fastgpt-private/*']
})
]
})
);
});
});
describe('MinioStorageAdapter.removeBucketLifecycle', () => {
it('removes the lifecycle with the AWS client', async () => {
const adapter = createAdapter();
const send = vi.fn().mockResolvedValue(undefined);
(adapter as any).client.send = send;
await adapter.removeBucketLifecycle();
expect(send.mock.calls[0]?.[0]).toEqual(
expect.objectContaining({
input: { Bucket: 'fastgpt-private' }
})
);
});
});
import { PassThrough } from 'node:stream';
import { describe, expect, it, vi } from 'vitest';
import { OssStorageAdapter } from '../../../src/adapters/oss.adapter';
const createAdapter = () =>
new OssStorageAdapter({
vendor: 'oss',
bucket: 'fastgpt-private',
endpoint: 'http://localhost:9000',
region: 'oss-cn-hangzhou',
secure: false,
credentials: {
accessKeyId: 'access-key',
secretAccessKey: 'secret-key'
}
});
describe('OssStorageAdapter.downloadObject', () => {
it('rejects a pre-aborted download without requesting a stream', async () => {
const adapter = createAdapter();
const getStream = vi.fn();
(adapter as any).client.getStream = getStream;
const controller = new AbortController();
controller.abort();
await expect(
adapter.downloadObject({ key: 'dataset/file.txt', abortSignal: controller.signal })
).rejects.toMatchObject({ name: 'AbortError' });
expect(getStream).not.toHaveBeenCalled();
});
it('destroys an in-flight stream with the caller abort reason', async () => {
const adapter = createAdapter();
const stream = new PassThrough();
(adapter as any).client.getStream = vi.fn().mockResolvedValue({ stream });
const controller = new AbortController();
const abortReason = new Error('client aborted');
const result = await adapter.downloadObject({
key: 'dataset/file.txt',
abortSignal: controller.signal
});
result.body.on('error', () => {});
controller.abort(abortReason);
expect(result.body.errored).toBe(abortReason);
expect(result.body.destroyed).toBe(true);
});
});
describe('OssStorageAdapter deletion boundaries', () => {
it('treats an empty key list as a no-op', async () => {
const adapter = createAdapter();
const deleteMulti = vi.fn();
(adapter as any).client.deleteMulti = deleteMulti;
await expect(adapter.deleteObjectsByMultiKeys({ keys: [] })).resolves.toEqual({
bucket: 'fastgpt-private',
keys: []
});
expect(deleteMulti).not.toHaveBeenCalled();
});
it('returns only keys missing from the verbose delete response as failures', async () => {
const adapter = createAdapter();
const deleteMulti = vi.fn().mockResolvedValue({ deleted: ['first.txt'] });
(adapter as any).client.deleteMulti = deleteMulti;
await expect(
adapter.deleteObjectsByMultiKeys({ keys: ['first.txt', 'second.txt'] })
).resolves.toEqual({
bucket: 'fastgpt-private',
keys: ['second.txt']
});
expect(deleteMulti).toHaveBeenCalledWith(['first.txt', 'second.txt'], { quiet: false });
});
it('normalizes the object-shaped Deleted entries produced by ali-oss XML parsing', async () => {
const adapter = createAdapter();
(adapter as any).client.deleteMulti = vi.fn().mockResolvedValue({
deleted: [{ Key: 'first.txt' }, { Key: 'second.txt', VersionId: 'version-1' }]
});
await expect(
adapter.deleteObjectsByMultiKeys({ keys: ['first.txt', 'second.txt'] })
).resolves.toEqual({
bucket: 'fastgpt-private',
keys: []
});
});
it('conservatively returns every key when the verbose response omits deleted entries', async () => {
const adapter = createAdapter();
(adapter as any).client.deleteMulti = vi.fn().mockResolvedValue({});
await expect(
adapter.deleteObjectsByMultiKeys({ keys: ['first.txt', 'second.txt'] })
).resolves.toEqual({
bucket: 'fastgpt-private',
keys: ['first.txt', 'second.txt']
});
});
it('preserves failures collected before a later listing page is empty', async () => {
const adapter = createAdapter();
(adapter as any).client.list = vi
.fn()
.mockResolvedValueOnce({
objects: [{ name: 'dataset/failed.txt' }],
isTruncated: true,
nextMarker: 'dataset/next.txt'
})
.mockResolvedValueOnce({ objects: [], isTruncated: false });
(adapter as any).client.deleteMulti = vi.fn().mockResolvedValue({ deleted: [] });
await expect(adapter.deleteObjectsByPrefix({ prefix: 'dataset/' })).resolves.toEqual({
bucket: 'fastgpt-private',
keys: ['dataset/failed.txt']
});
});
it('rejects a whitespace-only prefix without listing objects', async () => {
const adapter = createAdapter();
const list = vi.fn();
(adapter as any).client.list = list;
await expect(adapter.deleteObjectsByPrefix({ prefix: ' ' })).rejects.toThrow(
'Prefix is required'
);
expect(list).not.toHaveBeenCalled();
});
});
describe('OssStorageAdapter.generatePublicGetUrl', () => {
it.each([
[
false,
undefined,
'https://fastgpt-private.oss-cn-hangzhou.aliyuncs.com/folder%20%23/file%2B.txt'
],
[true, 'cdn.example.com', 'https://cdn.example.com/folder%20%23/file%2B.txt']
])('encodes keys with cname=%s', (cname, endpoint, expectedUrl) => {
const adapter = new OssStorageAdapter({
vendor: 'oss',
bucket: 'fastgpt-private',
endpoint,
region: 'oss-cn-hangzhou',
secure: true,
cname,
credentials: {
accessKeyId: 'access-key',
secretAccessKey: 'secret-key'
}
});
expect(adapter.generatePublicGetUrl({ key: 'folder #/file+.txt' }).url).toBe(expectedUrl);
});
});
import { describe, expect, it, vi } from 'vitest';
import type { IStorage } from '../../src/interface';
import { AwsS3StorageAdapter } from '../../src/adapters/aws-s3.adapter';
import { CosStorageAdapter } from '../../src/adapters/cos.adapter';
import { MinioStorageAdapter } from '../../src/adapters/minio.adapter';
import { OssStorageAdapter } from '../../src/adapters/oss.adapter';
import { InvalidStorageObjectKeyError } from '../../src/errors';
import { createVitestStorageMock } from '../../src/helper/mock';
type GuardedStorage = {
storage: IStorage;
remoteCall: ReturnType<typeof vi.fn>;
};
const unexpectedRemoteCall = () => {
throw new Error('Object key was not validated before the remote SDK call');
};
const createAwsStorage = (): GuardedStorage => {
const storage = new AwsS3StorageAdapter({
vendor: 'aws-s3',
bucket: 'test-bucket',
endpoint: 'http://127.0.0.1:1',
region: 'us-east-1',
forcePathStyle: true,
maxRetries: 1,
credentials: { accessKeyId: 'access-key', secretAccessKey: 'secret-key' }
});
const remoteCall = vi.fn(unexpectedRemoteCall);
(storage as any).client.send = remoteCall;
return { storage, remoteCall };
};
const createMinioStorage = (): GuardedStorage => {
const storage = new MinioStorageAdapter({
vendor: 'minio',
bucket: 'test-bucket',
endpoint: 'http://127.0.0.1:1',
region: 'us-east-1',
forcePathStyle: true,
maxRetries: 1,
credentials: { accessKeyId: 'access-key', secretAccessKey: 'secret-key' }
});
const remoteCall = vi.fn(unexpectedRemoteCall);
(storage as any).client.send = remoteCall;
(storage as any).minioClient.removeObject = remoteCall;
(storage as any).minioClient.removeObjects = remoteCall;
return { storage, remoteCall };
};
const createOssStorage = (): GuardedStorage => {
const storage = new OssStorageAdapter({
vendor: 'oss',
bucket: 'test-bucket',
endpoint: 'http://127.0.0.1:1',
region: 'oss-cn-hangzhou',
secure: false,
credentials: { accessKeyId: 'access-key', secretAccessKey: 'secret-key' }
});
const remoteCall = vi.fn(unexpectedRemoteCall);
Object.assign((storage as any).client, {
head: remoteCall,
put: remoteCall,
getStream: remoteCall,
delete: remoteCall,
deleteMulti: remoteCall,
list: remoteCall,
signatureUrlV4: remoteCall,
signatureUrl: remoteCall,
copy: remoteCall
});
return { storage, remoteCall };
};
const createCosStorage = (): GuardedStorage => {
const storage = new CosStorageAdapter({
vendor: 'cos',
bucket: 'test-bucket',
region: 'ap-guangzhou',
credentials: { accessKeyId: 'access-key', secretAccessKey: 'secret-key' }
});
const remoteCall = vi.fn(unexpectedRemoteCall);
Object.assign((storage as any).client, {
headObject: remoteCall,
putObject: remoteCall,
getObject: remoteCall,
deleteObject: remoteCall,
deleteMultipleObject: remoteCall,
getBucket: remoteCall,
getObjectUrl: remoteCall,
sliceCopyFile: remoteCall
});
return { storage, remoteCall };
};
const createMockStorage = (): GuardedStorage => ({
storage: createVitestStorageMock({ vi }),
remoteCall: vi.fn()
});
const storageFactories = [
['AWS S3', createAwsStorage],
['MinIO', createMinioStorage],
['OSS', createOssStorage],
['COS', createCosStorage],
['Vitest mock', createMockStorage]
] as const;
const invalidKey = 'invalid//key';
const keyOperations: ReadonlyArray<
[name: string, field: string, operation: (storage: IStorage) => unknown]
> = [
['checkObjectExists', 'key', (storage) => storage.checkObjectExists({ key: invalidKey })],
['getObjectMetadata', 'key', (storage) => storage.getObjectMetadata({ key: invalidKey })],
['uploadObject', 'key', (storage) => storage.uploadObject({ key: invalidKey, body: 'body' })],
['downloadObject', 'key', (storage) => storage.downloadObject({ key: invalidKey })],
['deleteObject', 'key', (storage) => storage.deleteObject({ key: invalidKey })],
[
'deleteObjectsByMultiKeys',
'keys[1]',
(storage) => storage.deleteObjectsByMultiKeys({ keys: ['valid/key', invalidKey] })
],
[
'deleteObjectsByPrefix',
'prefix',
(storage) => storage.deleteObjectsByPrefix({ prefix: invalidKey })
],
[
'generatePresignedPutUrl',
'key',
(storage) => storage.generatePresignedPutUrl({ key: invalidKey })
],
[
'generatePresignedGetUrl',
'key',
(storage) => storage.generatePresignedGetUrl({ key: invalidKey })
],
['generatePublicGetUrl', 'key', (storage) => storage.generatePublicGetUrl({ key: invalidKey })],
['listObjects', 'prefix', (storage) => storage.listObjects({ prefix: invalidKey })],
[
'copyObjectInSelfBucket source',
'sourceKey',
(storage) =>
storage.copyObjectInSelfBucket({ sourceKey: invalidKey, targetKey: 'valid/target' })
],
[
'copyObjectInSelfBucket target',
'targetKey',
(storage) =>
storage.copyObjectInSelfBucket({ sourceKey: 'valid/source', targetKey: invalidKey })
]
];
type KeyContractCase = readonly [
storageName: string,
operationName: string,
field: string,
createStorage: () => GuardedStorage,
operation: (storage: IStorage) => unknown
];
const keyContractCases: KeyContractCase[] = storageFactories.flatMap(
([storageName, createStorage]) =>
keyOperations.map<KeyContractCase>(([operationName, field, operation]) => [
storageName,
operationName,
field,
createStorage,
operation
])
);
describe('IStorage object key preflight contract', () => {
it.each(keyContractCases)(
'%s validates %s before dispatch',
async (_storageName, _operationName, field, createStorage, operation) => {
const { storage, remoteCall } = createStorage();
await expect(Promise.resolve().then(() => operation(storage))).rejects.toMatchObject({
name: InvalidStorageObjectKeyError.name,
field,
reason: 'empty_path_segment'
});
expect(remoteCall).not.toHaveBeenCalled();
}
);
it.each([
['AWS S3', createAwsStorage],
['MinIO', createMinioStorage]
] as const)(
'%s validates every key before deleting the first 1000-key batch',
async (_, createStorage) => {
const { storage, remoteCall } = createStorage();
const keys = Array.from({ length: 1000 }, (_, index) => `valid/${index}`).concat(invalidKey);
await expect(storage.deleteObjectsByMultiKeys({ keys })).rejects.toBeInstanceOf(
InvalidStorageObjectKeyError
);
expect(remoteCall).not.toHaveBeenCalled();
}
);
});
import { describe, expect, it } from 'vitest';
import { InvalidStorageObjectKeyError } from '../../src/errors';
import {
MAX_STORAGE_OBJECT_KEY_UTF8_BYTES,
assertRequiredStorageObjectPrefix,
assertStorageObjectKey,
assertStorageObjectKeys,
assertStorageObjectPrefix
} from '../../src/assert';
const expectInvalidKey = ({
value,
reason,
field = 'key'
}: {
value: unknown;
reason: InvalidStorageObjectKeyError['reason'];
field?: string;
}) => {
try {
assertStorageObjectKey(value, field);
throw new Error('Expected key validation to fail');
} catch (error) {
expect(error).toBeInstanceOf(InvalidStorageObjectKeyError);
expect(error).toMatchObject({ reason, field });
}
};
describe('storage object key validation', () => {
it.each([
'folder name/+ # & % ?/\u6587\u4ef6-\ud83d\ude00.txt',
'folder/.hidden',
'folder/..backup',
'folder/trailing/'
])('accepts portable URL-sensitive and Unicode key %j', (key) => {
expect(() => assertStorageObjectKey(key)).not.toThrow();
});
it('accepts exactly 850 UTF-8 bytes, including multibyte characters', () => {
const asciiKey = 'a'.repeat(MAX_STORAGE_OBJECT_KEY_UTF8_BYTES);
const unicodeKey = `${'\u4e2d'.repeat(282)}abcd`;
expect(Buffer.byteLength(asciiKey)).toBe(850);
expect(Buffer.byteLength(unicodeKey)).toBe(850);
expect(() => assertStorageObjectKey(asciiKey)).not.toThrow();
expect(() => assertStorageObjectKey(unicodeKey)).not.toThrow();
});
it('rejects 851 UTF-8 bytes even when the JavaScript string is shorter', () => {
const key = `${'\u4e2d'.repeat(283)}ab`;
expect(key.length).toBeLessThan(850);
expect(Buffer.byteLength(key)).toBe(851);
expectInvalidKey({ value: key, reason: 'too_long' });
});
it.each([
[undefined, 'invalid_type'],
[123, 'invalid_type'],
['', 'empty'],
['bad\ud800key', 'invalid_unicode'],
['bad\udcffkey', 'invalid_unicode'],
['/leading', 'leading_slash'],
['folder\\file', 'backslash'],
['folder//file', 'empty_path_segment'],
['folder\0file', 'control_character'],
['folder\nfile', 'control_character'],
['folder\u007ffile', 'control_character']
] as const)('rejects non-portable key %j as %s', (value, reason) => {
expectInvalidKey({ value, reason });
});
it.each(['.', '..', './file', 'folder/./file', 'folder/../file', 'folder/ .. /file'])(
'rejects dot path segment in %j',
(key) => {
expectInvalidKey({ value: key, reason: 'dot_path_segment' });
}
);
it.each(['\u0018', '\u0019', '\u001a', '\u001b'])(
'rejects COS-incompatible control character U+%s',
(character) => {
expectInvalidKey({ value: `folder/${character}/file`, reason: 'control_character' });
}
);
it('reports the first invalid batch item and validates the full batch before callers mutate', () => {
try {
assertStorageObjectKeys(['valid/first', 'invalid//second', 'valid/third']);
throw new Error('Expected key validation to fail');
} catch (error) {
expect(error).toBeInstanceOf(InvalidStorageObjectKeyError);
expect(error).toMatchObject({
field: 'keys[1]',
reason: 'empty_path_segment'
});
}
});
it('rejects a non-array batch with the structured key error', () => {
try {
assertStorageObjectKeys('not-an-array');
throw new Error('Expected key validation to fail');
} catch (error) {
expect(error).toBeInstanceOf(InvalidStorageObjectKeyError);
expect(error).toMatchObject({ field: 'keys', reason: 'invalid_type' });
}
});
it('rejects sparse arrays instead of skipping missing entries', () => {
const keys = new Array<string>(2);
keys[1] = 'valid.txt';
try {
assertStorageObjectKeys(keys);
throw new Error('Expected key validation to fail');
} catch (error) {
expect(error).toBeInstanceOf(InvalidStorageObjectKeyError);
expect(error).toMatchObject({ field: 'keys[0]', reason: 'invalid_type' });
}
});
it('allows omitted and empty list prefixes but validates non-empty prefixes', () => {
expect(() => assertStorageObjectPrefix(undefined)).not.toThrow();
expect(() => assertStorageObjectPrefix('')).not.toThrow();
expect(() => assertStorageObjectPrefix('folder name/\u4e2d\u6587/')).not.toThrow();
try {
assertStorageObjectPrefix('invalid//prefix');
throw new Error('Expected prefix validation to fail');
} catch (error) {
expect(error).toBeInstanceOf(InvalidStorageObjectKeyError);
expect(error).toMatchObject({ field: 'prefix', reason: 'empty_path_segment' });
}
});
it.each(['', ' '])('rejects required delete prefix %j', (prefix) => {
expect(() => assertRequiredStorageObjectPrefix(prefix)).toThrow('Prefix is required');
});
it('accepts a valid required delete prefix', () => {
expect(() => assertRequiredStorageObjectPrefix('team/files/')).not.toThrow();
});
});
import { describe, expect, it } from 'vitest';
import { AwsS3StorageAdapter } from '../../src/adapters/aws-s3.adapter';
import { CosStorageAdapter } from '../../src/adapters/cos.adapter';
import { MinioStorageAdapter } from '../../src/adapters/minio.adapter';
import { OssStorageAdapter } from '../../src/adapters/oss.adapter';
import { createStorage } from '../../src/factory';
const credentials = {
accessKeyId: 'access-key',
secretAccessKey: 'secret-key'
};
describe('createStorage', () => {
it('creates every supported adapter from its discriminated options', () => {
expect(
createStorage({
vendor: 'aws-s3',
bucket: 'bucket',
endpoint: 'http://localhost:9000',
region: 'us-east-1',
credentials
})
).toBeInstanceOf(AwsS3StorageAdapter);
expect(
createStorage({
vendor: 'minio',
bucket: 'bucket',
endpoint: 'http://localhost:9000',
region: 'us-east-1',
credentials
})
).toBeInstanceOf(MinioStorageAdapter);
expect(
createStorage({
vendor: 'oss',
bucket: 'bucket',
region: 'oss-cn-hangzhou',
credentials
})
).toBeInstanceOf(OssStorageAdapter);
expect(
createStorage({
vendor: 'cos',
bucket: 'bucket',
region: 'ap-guangzhou',
credentials
})
).toBeInstanceOf(CosStorageAdapter);
});
it('rejects an unsupported runtime vendor', () => {
expect(() => createStorage({ vendor: 'unknown' } as never)).toThrow(
'Unsupported storage vendor: unknown'
);
});
});
import { describe, expect, it, vi } from 'vitest';
import { createVitestStorageMock } from '../../src/helper/mock';
import { removeIntegrationBucketIfExists } from '../integration/helpers';
describe('removeIntegrationBucketIfExists', () => {
it('does nothing when the stable test bucket does not exist', async () => {
const storage = createVitestStorageMock({ vi });
const bucketExists = vi.fn().mockResolvedValue(false);
const deleteBucket = vi.fn();
await removeIntegrationBucketIfExists({ storage, bucketExists, deleteBucket });
expect(storage.listObjects).not.toHaveBeenCalled();
expect(deleteBucket).not.toHaveBeenCalled();
});
it('deletes an existing empty bucket', async () => {
const storage = createVitestStorageMock({ vi });
const deleteBucket = vi.fn().mockResolvedValue(undefined);
await removeIntegrationBucketIfExists({
storage,
bucketExists: vi.fn().mockResolvedValue(true),
deleteBucket
});
expect(storage.listObjects).toHaveBeenCalledWith({});
expect(storage.deleteObjectsByMultiKeys).not.toHaveBeenCalled();
expect(deleteBucket).toHaveBeenCalledOnce();
});
it('clears every object before deleting an existing bucket', async () => {
const storage = createVitestStorageMock({ vi });
storage.__putObject('first.txt', { body: Buffer.from('first') });
storage.__putObject('nested/second.txt', { body: Buffer.from('second') });
const deleteBucket = vi.fn().mockResolvedValue(undefined);
await removeIntegrationBucketIfExists({
storage,
bucketExists: vi.fn().mockResolvedValue(true),
deleteBucket
});
expect(storage.deleteObjectsByMultiKeys).toHaveBeenCalledWith({
keys: ['first.txt', 'nested/second.txt']
});
expect(storage.__objects.size).toBe(0);
expect(deleteBucket).toHaveBeenCalledOnce();
});
it('keeps the bucket when any object deletion fails', async () => {
const storage = createVitestStorageMock({ vi });
storage.__putObject('failed.txt', { body: Buffer.from('failed') });
vi.spyOn(storage, 'deleteObjectsByMultiKeys').mockResolvedValue({
bucket: storage.bucketName,
keys: ['failed.txt']
});
const deleteBucket = vi.fn();
await expect(
removeIntegrationBucketIfExists({
storage,
bucketExists: vi.fn().mockResolvedValue(true),
deleteBucket
})
).rejects.toThrow('Failed to clean integration test bucket: failed.txt');
expect(deleteBucket).not.toHaveBeenCalled();
});
it('retries the same bucket on the next setup after bucket deletion fails', async () => {
const storage = createVitestStorageMock({ vi });
const bucketExists = vi.fn().mockResolvedValue(true);
const deleteBucket = vi
.fn()
.mockRejectedValueOnce(new Error('temporary delete failure'))
.mockResolvedValueOnce(undefined);
await expect(
removeIntegrationBucketIfExists({ storage, bucketExists, deleteBucket })
).rejects.toThrow('temporary delete failure');
await expect(
removeIntegrationBucketIfExists({ storage, bucketExists, deleteBucket })
).resolves.toBeUndefined();
expect(bucketExists).toHaveBeenCalledTimes(2);
expect(deleteBucket).toHaveBeenCalledTimes(2);
});
});
import { Readable } from 'node:stream';
import { describe, expect, it, vi } from 'vitest';
import { createVitestStorageMock } from '../../src/helper/mock';
const readBody = async (body: Readable) => {
const chunks: Buffer[] = [];
for await (const chunk of body) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
return Buffer.concat(chunks);
};
describe('createVitestStorageMock', () => {
it('implements bucket lifecycle, upload forms, download and metadata', async () => {
const storage = createVitestStorageMock({ vi, bucketName: 'test-bucket' });
await expect(storage.ensureBucket()).resolves.toEqual({
bucket: 'test-bucket',
exists: false,
created: true
});
await expect(storage.ensureBucket()).resolves.toEqual({
bucket: 'test-bucket',
exists: true,
created: false
});
await storage.uploadObject({
key: 'buffer.bin',
body: Buffer.from([0, 255]),
metadata: { source: 'buffer' }
});
await storage.uploadObject({ key: 'string.txt', body: 'string' });
await storage.uploadObject({ key: 'stream.txt', body: Readable.from(['stream']) });
const download = await storage.downloadObject({ key: 'buffer.bin' });
await expect(readBody(download.body)).resolves.toEqual(Buffer.from([0, 255]));
await expect(storage.getObjectMetadata({ key: 'buffer.bin' })).resolves.toMatchObject({
metadata: { source: 'buffer' },
contentLength: 2,
etag: expect.any(String)
});
await expect(storage.listObjects({})).resolves.toMatchObject({
keys: ['buffer.bin', 'stream.txt', 'string.txt']
});
});
it('returns failed keys rather than deleted keys from successful deletion methods', async () => {
const storage = createVitestStorageMock({ vi });
storage.__putObject('prefix/first.txt', { body: Buffer.from('first') });
storage.__putObject('prefix/second.txt', { body: Buffer.from('second') });
await expect(
storage.deleteObjectsByMultiKeys({ keys: ['prefix/first.txt', 'missing.txt'] })
).resolves.toEqual({ bucket: 'mock-bucket', keys: [] });
await expect(storage.deleteObjectsByPrefix({ prefix: 'prefix/' })).resolves.toEqual({
bucket: 'mock-bucket',
keys: []
});
expect(storage.__objects.size).toBe(0);
});
it('rejects empty prefixes and pre-aborted downloads', async () => {
const storage = createVitestStorageMock({ vi });
storage.__putObject('file.txt', { body: Buffer.from('file') });
await expect(storage.deleteObjectsByPrefix({ prefix: ' ' })).rejects.toThrow(
'Prefix is required'
);
const controller = new AbortController();
controller.abort();
await expect(
storage.downloadObject({ key: 'file.txt', abortSignal: controller.signal })
).rejects.toMatchObject({ name: 'AbortError' });
});
it('destroys an in-flight download with the caller abort reason', async () => {
const storage = createVitestStorageMock({ vi });
storage.__putObject('file.txt', { body: Buffer.from('file') });
const controller = new AbortController();
const abortReason = new Error('client aborted');
const { body } = await storage.downloadObject({
key: 'file.txt',
abortSignal: controller.signal
});
body.on('error', () => {});
controller.abort(abortReason);
expect(body.errored).toBe(abortReason);
expect(body.destroyed).toBe(true);
});
it('copies object buffers independently and resets state', async () => {
const storage = createVitestStorageMock({ vi });
storage.__putObject('source.txt', {
body: Buffer.from('source'),
metadata: { copied: 'true' }
});
await storage.copyObjectInSelfBucket({
sourceKey: 'source.txt',
targetKey: 'target.txt'
});
storage.__objects.get('source.txt')?.body.fill(0);
expect(storage.__objects.get('target.txt')?.body.toString()).toBe('source');
expect(storage.__objects.get('target.txt')?.metadata).toEqual({ copied: 'true' });
storage.__reset();
expect(storage.__objects.size).toBe(0);
});
it('encodes keys and response overrides in generated URLs', async () => {
const storage = createVitestStorageMock({ vi, baseUrl: 'https://storage.test' });
const key = 'folder name/file#+.txt';
await expect(storage.generatePresignedPutUrl({ key })).resolves.toMatchObject({
url: `https://storage.test/put/mock-bucket/${encodeURIComponent(key)}`
});
await expect(
storage.generatePresignedGetUrl({ key, responseContentType: 'text/plain; charset=utf-8' })
).resolves.toMatchObject({
url: expect.stringContaining('response-content-type=text%2Fplain%3B%20charset%3Dutf-8')
});
expect(storage.generatePublicGetUrl({ key }).url).toBe(
`https://storage.test/public/mock-bucket/${encodeURIComponent(key)}`
);
});
});
import { describe, expect, it } from 'vitest';
import { encodeObjectKeyPath } from '../../src/utils';
describe('encodeObjectKeyPath', () => {
it.each([
['', ''],
['folder/file.txt', 'folder/file.txt'],
['folder name/file #+&.txt', 'folder%20name/file%20%23%2B%26.txt'],
['literal/%2F/\u6587\u4ef6.txt', 'literal/%252F/%E6%96%87%E4%BB%B6.txt'],
['/leading//empty/', '/leading//empty/']
])('encodes %j as an object URL path', (key, expected) => {
expect(encodeObjectKeyPath(key)).toBe(expected);
});
});
{
"extends": "./tsconfig.json",
"compilerOptions": {
"allowJs": false,
"noEmit": true
},
"include": ["src/**/*.ts", "test/**/*.ts", "vitest.config.ts"]
}
import path from 'node:path';
import { defineConfig } from 'vitest/config';
export default defineConfig(() => {
return {
test: {
globalSetup: path.join(import.meta.dirname, 'test/global-setup.ts'),
include: ['test/**/*.test.ts'],
pool: 'threads',
fileParallelism: false,
testTimeout: 120_000,
hookTimeout: 120_000,
reporters: ['default']
}
};
});
import { vi } from 'vitest'; import { vi } from 'vitest';
import { createVitestStorageMock } from '../../../sdk/storage/src/testing/vitestMock'; import { createVitestStorageMock } from '../../../sdk/storage/src/helper/mock';
const mockStorageByBucket = new Map<string, ReturnType<typeof createVitestStorageMock>>(); const mockStorageByBucket = new Map<string, ReturnType<typeof createVitestStorageMock>>();
const getMockStorage = (bucketName: string) => { const getMockStorage = (bucketName: string) => {
......
import { Readable } from 'node:stream';
import { describe, expect, it, vi } from 'vitest';
import { AwsS3StorageAdapter } from '../../../../sdk/storage/src/adapters/aws-s3.adapter';
const createAdapter = () =>
new AwsS3StorageAdapter({
vendor: 'aws-s3',
bucket: 'fastgpt-private',
endpoint: 'http://localhost:9000',
region: 'us-east-1',
forcePathStyle: true,
maxRetries: 1,
credentials: {
accessKeyId: 'access-key',
secretAccessKey: 'secret-key'
}
});
describe('AwsS3StorageAdapter.downloadObject', () => {
it('passes the caller abort signal to the AWS request handler', async () => {
const adapter = createAdapter();
const body = Readable.from([Buffer.from('file')]);
const send = vi.fn().mockResolvedValue({ Body: body });
(adapter as any).client.send = send;
const controller = new AbortController();
const result = await adapter.downloadObject({
key: 'dataset/team/file.txt',
abortSignal: controller.signal
});
expect(result.body).toBe(body);
expect(send).toHaveBeenCalledWith(
expect.objectContaining({
input: {
Bucket: 'fastgpt-private',
Key: 'dataset/team/file.txt'
}
}),
{ abortSignal: controller.signal }
);
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment