Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit edd12c8a by YeYuheng Committed by GitHub
Options

fix: set charset for source file previews (#6916) 

* fix: set charset for source file previews

* fix: avoid charset on explicit binary source previews

---------

Co-authored-by: xqvvu <whoeverimf5@gmail.com>
parent 8029b133
Showing with 419 additions and 25 deletions
document/content/self-host/upgrading/4-15/4150.mdx
...@@ -70,6 +70,7 @@ FILE_TOKEN_KEY= ...@@ -70,6 +70,7 @@ FILE_TOKEN_KEY=
## 🐛 修复 ## 🐛 修复
1. 修复 Agent v2 模式下，模型响应报错会导致 step 重复执行 1. 修复 Agent v2 模式下，模型响应报错会导致 step 重复执行
2. 修复知识库源文件预览和下载时文本类型响应缺少 charset 的问题。
## 代码优化 ## 代码优化
......
document/data/doc-last-modified.json
...@@ -273,7 +273,7 @@ ...@@ -273,7 +273,7 @@
"content/self-host/upgrading/4-14/41481.mdx": "2026-04-26T21:08:47+08:00", "content/self-host/upgrading/4-14/41481.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/4149.en.mdx": "2026-04-26T21:08:47+08:00", "content/self-host/upgrading/4-14/4149.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/4149.mdx": "2026-04-26T21:08:47+08:00", "content/self-host/upgrading/4-14/4149.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-15/4150.mdx": "2026-05-09T16:13:01+08:00", "content/self-host/upgrading/4-15/4150.mdx": "2026-05-19T11:28:13+08:00",
"content/self-host/upgrading/4-15/41502.mdx": "2026-05-18T17:26:16+08:00", "content/self-host/upgrading/4-15/41502.mdx": "2026-05-18T17:26:16+08:00",
"content/self-host/upgrading/outdated/40.en.mdx": "2026-04-26T21:08:47+08:00", "content/self-host/upgrading/outdated/40.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/40.mdx": "2026-04-26T21:08:47+08:00", "content/self-host/upgrading/outdated/40.mdx": "2026-04-26T21:08:47+08:00",
......
packages/service/common/s3/buckets/base.ts
...@@ -236,7 +236,7 @@ export class S3BaseBucket { ...@@ -236,7 +236,7 @@ export class S3BaseBucket {
async createExternalUrl(params: createPreviewUrlParams) { async createExternalUrl(params: createPreviewUrlParams) {
const parsed = CreateGetPresignedUrlParamsSchema.parse(params); const parsed = CreateGetPresignedUrlParamsSchema.parse(params);
const { key, expiredHours, mode } = parsed; const { key, expiredHours, mode, responseContentType } = parsed;
const expires = expiredHours ? expiredHours * 60 * 60 : 30 * 60; // expires 的单位是秒 默认 30 分钟 const expires = expiredHours ? expiredHours * 60 * 60 : 30 * 60; // expires 的单位是秒 默认 30 分钟
if ((mode || storageDownloadMode) === 'proxy') { if ((mode || storageDownloadMode) === 'proxy') {
...@@ -254,7 +254,8 @@ export class S3BaseBucket { ...@@ -254,7 +254,8 @@ export class S3BaseBucket {
const result = await this.externalClient.generatePresignedGetUrl({ const result = await this.externalClient.generatePresignedGetUrl({
key, key,
expiredSeconds: expires expiredSeconds: expires,
...(responseContentType ? { responseContentType } : {})
}); });
return { return {
...@@ -266,10 +267,14 @@ export class S3BaseBucket { ...@@ -266,10 +267,14 @@ export class S3BaseBucket {
async createPreviewUrl(params: createPreviewUrlParams) { async createPreviewUrl(params: createPreviewUrlParams) {
const parsed = CreateGetPresignedUrlParamsSchema.parse(params); const parsed = CreateGetPresignedUrlParamsSchema.parse(params);
const { key, expiredHours } = parsed; const { key, expiredHours, responseContentType } = parsed;
const expires = expiredHours ? expiredHours * 60 * 60 : 30 * 60; // expires 的单位是秒 默认 30 分钟 const expires = expiredHours ? expiredHours * 60 * 60 : 30 * 60; // expires 的单位是秒 默认 30 分钟
return await this.client.generatePresignedGetUrl({ key, expiredSeconds: expires }); return await this.client.generatePresignedGetUrl({
key,
expiredSeconds: expires,
...(responseContentType ? { responseContentType } : {})
});
} }
async uploadFileByBody(params: UploadFileByBufferParams) { async uploadFileByBody(params: UploadFileByBufferParams) {
......
packages/service/common/s3/contracts/type.ts
...@@ -65,7 +65,8 @@ export type CreatePostPresignedUrlResult = z.infer<typeof CreatePostPresignedUrl ...@@ -65,7 +65,8 @@ export type CreatePostPresignedUrlResult = z.infer<typeof CreatePostPresignedUrl
export const CreateGetPresignedUrlParamsSchema = z.object({ export const CreateGetPresignedUrlParamsSchema = z.object({
key: z.string().nonempty(), key: z.string().nonempty(),
expiredHours: z.number().positive().optional(), expiredHours: z.number().positive().optional(),
mode: DownloadModeSchema.optional() mode: DownloadModeSchema.optional(),
responseContentType: z.string().nonempty().optional()
}); });
export type createPreviewUrlParams = z.infer<typeof CreateGetPresignedUrlParamsSchema>; export type createPreviewUrlParams = z.infer<typeof CreateGetPresignedUrlParamsSchema>;
......
packages/service/common/s3/sources/dataset/index.ts
...@@ -19,7 +19,7 @@ import { getLogger, LogCategories } from '../../../logger'; ...@@ -19,7 +19,7 @@ import { getLogger, LogCategories } from '../../../logger';
import { detectFileEncoding } from '@fastgpt/global/common/file/tools'; import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
import { readFileContentByBuffer } from '../../../file/read/utils'; import { readFileContentByBuffer } from '../../../file/read/utils';
import path from 'node:path'; import path from 'node:path';
import { resolveMimeType } from '../../utils/mime'; import { ensureTextContentTypeCharset, isTextLikeFile, resolveMimeType } from '../../utils/mime';
import { datasetAllowedExtensions } from '../../utils/uploadConstraints'; import { datasetAllowedExtensions } from '../../utils/uploadConstraints';
import { getFileS3Key, truncateFilename } from '../../utils'; import { getFileS3Key, truncateFilename } from '../../utils';
import type { S3RawTextSource } from '../rawText'; import type { S3RawTextSource } from '../rawText';
...@@ -38,11 +38,19 @@ export class S3DatasetSource extends S3PrivateBucket { ...@@ -38,11 +38,19 @@ export class S3DatasetSource extends S3PrivateBucket {
// 下载链接 // 下载链接
async createGetDatasetFileURL(params: CreateGetDatasetFileURLParams) { async createGetDatasetFileURL(params: CreateGetDatasetFileURLParams) {
const { key, expiredHours, external } = CreateGetDatasetFileURLParamsSchema.parse(params); const { key, expiredHours, external } = CreateGetDatasetFileURLParamsSchema.parse(params);
const fileMetadata = await this.getFileMetadata(key).catch(() => undefined);
const responseContentType =
fileMetadata && isTextLikeFile(fileMetadata)
? ensureTextContentTypeCharset({
contentType: fileMetadata.contentType,
filename: fileMetadata.filename
})
: undefined;
if (external) { if (external) {
return await this.createExternalUrl({ key, expiredHours }); return await this.createExternalUrl({ key, expiredHours, responseContentType });
} }
return await this.createPreviewUrl({ key, expiredHours }); return await this.createPreviewUrl({ key, expiredHours, responseContentType });
} }
// 上传链接 // 上传链接
......
packages/service/common/s3/utils/mime.ts
import { extension as getMimeExtension, lookup as lookupMimeType } from 'mime-types'; import { extension as getMimeExtension, lookup as lookupMimeType } from 'mime-types';
import path from 'node:path';
export const DEFAULT_CONTENT_TYPE = 'application/octet-stream'; export const DEFAULT_CONTENT_TYPE = 'application/octet-stream';
...@@ -32,3 +33,69 @@ export const resolveMimeExtension = (mimeType?: string | false): `.${string}` | ...@@ -32,3 +33,69 @@ export const resolveMimeExtension = (mimeType?: string | false): `.${string}` |
const extension = getMimeExtension(normalizeMimeType(mimeType, '')); const extension = getMimeExtension(normalizeMimeType(mimeType, ''));
return extension ? `.${extension}` : ''; return extension ? `.${extension}` : '';
}; };
const textLikeExtensions = new Set([
'.csv',
'.htm',
'.html',
'.json',
'.log',
'.md',
'.markdown',
'.txt',
'.xml',
'.yaml',
'.yml'
]);
const textLikeMimeTypes = new Set([
'application/javascript',
'application/json',
'application/ld+json',
'application/markdown',
'application/x-javascript',
'application/xml',
'image/svg+xml'
]);
export const isTextLikeFile = ({
contentType,
filename
}: {
contentType?: string;
filename?: string;
}) => {
const normalizedContentType = normalizeMimeType(contentType, '');
if (normalizedContentType.startsWith('text/')) return true;
if (textLikeMimeTypes.has(normalizedContentType)) return true;
if (normalizedContentType && normalizedContentType !== DEFAULT_CONTENT_TYPE) return false;
const extension = path.extname(filename || '').toLowerCase();
return textLikeExtensions.has(extension);
};
export const ensureTextContentTypeCharset = ({
contentType,
filename,
charset = 'utf-8'
}: {
contentType?: string;
filename?: string;
charset?: string;
}) => {
const normalizedContentType = normalizeMimeType(contentType, '');
const resolvedContentType =
normalizedContentType && normalizedContentType !== DEFAULT_CONTENT_TYPE
? normalizedContentType
: resolveMimeType([filename], normalizedContentType || DEFAULT_CONTENT_TYPE);
if (!isTextLikeFile({ contentType: resolvedContentType, filename })) {
return resolvedContentType;
}
if (/;\s*charset=/i.test(contentType || '')) {
return contentType || resolvedContentType;
}
return `${resolvedContentType}; charset=${charset}`;
};
packages/service/core/workflow/dispatch/dataset/nodeResponse.ts
...@@ -22,7 +22,7 @@ const createDatasetSearchChildNodeResponse = ({ ...@@ -22,7 +22,7 @@ const createDatasetSearchChildNodeResponse = ({
textOutput?: string; textOutput?: string;
}): ChatHistoryItemResType => { }): ChatHistoryItemResType => {
const llmRequestIds = requestIds?.length ? requestIds : requestId ? [requestId] : undefined; const llmRequestIds = requestIds?.length ? requestIds : requestId ? [requestId] : undefined;
const id = requestId || getNanoid(); const id = requestId || llmRequestIds?.[0] || getNanoid();
return { return {
id, id,
......
packages/service/test/common/s3/config/constants.test.ts
...@@ -72,4 +72,29 @@ describe('s3 storage constants', () => { ...@@ -72,4 +72,29 @@ describe('s3 storage constants', () => {
'https://cdn.example.com/get/fastgpt-private/chat%2Fapp%2Fuser%2Fchat%2Ffile.png' 'https://cdn.example.com/get/fastgpt-private/chat%2Fapp%2Fuser%2Fchat%2Ffile.png'
); );
}); });
it('passes response content type overrides into external presigned URLs', async () => {
const { S3BaseBucket } = await vi.importActual<
typeof import('@fastgpt/service/common/s3/buckets/base')
>('@fastgpt/service/common/s3/buckets/base');
const storage = createVitestStorageMock({
vi,
bucketName: 'fastgpt-private',
baseUrl: 'https://s3.example.com'
});
const bucket = new S3BaseBucket(storage, undefined);
const result = await bucket.createExternalUrl({
key: 'dataset/team/aaa.md',
mode: 'presigned',
responseContentType: 'text/markdown; charset=utf-8'
});
expect(storage.generatePresignedGetUrl).toHaveBeenCalledWith({
key: 'dataset/team/aaa.md',
expiredSeconds: 1800,
responseContentType: 'text/markdown; charset=utf-8'
});
expect(result.url).toContain('response-content-type=text%2Fmarkdown%3B%20charset%3Dutf-8');
});
}); });
packages/service/test/common/s3/mime.test.ts
import { describe, expect, it } from 'vitest'; import { describe, expect, it } from 'vitest';
import { import {
DEFAULT_CONTENT_TYPE, DEFAULT_CONTENT_TYPE,
ensureTextContentTypeCharset,
isTextLikeFile,
normalizeMimeType, normalizeMimeType,
resolveMimeExtension, resolveMimeExtension,
resolveMimeType resolveMimeType
...@@ -42,3 +44,68 @@ describe('resolveMimeExtension', () => { ...@@ -42,3 +44,68 @@ describe('resolveMimeExtension', () => {
expect(resolveMimeExtension('application/x-custom-type')).toBe(''); expect(resolveMimeExtension('application/x-custom-type')).toBe('');
}); });
}); });
describe('isTextLikeFile', () => {
it('detects text-like MIME types', () => {
expect(isTextLikeFile({ contentType: 'text/markdown' })).toBe(true);
expect(isTextLikeFile({ contentType: 'application/json' })).toBe(true);
expect(isTextLikeFile({ contentType: 'image/svg+xml' })).toBe(true);
});
it('detects text-like extensions when MIME type is generic', () => {
expect(isTextLikeFile({ contentType: 'application/octet-stream', filename: 'aaa.md' })).toBe(
true
);
expect(isTextLikeFile({ filename: 'page.HTML' })).toBe(true);
});
it('does not mark binary files as text-like', () => {
expect(isTextLikeFile({ contentType: 'application/pdf', filename: 'report.pdf' })).toBe(false);
expect(isTextLikeFile({ contentType: 'application/pdf', filename: 'a.md' })).toBe(false);
expect(isTextLikeFile({ contentType: 'image/png', filename: 'image.png' })).toBe(false);
});
});
describe('ensureTextContentTypeCharset', () => {
it('adds utf-8 charset for text-like content types', () => {
expect(ensureTextContentTypeCharset({ contentType: 'text/markdown', filename: 'aaa.md' })).toBe(
'text/markdown; charset=utf-8'
);
expect(ensureTextContentTypeCharset({ contentType: 'text/html', filename: 'page.html' })).toBe(
'text/html; charset=utf-8'
);
});
it('keeps existing charset untouched', () => {
expect(
ensureTextContentTypeCharset({
contentType: 'text/markdown; charset=gbk',
filename: 'aaa.md'
})
).toBe('text/markdown; charset=gbk');
});
it('resolves a text MIME type from filename when metadata is generic', () => {
expect(
ensureTextContentTypeCharset({
contentType: 'application/octet-stream',
filename: 'aaa.md'
})
).toBe('text/markdown; charset=utf-8');
});
it('does not add charset for binary files', () => {
expect(
ensureTextContentTypeCharset({
contentType: 'application/pdf',
filename: 'report.pdf'
})
).toBe('application/pdf');
expect(
ensureTextContentTypeCharset({
contentType: 'application/pdf',
filename: 'a.md'
})
).toBe('application/pdf');
});
});
projects/app/src/pages/api/system/file/[jwt].ts
...@@ -3,6 +3,7 @@ import { jsonRes } from '@fastgpt/service/common/response'; ...@@ -3,6 +3,7 @@ import { jsonRes } from '@fastgpt/service/common/response';
import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset'; import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset';
import { getLogger, LogCategories } from '@fastgpt/service/common/logger'; import { getLogger, LogCategories } from '@fastgpt/service/common/logger';
import { jwtVerifyS3ObjectKey, isS3ObjectKey } from '@fastgpt/service/common/s3/utils'; import { jwtVerifyS3ObjectKey, isS3ObjectKey } from '@fastgpt/service/common/s3/utils';
import { ensureTextContentTypeCharset } from '@fastgpt/service/common/s3/utils/mime';
import { getS3ChatSource } from '@fastgpt/service/common/s3/sources/chat'; import { getS3ChatSource } from '@fastgpt/service/common/s3/sources/chat';
import { getContentDisposition } from '@fastgpt/global/common/file/tools'; import { getContentDisposition } from '@fastgpt/global/common/file/tools';
import path from 'path'; import path from 'path';
...@@ -43,7 +44,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -43,7 +44,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
} }
if (metadata?.contentType) { if (metadata?.contentType) {
res.setHeader('Content-Type', metadata.contentType); res.setHeader(
'Content-Type',
ensureTextContentTypeCharset({
contentType: metadata.contentType,
filename: metadata.filename
})
);
} }
if (metadata?.contentLength) { if (metadata?.contentLength) {
res.setHeader('Content-Length', metadata.contentLength); res.setHeader('Content-Length', metadata.contentLength);
......
projects/app/src/pages/api/system/file/download/[token].ts
...@@ -2,6 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; ...@@ -2,6 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response'; import { jsonRes } from '@fastgpt/service/common/response';
import { getLogger, LogCategories } from '@fastgpt/service/common/logger'; import { getLogger, LogCategories } from '@fastgpt/service/common/logger';
import { jwtVerifyS3DownloadToken } from '@fastgpt/service/common/s3/security/token'; import { jwtVerifyS3DownloadToken } from '@fastgpt/service/common/s3/security/token';
import { ensureTextContentTypeCharset } from '@fastgpt/service/common/s3/utils/mime';
import { getContentDisposition } from '@fastgpt/global/common/file/tools'; import { getContentDisposition } from '@fastgpt/global/common/file/tools';
import path from 'path'; import path from 'path';
...@@ -46,7 +47,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -46,7 +47,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
} }
if (metadata?.contentType) { if (metadata?.contentType) {
res.setHeader('Content-Type', metadata.contentType); res.setHeader(
'Content-Type',
ensureTextContentTypeCharset({
contentType: metadata.contentType,
filename: metadata.filename
})
);
} }
if (metadata?.contentLength) { if (metadata?.contentLength) {
res.setHeader('Content-Length', metadata.contentLength); res.setHeader('Content-Length', metadata.contentLength);
......
projects/app/test/api/system/file/sourceContentType.test.ts 0 → 100644
import { beforeEach, describe, expect, it, vi } from 'vitest';
import proxyDownloadHandler from '@/pages/api/system/file/download/[token]';
import legacyFileHandler from '@/pages/api/system/file/[jwt]';
import { jwtVerifyS3DownloadToken } from '@fastgpt/service/common/s3/security/token';
import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset';
import { getS3ChatSource } from '@fastgpt/service/common/s3/sources/chat';
import { jwtVerifyS3ObjectKey } from '@fastgpt/service/common/s3/utils';
vi.mock('@fastgpt/service/common/s3/security/token', () => ({
jwtVerifyS3DownloadToken: vi.fn()
}));
vi.mock('@fastgpt/service/common/s3/utils', () => ({
jwtVerifyS3ObjectKey: vi.fn(),
isS3ObjectKey: vi.fn(
(key: string | undefined, source: string) =>
typeof key === 'string' && key.startsWith(`${source}/`)
)
}));
vi.mock('@fastgpt/service/common/s3/sources/dataset', () => ({
getS3DatasetSource: vi.fn()
}));
vi.mock('@fastgpt/service/common/s3/sources/chat', () => ({
getS3ChatSource: vi.fn()
}));
const makeMockStream = () => {
const stream = {
pipe: vi.fn(),
on: vi.fn(() => stream)
};
return stream;
};
const makeMockRes = () => {
const headers: Record<string, string | number> = {};
const res = {
headers,
headersSent: false,
setHeader: vi.fn((key: string, value: string | number) => {
headers[key] = value;
}),
status: vi.fn(() => res),
end: vi.fn(),
json: vi.fn()
};
return res;
};
describe('system file response content type', () => {
beforeEach(() => {
vi.clearAllMocks();
(global as any).s3BucketMap = {};
});
it('adds utf-8 charset for text files in proxy download mode', async () => {
const stream = makeMockStream();
const bucket = {
getFileStream: vi.fn().mockResolvedValue(stream),
getFileMetadata: vi.fn().mockResolvedValue({
filename: 'aaa.md',
contentType: 'text/markdown',
contentLength: 32
})
};
(global as any).s3BucketMap = {
'fastgpt-private': bucket
};
vi.mocked(jwtVerifyS3DownloadToken).mockResolvedValue({
objectKey: 'dataset/team/aaa.md',
bucketName: 'fastgpt-private',
type: 'download'
});
const req = { method: 'GET', query: { token: 'token' } } as any;
const res = makeMockRes() as any;
await proxyDownloadHandler(req, res);
expect(res.headers['Content-Type']).toBe('text/markdown; charset=utf-8');
expect(res.headers['Content-Length']).toBe(32);
expect(stream.pipe).toHaveBeenCalledWith(res);
});
it('keeps binary content type unchanged in proxy download mode', async () => {
const stream = makeMockStream();
const bucket = {
getFileStream: vi.fn().mockResolvedValue(stream),
getFileMetadata: vi.fn().mockResolvedValue({
filename: 'image.png',
contentType: 'image/png',
contentLength: 64
})
};
(global as any).s3BucketMap = {
'fastgpt-private': bucket
};
vi.mocked(jwtVerifyS3DownloadToken).mockResolvedValue({
objectKey: 'dataset/team/image.png',
bucketName: 'fastgpt-private',
type: 'download'
});
const req = { method: 'GET', query: { token: 'token' } } as any;
const res = makeMockRes() as any;
await proxyDownloadHandler(req, res);
expect(res.headers['Content-Type']).toBe('image/png');
});
it('adds utf-8 charset for text files in legacy file entry', async () => {
const stream = makeMockStream();
const datasetSource = {
getFileStream: vi.fn().mockResolvedValue(stream),
getFileMetadata: vi.fn().mockResolvedValue({
filename: 'page.html',
contentType: 'text/html',
contentLength: 128
})
};
vi.mocked(getS3DatasetSource).mockReturnValue(datasetSource as any);
vi.mocked(getS3ChatSource).mockReturnValue({} as any);
vi.mocked(jwtVerifyS3ObjectKey).mockResolvedValue({
objectKey: 'dataset/team/page.html'
});
const req = { query: { jwt: 'jwt' } } as any;
const res = makeMockRes() as any;
await legacyFileHandler(req, res);
expect(res.headers['Content-Type']).toBe('text/html; charset=utf-8');
expect(stream.pipe).toHaveBeenCalledWith(res);
});
});
sdk/storage/src/adapters/aws-s3.adapter.ts
...@@ -105,7 +105,7 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -105,7 +105,7 @@ export class AwsS3StorageAdapter implements IStorage {
}) })
); );
let metadata: StorageObjectMetadata = {}; const metadata: StorageObjectMetadata = {};
if (result.Metadata) { if (result.Metadata) {
for (const [k, v] of Object.entries(result.Metadata)) { for (const [k, v] of Object.entries(result.Metadata)) {
if (!k) continue; if (!k) continue;
...@@ -136,7 +136,7 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -136,7 +136,7 @@ export class AwsS3StorageAdapter implements IStorage {
async uploadObject(params: UploadObjectParams): Promise<UploadObjectResult> { async uploadObject(params: UploadObjectParams): Promise<UploadObjectResult> {
const { key, body, contentType, contentLength, contentDisposition, metadata } = params; const { key, body, contentType, contentLength, contentDisposition, metadata } = params;
let meta: StorageObjectMetadata = {}; const meta: StorageObjectMetadata = {};
if (metadata) { if (metadata) {
for (const [k, v] of Object.entries(metadata)) { for (const [k, v] of Object.entries(metadata)) {
if (!k) continue; if (!k) continue;
...@@ -240,7 +240,7 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -240,7 +240,7 @@ export class AwsS3StorageAdapter implements IStorage {
throw new Error('Prefix is required'); throw new Error('Prefix is required');
} }
let fails: StorageObjectKey[] = []; const fails: StorageObjectKey[] = [];
let isTruncated = false; let isTruncated = false;
let continuationToken: string | undefined = undefined; let continuationToken: string | undefined = undefined;
...@@ -339,7 +339,7 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -339,7 +339,7 @@ export class AwsS3StorageAdapter implements IStorage {
} }
async generatePresignedGetUrl(params: PresignedGetUrlParams): Promise<PresignedGetUrlResult> { async generatePresignedGetUrl(params: PresignedGetUrlParams): Promise<PresignedGetUrlResult> {
const { key, expiredSeconds } = params; const { key, expiredSeconds, responseContentType } = params;
const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS; const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS;
...@@ -347,7 +347,8 @@ export class AwsS3StorageAdapter implements IStorage { ...@@ -347,7 +347,8 @@ export class AwsS3StorageAdapter implements IStorage {
this.client, this.client,
new GetObjectCommand({ new GetObjectCommand({
Bucket: this.options.bucket, Bucket: this.options.bucket,
Key: key Key: key,
ResponseContentType: responseContentType
}), }),
{ {
expiresIn expiresIn
......
sdk/storage/src/adapters/cos.adapter.ts
...@@ -114,7 +114,7 @@ export class CosStorageAdapter implements IStorage { ...@@ -114,7 +114,7 @@ export class CosStorageAdapter implements IStorage {
); );
}); });
let metadata: StorageObjectMetadata = {}; const metadata: StorageObjectMetadata = {};
if (result.headers) { if (result.headers) {
Object.entries(result.headers).forEach(([key, val]) => { Object.entries(result.headers).forEach(([key, val]) => {
if (key.startsWith('x-cos-meta-')) { if (key.startsWith('x-cos-meta-')) {
...@@ -387,7 +387,7 @@ export class CosStorageAdapter implements IStorage { ...@@ -387,7 +387,7 @@ export class CosStorageAdapter implements IStorage {
} }
async generatePresignedGetUrl(params: PresignedGetUrlParams): Promise<PresignedGetUrlResult> { async generatePresignedGetUrl(params: PresignedGetUrlParams): Promise<PresignedGetUrlResult> {
const { key, expiredSeconds } = params; const { key, expiredSeconds, responseContentType } = params;
const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS; const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS;
const url = await new Promise<string>((resolve, reject) => { const url = await new Promise<string>((resolve, reject) => {
...@@ -398,7 +398,10 @@ export class CosStorageAdapter implements IStorage { ...@@ -398,7 +398,10 @@ export class CosStorageAdapter implements IStorage {
Key: key, Key: key,
Expires: expiresIn, Expires: expiresIn,
Sign: true, Sign: true,
Method: 'GET' Method: 'GET',
...(responseContentType
? { Query: { 'response-content-type': responseContentType } }
: {})
}, },
(err, data) => { (err, data) => {
if (err) { if (err) {
......
sdk/storage/src/adapters/oss.adapter.ts
...@@ -86,7 +86,7 @@ export class OssStorageAdapter implements IStorage { ...@@ -86,7 +86,7 @@ export class OssStorageAdapter implements IStorage {
const result = await this.client.head(key); const result = await this.client.head(key);
let metadata: StorageObjectMetadata = {}; const metadata: StorageObjectMetadata = {};
if (result.meta) { if (result.meta) {
for (const [k, v] of Object.entries(result.meta)) { for (const [k, v] of Object.entries(result.meta)) {
if (!k) continue; if (!k) continue;
...@@ -131,7 +131,7 @@ export class OssStorageAdapter implements IStorage { ...@@ -131,7 +131,7 @@ export class OssStorageAdapter implements IStorage {
if (contentLength !== undefined) headers['Content-Length'] = String(contentLength); if (contentLength !== undefined) headers['Content-Length'] = String(contentLength);
if (contentDisposition) headers['Content-Disposition'] = contentDisposition; if (contentDisposition) headers['Content-Disposition'] = contentDisposition;
let meta = {} as StorageObjectMetadata & OSS.UserMeta; const meta = {} as StorageObjectMetadata & OSS.UserMeta;
if (metadata) { if (metadata) {
for (const [k, v] of Object.entries(metadata)) { for (const [k, v] of Object.entries(metadata)) {
if (!k) continue; if (!k) continue;
...@@ -268,12 +268,19 @@ export class OssStorageAdapter implements IStorage { ...@@ -268,12 +268,19 @@ export class OssStorageAdapter implements IStorage {
} }
async generatePresignedGetUrl(params: PresignedGetUrlParams): Promise<PresignedGetUrlResult> { async generatePresignedGetUrl(params: PresignedGetUrlParams): Promise<PresignedGetUrlResult> {
const { key, expiredSeconds } = params; const { key, expiredSeconds, responseContentType } = params;
const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS; const expiresIn = expiredSeconds ? expiredSeconds : DEFAULT_PRESIGNED_URL_EXPIRED_SECONDS;
const url = this.client.signatureUrl(key, { const url = this.client.signatureUrl(key, {
method: 'GET', method: 'GET',
expires: expiresIn expires: expiresIn,
...(responseContentType
? {
response: {
'content-type': responseContentType
}
}
: {})
}); });
return { return {
......
sdk/storage/src/testing/vitestMock.ts
...@@ -164,7 +164,10 @@ export function createVitestStorageMock(params: CreateVitestStorageMockParams): ...@@ -164,7 +164,10 @@ export function createVitestStorageMock(params: CreateVitestStorageMockParams):
const generatePresignedGetUrl = vi.fn( const generatePresignedGetUrl = vi.fn(
async (p: PresignedGetUrlParams): Promise<PresignedGetUrlResult> => { async (p: PresignedGetUrlParams): Promise<PresignedGetUrlResult> => {
const getUrl = `${baseUrl}/get/${encodeURIComponent(bucketName)}/${encodeURIComponent(p.key)}`; const query = p.responseContentType
? `?response-content-type=${encodeURIComponent(p.responseContentType)}`
: '';
const getUrl = `${baseUrl}/get/${encodeURIComponent(bucketName)}/${encodeURIComponent(p.key)}${query}`;
return { bucket: bucketName, key: p.key, url: getUrl }; return { bucket: bucketName, key: p.key, url: getUrl };
} }
); );
......
sdk/storage/src/types.ts
...@@ -240,6 +240,8 @@ export type PresignedGetUrlParams = { ...@@ -240,6 +240,8 @@ export type PresignedGetUrlParams = {
key: StorageObjectKey; key: StorageObjectKey;
/** 过期时间（秒），可选，默认 1800 秒。 */ /** 过期时间（秒），可选，默认 1800 秒。 */
expiredSeconds?: number; expiredSeconds?: number;
/** 覆盖下载响应的 Content-Type。 */
responseContentType?: string;
}; };
/** /**
......
test/sdk/storage/adapters/cos.adapter.test.ts 0 → 100644
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { CosStorageAdapter } from '../../../../sdk/storage/src/adapters/cos.adapter';
const createAdapter = () =>
new CosStorageAdapter({
vendor: 'cos',
bucket: 'fastgpt-private',
region: 'ap-guangzhou',
credentials: {
accessKeyId: 'secret-id',
secretAccessKey: 'secret-key'
}
});
describe('CosStorageAdapter.generatePresignedGetUrl', () => {
const getObjectUrlMock = vi.fn();
beforeEach(() => {
vi.clearAllMocks();
getObjectUrlMock.mockImplementation((params, callback) => {
callback(null, {
Url: `https://cos.example.com/${params.Key}`
});
});
});
it('maps response content type overrides to COS query params', async () => {
const adapter = createAdapter();
(adapter as any).client.getObjectUrl = getObjectUrlMock;
await adapter.generatePresignedGetUrl({
key: 'dataset/team/aaa.md',
expiredSeconds: 300,
responseContentType: 'text/markdown; charset=utf-8'
});
expect(getObjectUrlMock).toHaveBeenCalledWith(
expect.objectContaining({
Bucket: 'fastgpt-private',
Region: 'ap-guangzhou',
Key: 'dataset/team/aaa.md',
Expires: 300,
Sign: true,
Method: 'GET',
Query: {
'response-content-type': 'text/markdown; charset=utf-8'
}
}),
expect.any(Function)
);
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment