Commit decb6d2f by Archer Committed by GitHub

fix: bind S3 object keys to authorized resources (#7104)

* fix: validate helperbot file preview input

* refactor: split s3 key authorization helpers

* docs: add s3 key auth fix note
parent 602bdf85
......@@ -28,12 +28,12 @@ FastGPT 的私有对象存储 key 是 bucket 内的全局路径字符串,例
## 统一授权 helper
新增入口必须优先复用 `packages/service/common/s3/utils.ts` 中的 helper:
新增入口必须优先复用各 S3 source 的 key helper:
- `parseChatFileS3Key` / `isAuthorizedChatFileS3Key`
- `parseDatasetFileS3Key` / `isAuthorizedDatasetFileS3Key`
- `parseHelperBotFileS3Key` / `isAuthorizedHelperBotFileS3Key`
- `isAuthorizedTempFileS3Key`
- `packages/service/common/s3/sources/chat/key.ts`: `parseChatFileS3Key` / `isAuthorizedChatFileS3Key`
- `packages/service/common/s3/sources/dataset/key.ts`: `parseDatasetFileS3Key` / `isAuthorizedDatasetFileS3Key`
- `packages/service/common/s3/sources/helperbot/key.ts`: `parseHelperBotFileS3Key` / `isAuthorizedHelperBotFileS3Key`
- `packages/service/common/s3/sources/temp/key.ts`: `isAuthorizedTempFileS3Key`
这些 helper 只负责 key 结构解析和 key 与已鉴权上下文的绑定判断。业务权限仍由对应 auth 函数负责:
......
......@@ -11,3 +11,7 @@ description: 'FastGPT V4.15.0-beta5 Release Notes'
1. HTML output now automatically switches to preview mode after generation, reducing the need to open the preview manually.
2. Improved long-name display for apps, datasets, files, and folders: names are truncated when they exceed the available width, and the full name is shown when hovering over the name.
## 🐛 Bug Fixes
1. Fixed a potential cross-resource file access risk when private S3 object keys were not bound to the already-authorized resource.
......@@ -11,3 +11,7 @@ description: 'FastGPT V4.15.0-beta5 更新说明'
1. HTML 输出后自动切换为预览,减少手动打开预览的操作。
2. 优化应用、知识库、文件和文件夹等长名称展示:超出宽度时自动省略,并在 hover 名称时展示完整内容。
## 🐛 修复
1. 修复 S3 私有对象 key 未绑定已鉴权资源时可能导致的跨资源文件访问风险。
......@@ -237,7 +237,7 @@ export class S3BaseBucket {
* 为对象 key 生成外部可访问 URL。
*
* 该方法只负责存储层签名,不做 team/app/dataset/user 的业务归属校验。任何 API 边界或
* 用户可控 key 调用到这里前,必须先使用 common/s3/utils 中的授权绑定 helper 校验 key
* 用户可控 key 调用到这里前,必须先使用对应 S3 source 的 key helper 校验 key
* 属于当前已鉴权资源。
*/
async createExternalUrl(params: createPreviewUrlParams) {
......
import { isS3ObjectKey } from '../../utils';
/**
* 解析聊天文件的 S3 key。
*
* 聊天文件 key 的授权边界由 appId 与 uid 决定。任何签名或读取前都应先解析
* 这些路径段并与已鉴权上下文绑定,避免只校验一个无关 app 后签发任意 object key。
*/
export function parseChatFileS3Key(key: string): {
appId: string;
uid: string;
chatId: string;
filename: string;
} | null {
if (!isS3ObjectKey(key, 'chat')) return null;
const [, appId, uid, chatId, ...filenameParts] = key.split('/');
const filename = filenameParts.join('/');
if (!appId || !uid || !chatId || !filename) return null;
return {
appId,
uid,
chatId,
filename
};
}
/**
* 判断聊天文件 key 是否属于已鉴权的 app 与聊天用户。
*/
export function isAuthorizedChatFileS3Key({
key,
appId,
uid
}: {
key: string;
appId: string;
uid: string;
}) {
const parsedKey = parseChatFileS3Key(key);
return (
!!parsedKey &&
String(parsedKey.appId) === String(appId) &&
String(parsedKey.uid) === String(uid)
);
}
......@@ -21,7 +21,7 @@ import { readFileContentByBuffer } from '../../../file/read/utils';
import { ensureTextContentTypeCharset, isTextLikeFile, resolveMimeType } from '../../utils/mime';
import { datasetAllowedExtensions } from '../../utils/uploadConstraints';
import { getFileS3Key, truncateFilename } from '../../utils';
import { isAuthorizedDatasetFileS3Key } from '../../utils';
import { isAuthorizedDatasetFileS3Key } from './key';
import type { S3RawTextSource } from '../rawText';
import { getS3RawTextSource } from '../rawText';
......
import { isS3ObjectKey } from '../../utils';
/**
* 解析数据集文件的 S3 key。
*
* 数据集文件的第二段是 datasetId。调用方应基于解析出的 datasetId 做权限校验,而不是把
* S3 对象存在性当作访问权限。
*/
export function parseDatasetFileS3Key(key: string): {
datasetId: string;
filename: string;
} | null {
if (!isS3ObjectKey(key, 'dataset')) return null;
const [, datasetId, ...filenameParts] = key.split('/');
const filename = filenameParts.join('/');
if (!datasetId || !filename) return null;
return {
datasetId,
filename
};
}
/**
* 判断数据集文件 key 是否属于指定 dataset。
*/
export function isAuthorizedDatasetFileS3Key({
key,
datasetId
}: {
key: string;
datasetId: string;
}) {
const parsedKey = parseDatasetFileS3Key(key);
return !!parsedKey && String(parsedKey.datasetId) === String(datasetId);
}
......@@ -10,7 +10,8 @@ import {
import { differenceInHours } from 'date-fns';
import { S3Buckets } from '../../config/constants';
import path from 'path';
import { getFileS3Key, parseHelperBotFileS3Key } from '../../utils';
import { getFileS3Key } from '../../utils';
import { parseHelperBotFileS3Key } from './key';
export class S3HelperBotSource extends S3PrivateBucket {
private static instance: S3HelperBotSource;
......
import type { HelperBotTypeEnumType } from '@fastgpt/global/core/chat/helperBot/type';
import { HelperBotTypeEnumSchema } from '@fastgpt/global/core/chat/helperBot/type';
import { isS3ObjectKey } from '../../utils';
/**
* 解析 HelperBot 文件 key。
*
* HelperBot 文件 key 的第一段是固定 source,后续才是 type/user/chat 维度。
*/
export function parseHelperBotFileS3Key(key: string): {
type: HelperBotTypeEnumType;
userId: string;
chatId: string;
filename: string;
} | null {
if (!isS3ObjectKey(key, 'helperBot')) return null;
const [, type, userId, chatId, ...filenameParts] = key.split('/');
const filename = filenameParts.join('/');
const parsedType = HelperBotTypeEnumSchema.safeParse(type);
if (!parsedType.success || !userId || !chatId || !filename) return null;
return {
type: parsedType.data,
userId,
chatId,
filename
};
}
/**
* 判断 HelperBot 文件 key 是否属于当前用户。
*/
export function isAuthorizedHelperBotFileS3Key({ key, userId }: { key: string; userId: string }) {
const parsedKey = parseHelperBotFileS3Key(key);
return !!parsedKey && String(parsedKey.userId) === String(userId);
}
import { isS3ObjectKey } from '../../utils';
/**
* 判断临时文件 key 是否属于指定团队。
*/
export function isAuthorizedTempFileS3Key({ key, teamId }: { key: string; teamId: string }) {
return isS3ObjectKey(key, 'temp') && key.startsWith(`temp/${teamId}/`);
}
......@@ -9,7 +9,6 @@ import { getNanoid } from '@fastgpt/global/common/string/tools';
import path from 'node:path';
import type { ParsedFileContentS3KeyParams } from './sources/dataset/type';
import type { HelperBotTypeEnumType } from '@fastgpt/global/core/chat/helperBot/type';
import { HelperBotTypeEnumSchema } from '@fastgpt/global/core/chat/helperBot/type';
export { jwtSignS3ObjectKey, jwtVerifyS3ObjectKey, jwtSignS3DownloadToken } from './security/token';
......@@ -266,135 +265,6 @@ export function isS3ObjectKey<T extends keyof typeof S3Sources>(
return typeof key === 'string' && key.startsWith(`${S3Sources[source]}/`);
}
/**
* 解析聊天文件的 S3 key。
*
* 聊天文件 key 的授权边界由 appId、uid、chatId 共同决定。任何签名或读取前都应先解析
* 这些路径段并与已鉴权上下文绑定,避免只校验一个无关 app 后签发任意 object key。
*/
export function parseChatFileS3Key(key: string): {
appId: string;
uid: string;
chatId: string;
filename: string;
} | null {
if (!isS3ObjectKey(key, 'chat')) return null;
const [, appId, uid, chatId, ...filenameParts] = key.split('/');
const filename = filenameParts.join('/');
if (!appId || !uid || !chatId || !filename) return null;
return {
appId,
uid,
chatId,
filename
};
}
/**
* 判断聊天文件 key 是否属于已鉴权的 app 与聊天用户。
*/
export function isAuthorizedChatFileS3Key({
key,
appId,
uid
}: {
key: string;
appId: string;
uid: string;
}) {
const parsedKey = parseChatFileS3Key(key);
return (
!!parsedKey &&
String(parsedKey.appId) === String(appId) &&
String(parsedKey.uid) === String(uid)
);
}
/**
* 解析数据集文件的 S3 key。
*
* 数据集文件的第二段是 datasetId。调用方应基于解析出的 datasetId 做权限校验,而不是把
* S3 对象存在性当作访问权限。
*/
export function parseDatasetFileS3Key(key: string): {
datasetId: string;
filename: string;
} | null {
if (!isS3ObjectKey(key, 'dataset')) return null;
const [, datasetId, ...filenameParts] = key.split('/');
const filename = filenameParts.join('/');
if (!datasetId || !filename) return null;
return {
datasetId,
filename
};
}
/**
* 判断数据集文件 key 是否属于指定 dataset。
*/
export function isAuthorizedDatasetFileS3Key({
key,
datasetId
}: {
key: string;
datasetId: string;
}) {
const parsedKey = parseDatasetFileS3Key(key);
return !!parsedKey && String(parsedKey.datasetId) === String(datasetId);
}
/**
* 解析 HelperBot 文件 key。
*
* HelperBot 文件 key 的第一段是固定 source,后续才是 type/user/chat 维度。
*/
export function parseHelperBotFileS3Key(key: string): {
type: HelperBotTypeEnumType;
userId: string;
chatId: string;
filename: string;
} | null {
if (!isS3ObjectKey(key, 'helperBot')) return null;
const [, type, userId, chatId, ...filenameParts] = key.split('/');
const filename = filenameParts.join('/');
const parsedType = HelperBotTypeEnumSchema.safeParse(type);
if (!parsedType.success || !userId || !chatId || !filename) return null;
return {
type: parsedType.data,
userId,
chatId,
filename
};
}
/**
* 判断 HelperBot 文件 key 是否属于当前用户。
*/
export function isAuthorizedHelperBotFileS3Key({ key, userId }: { key: string; userId: string }) {
const parsedKey = parseHelperBotFileS3Key(key);
return !!parsedKey && String(parsedKey.userId) === String(userId);
}
/**
* 判断临时文件 key 是否属于指定团队。
*/
export function isAuthorizedTempFileS3Key({ key, teamId }: { key: string; teamId: string }) {
return isS3ObjectKey(key, 'temp') && key.startsWith(`temp/${teamId}/`);
}
export function sanitizeS3ObjectKey(key: string) {
// 替换掉圆括号
const replaceParentheses = (key: string) => {
......
......@@ -16,7 +16,8 @@ import { getFileMaxSize } from '../../common/file/utils';
import { UserError } from '@fastgpt/global/common/error/utils';
import { getAxiosHeaderValue } from '@fastgpt/global/common/axios/utils';
import { getS3DatasetSource } from '../../common/s3/sources/dataset';
import { getFileS3Key, isS3ObjectKey, isAuthorizedDatasetFileS3Key } from '../../common/s3/utils';
import { getFileS3Key, isS3ObjectKey } from '../../common/s3/utils';
import { isAuthorizedDatasetFileS3Key } from '../../common/s3/sources/dataset/key';
import { getLogger, LogCategories } from '../../common/logger';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
......
......@@ -5,7 +5,7 @@ import type { FileTokenQuery } from '@fastgpt/global/common/file/type';
import jwt from 'jsonwebtoken';
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import { getS3DatasetSource } from '../../../common/s3/sources/dataset';
import { parseDatasetFileS3Key } from '../../../common/s3/utils';
import { parseDatasetFileS3Key } from '../../../common/s3/sources/dataset/key';
import { serviceEnv } from '../../../env';
import { authDataset } from '../dataset/auth';
......
import { describe, expect, it } from 'vitest';
import {
isAuthorizedChatFileS3Key,
parseChatFileS3Key
} from '@fastgpt/service/common/s3/sources/chat/key';
import {
isAuthorizedDatasetFileS3Key,
parseDatasetFileS3Key
} from '@fastgpt/service/common/s3/sources/dataset/key';
import {
isAuthorizedHelperBotFileS3Key,
parseHelperBotFileS3Key
} from '@fastgpt/service/common/s3/sources/helperbot/key';
import { isAuthorizedTempFileS3Key } from '@fastgpt/service/common/s3/sources/temp/key';
describe('authorized S3 object key helpers', () => {
it('parses and authorizes chat file keys by appId and uid', () => {
const key = 'chat/app-1/user-1/chat-1/folder/demo.pdf';
expect(parseChatFileS3Key(key)).toEqual({
appId: 'app-1',
uid: 'user-1',
chatId: 'chat-1',
filename: 'folder/demo.pdf'
});
expect(isAuthorizedChatFileS3Key({ key, appId: 'app-1', uid: 'user-1' })).toBe(true);
expect(isAuthorizedChatFileS3Key({ key, appId: 'app-2', uid: 'user-1' })).toBe(false);
expect(isAuthorizedChatFileS3Key({ key, appId: 'app-1', uid: 'user-2' })).toBe(false);
expect(parseChatFileS3Key('temp/app-1/user-1/chat-1/demo.pdf')).toBeNull();
});
it('parses and authorizes dataset file keys by datasetId', () => {
const key = 'dataset/dataset-1/folder/demo.pdf';
expect(parseDatasetFileS3Key(key)).toEqual({
datasetId: 'dataset-1',
filename: 'folder/demo.pdf'
});
expect(isAuthorizedDatasetFileS3Key({ key, datasetId: 'dataset-1' })).toBe(true);
expect(isAuthorizedDatasetFileS3Key({ key, datasetId: 'dataset-2' })).toBe(false);
expect(parseDatasetFileS3Key('dataset/dataset-1')).toBeNull();
});
it('parses and authorizes helper bot file keys by userId', () => {
const key = 'helperBot/topAgent/user-1/chat-1/demo.pdf';
expect(parseHelperBotFileS3Key(key)).toEqual({
type: 'topAgent',
userId: 'user-1',
chatId: 'chat-1',
filename: 'demo.pdf'
});
expect(isAuthorizedHelperBotFileS3Key({ key, userId: 'user-1' })).toBe(true);
expect(isAuthorizedHelperBotFileS3Key({ key, userId: 'user-2' })).toBe(false);
expect(parseHelperBotFileS3Key('helperBot/unknown/user-1/chat-1/demo.pdf')).toBeNull();
});
it('authorizes temp file keys by exact team path segment', () => {
expect(isAuthorizedTempFileS3Key({ key: 'temp/team-1/demo.pdf', teamId: 'team-1' })).toBe(true);
expect(isAuthorizedTempFileS3Key({ key: 'temp/team-11/demo.pdf', teamId: 'team-1' })).toBe(
false
);
expect(isAuthorizedTempFileS3Key({ key: 'dataset/team-1/demo.pdf', teamId: 'team-1' })).toBe(
false
);
});
});
......@@ -5,14 +5,7 @@ import {
truncateFilename,
S3_FILENAME_MAX_LENGTH,
isS3ObjectKey,
getFileS3Key,
isAuthorizedChatFileS3Key,
isAuthorizedDatasetFileS3Key,
isAuthorizedHelperBotFileS3Key,
isAuthorizedTempFileS3Key,
parseChatFileS3Key,
parseDatasetFileS3Key,
parseHelperBotFileS3Key
getFileS3Key
} from '@fastgpt/service/common/s3/utils';
import * as stringTools from '@fastgpt/global/common/string/tools';
......@@ -512,59 +505,6 @@ describe('isS3ObjectKey', () => {
});
});
describe('authorized S3 object key helpers', () => {
it('parses and authorizes chat file keys by appId and uid', () => {
const key = 'chat/app-1/user-1/chat-1/folder/demo.pdf';
expect(parseChatFileS3Key(key)).toEqual({
appId: 'app-1',
uid: 'user-1',
chatId: 'chat-1',
filename: 'folder/demo.pdf'
});
expect(isAuthorizedChatFileS3Key({ key, appId: 'app-1', uid: 'user-1' })).toBe(true);
expect(isAuthorizedChatFileS3Key({ key, appId: 'app-2', uid: 'user-1' })).toBe(false);
expect(isAuthorizedChatFileS3Key({ key, appId: 'app-1', uid: 'user-2' })).toBe(false);
expect(parseChatFileS3Key('temp/app-1/user-1/chat-1/demo.pdf')).toBeNull();
});
it('parses and authorizes dataset file keys by datasetId', () => {
const key = 'dataset/dataset-1/folder/demo.pdf';
expect(parseDatasetFileS3Key(key)).toEqual({
datasetId: 'dataset-1',
filename: 'folder/demo.pdf'
});
expect(isAuthorizedDatasetFileS3Key({ key, datasetId: 'dataset-1' })).toBe(true);
expect(isAuthorizedDatasetFileS3Key({ key, datasetId: 'dataset-2' })).toBe(false);
expect(parseDatasetFileS3Key('dataset/dataset-1')).toBeNull();
});
it('parses and authorizes helper bot file keys by userId', () => {
const key = 'helperBot/topAgent/user-1/chat-1/demo.pdf';
expect(parseHelperBotFileS3Key(key)).toEqual({
type: 'topAgent',
userId: 'user-1',
chatId: 'chat-1',
filename: 'demo.pdf'
});
expect(isAuthorizedHelperBotFileS3Key({ key, userId: 'user-1' })).toBe(true);
expect(isAuthorizedHelperBotFileS3Key({ key, userId: 'user-2' })).toBe(false);
expect(parseHelperBotFileS3Key('helperBot/unknown/user-1/chat-1/demo.pdf')).toBeNull();
});
it('authorizes temp file keys by exact team path segment', () => {
expect(isAuthorizedTempFileS3Key({ key: 'temp/team-1/demo.pdf', teamId: 'team-1' })).toBe(true);
expect(isAuthorizedTempFileS3Key({ key: 'temp/team-11/demo.pdf', teamId: 'team-1' })).toBe(
false
);
expect(isAuthorizedTempFileS3Key({ key: 'dataset/team-1/demo.pdf', teamId: 'team-1' })).toBe(
false
);
});
});
describe('getFileS3Key', () => {
let mockNanoid: ReturnType<typeof vi.fn>;
......
......@@ -4,7 +4,7 @@ import { getS3ChatSource } from '@fastgpt/service/common/s3/sources/chat';
import { authChatCrud } from '@/service/support/permission/auth/chat';
import { PresignChatFileGetUrlSchema } from '@fastgpt/global/openapi/core/chat/file/api';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { isAuthorizedChatFileS3Key } from '@fastgpt/service/common/s3/utils';
import { isAuthorizedChatFileS3Key } from '@fastgpt/service/common/s3/sources/chat/key';
import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
async function handler(req: ApiRequestProps): Promise<string> {
......
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { NextAPI } from '@/service/middleware/entry';
import type { GetHelperBotFilePreviewParamsType } from '@fastgpt/global/openapi/core/chat/helperBot/api';
import {
GetHelperBotFilePreviewParamsSchema,
type GetHelperBotFilePreviewParamsType
} from '@fastgpt/global/openapi/core/chat/helperBot/api';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { getS3HelperBotSource } from '@fastgpt/service/common/s3/sources/helperbot';
import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
import { isAuthorizedHelperBotFileS3Key } from '@fastgpt/service/common/s3/utils';
import { isAuthorizedHelperBotFileS3Key } from '@fastgpt/service/common/s3/sources/helperbot/key';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
async function handler(req: ApiRequestProps<GetHelperBotFilePreviewParamsType>): Promise<string> {
const { key, mode } = req.body;
const { key, mode } = parseApiInput({
req,
bodySchema: GetHelperBotFilePreviewParamsSchema
}).body;
const { userId } = await authCert({
req,
authToken: true
......
......@@ -5,7 +5,7 @@ import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { authDatasetFileKey } from '@fastgpt/service/support/permission/auth/file';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { isAuthorizedDatasetFileS3Key } from '@fastgpt/service/common/s3/utils';
import { isAuthorizedDatasetFileS3Key } from '@fastgpt/service/common/s3/sources/dataset/key';
import {
computedCollectionChunkSettings,
getLLMMaxChunkSize
......
......@@ -4,10 +4,8 @@ import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { addHours } from 'date-fns';
import { S3Buckets } from '@fastgpt/service/common/s3/config/constants';
import {
isAuthorizedTempFileS3Key,
jwtSignS3DownloadToken
} from '@fastgpt/service/common/s3/utils';
import { jwtSignS3DownloadToken } from '@fastgpt/service/common/s3/utils';
import { isAuthorizedTempFileS3Key } from '@fastgpt/service/common/s3/sources/temp/key';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import {
GetSearchTestImagePreviewUrlsBodySchema,
......
......@@ -12,7 +12,7 @@ import { getRerankModel } from '@fastgpt/service/core/ai/model';
import { addAuditLog } from '@fastgpt/service/support/user/audit/util';
import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
import { isAuthorizedTempFileS3Key } from '@fastgpt/service/common/s3/utils';
import { isAuthorizedTempFileS3Key } from '@fastgpt/service/common/s3/sources/temp/key';
import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset';
import {
SearchDatasetTestBodySchema,
......
import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
import { ApiRequestInputParseError } from '@fastgpt/service/common/zod/requestParseError';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { beforeEach, describe, expect, it, vi } from 'vitest';
......@@ -75,4 +76,11 @@ describe('getFilePreviewUrl', () => {
expect(mocks.createGetFileURL).not.toHaveBeenCalled();
});
it('rejects invalid request body before auth or signing', async () => {
await expect(callHandler({})).rejects.toBeInstanceOf(ApiRequestInputParseError);
expect(mocks.authCert).not.toHaveBeenCalled();
expect(mocks.createGetFileURL).not.toHaveBeenCalled();
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment