Commit ff1a1a10 by DigHuang Committed by GitHub

feat(sandbox): support dynamic WS limits and multipart upload (#7205)

* feat(sandbox): support dynamic WS limits and multipart upload

* feat(sandbox-adapter): implement readFileStream for OpenSandbox and SealosDevbox
parent d68b7cec
......@@ -34,6 +34,44 @@ export const SandboxDownloadResponseSchema = z
.meta({ format: 'binary', description: '文件流或 ZIP 包' });
/**
* 上传文件到沙盒工作区 - multipart/form-data 文档结构。
*/
export const SandboxUploadMultipartSchema = z.object({
file: z.any().meta({
format: 'binary',
description: '上传文件,multipart/form-data 的 file 字段'
}),
data: createOutLinkChatTargetInputSchema({
...SandboxBaseShape,
path: z.string().meta({
example: 'src/main.py',
description: '目标文件路径,相对于沙盒工作区根目录'
})
}).meta({
description: '上传参数,JSON 序列化后传入 multipart/form-data 的 data 字段'
})
});
export const SandboxUploadBodySchema = withSandboxTarget({
path: z.string().meta({
example: 'src/main.py',
description: '目标文件路径,相对于沙盒工作区根目录'
})
});
export const SandboxUploadResponseSchema = z.object({
path: z.string().meta({
example: 'src/main.py',
description: '上传成功后的目标文件路径'
}),
bytesWritten: z.number().int().nonnegative().meta({
example: 1024,
description: '写入字节数'
})
});
export type SandboxUploadBody = z.input<typeof SandboxUploadBodySchema>;
export type SandboxUploadRuntimeBody = z.output<typeof SandboxUploadBodySchema>;
export type SandboxUploadResponse = z.infer<typeof SandboxUploadResponseSchema>;
/**
* 检查沙盒是否存在
*/
export const SandboxCheckExistBodyRawSchema = createOutLinkChatTargetInputSchema(SandboxBaseShape);
......
......@@ -3,6 +3,8 @@ import { DevApiTagsMap } from '../../../tag';
import {
SandboxDownloadBodyRawSchema,
SandboxDownloadResponseSchema,
SandboxUploadMultipartSchema,
SandboxUploadResponseSchema,
SandboxCheckExistBodyRawSchema,
SandboxCheckExistResponseSchema,
SandboxGetTicketBodyRawSchema,
......@@ -36,6 +38,35 @@ export const SandboxPath: OpenAPIPath = {
}
},
'/core/ai/sandbox/upload': {
post: {
summary: '上传文件到沙盒',
description:
'通过 multipart/form-data 上传文件,并写入指定沙盒工作区路径。`file` 字段为二进制文件,`data` 字段为 JSON 序列化的上传参数对象',
tags: [DevApiTagsMap.sandbox],
requestBody: {
content: {
'multipart/form-data': {
schema: SandboxUploadMultipartSchema,
encoding: {
data: { contentType: 'application/json' }
}
}
}
},
responses: {
200: {
description: '上传结果',
content: {
'application/json': {
schema: SandboxUploadResponseSchema
}
}
}
}
}
},
'/core/ai/sandbox/getHtmlPreviewLink': {
post: {
summary: '获取 HTML 文件预览链接',
......
......@@ -121,7 +121,11 @@ export function getSandboxAdapterConfig({
sessionId,
workDirectory: profile.workDirectory,
ideAgentBindAddr: serviceEnv.IDE_AGENT_BIND_ADDR,
ideAgentMaxFileBytes: getAgentSandboxMaxFileBytes()
ideAgentMaxFileBytes: getAgentSandboxMaxFileBytes(),
ideAgentWsLimits: {
maxMessageBytes: serviceEnv.AGENT_SANDBOX_WS_MAX_MESSAGE_BYTES,
maxFrameBytes: serviceEnv.AGENT_SANDBOX_WS_MAX_FRAME_BYTES
}
})
: undefined;
......
......@@ -57,18 +57,25 @@ export function buildBaseSandboxRuntimeEnv({
sessionId,
workDirectory,
ideAgentBindAddr,
ideAgentMaxFileBytes
ideAgentMaxFileBytes,
ideAgentWsLimits
}: {
sessionId: string;
workDirectory: string;
ideAgentBindAddr: string;
ideAgentMaxFileBytes: number;
ideAgentWsLimits: {
maxMessageBytes: number;
maxFrameBytes: number;
};
}): Record<string, string> {
return {
FASTGPT_SESSION_ID: sessionId,
FASTGPT_WORKDIR: workDirectory,
IDE_AGENT_ENABLED: 'true',
IDE_AGENT_BIND_ADDR: ideAgentBindAddr,
FASTGPT_IDE_MAX_FILE_BYTES: String(ideAgentMaxFileBytes)
FASTGPT_IDE_MAX_FILE_BYTES: String(ideAgentMaxFileBytes),
FASTGPT_IDE_WS_MAX_MESSAGE_BYTES: String(ideAgentWsLimits.maxMessageBytes),
FASTGPT_IDE_WS_MAX_FRAME_BYTES: String(ideAgentWsLimits.maxFrameBytes)
};
}
......@@ -102,6 +102,18 @@ export const serviceEnv = createEnv({
AGENT_SANDBOX_ENTRYPOINT_TIMEOUT_SECONDS: IntSchema.min(1).max(600).default(30).meta({
description: 'Agent sandbox entrypoint 执行超时时间(秒)'
}),
AGENT_SANDBOX_WS_MAX_MESSAGE_BYTES: IntSchema.min(1)
.default(64 * 1024 * 1024)
.meta({
description:
'Agent sandbox WebSocket 单消息上限(字节),由 FastGPT app 统一下发给 proxy/IDE Agent'
}),
AGENT_SANDBOX_WS_MAX_FRAME_BYTES: IntSchema.min(1)
.default(16 * 1024 * 1024)
.meta({
description:
'Agent sandbox WebSocket 单帧上限(字节),由 FastGPT app 统一下发给 proxy/IDE Agent'
}),
AGENT_SANDBOX_NPM_REGISTRY: z.string().optional(),
AGENT_SANDBOX_PYPI_INDEX_URL: z.string().optional(),
......
......@@ -13,6 +13,8 @@ const originalEnv = {
AGENT_SANDBOX_OPENSANDBOX_IMAGE_REPO: process.env.AGENT_SANDBOX_OPENSANDBOX_IMAGE_REPO,
AGENT_SANDBOX_OPENSANDBOX_IMAGE_TAG: process.env.AGENT_SANDBOX_OPENSANDBOX_IMAGE_TAG,
AGENT_SANDBOX_DISK_MB: process.env.AGENT_SANDBOX_DISK_MB,
AGENT_SANDBOX_WS_MAX_MESSAGE_BYTES: process.env.AGENT_SANDBOX_WS_MAX_MESSAGE_BYTES,
AGENT_SANDBOX_WS_MAX_FRAME_BYTES: process.env.AGENT_SANDBOX_WS_MAX_FRAME_BYTES,
AGENT_SANDBOX_PROXY_SECRET: process.env.AGENT_SANDBOX_PROXY_SECRET
};
......@@ -63,6 +65,11 @@ describe('sandbox provider config', () => {
originalEnv.AGENT_SANDBOX_OPENSANDBOX_IMAGE_TAG
);
vi.stubEnv('AGENT_SANDBOX_DISK_MB', originalEnv.AGENT_SANDBOX_DISK_MB);
vi.stubEnv(
'AGENT_SANDBOX_WS_MAX_MESSAGE_BYTES',
originalEnv.AGENT_SANDBOX_WS_MAX_MESSAGE_BYTES
);
vi.stubEnv('AGENT_SANDBOX_WS_MAX_FRAME_BYTES', originalEnv.AGENT_SANDBOX_WS_MAX_FRAME_BYTES);
vi.stubEnv('AGENT_SANDBOX_PROXY_SECRET', originalEnv.AGENT_SANDBOX_PROXY_SECRET);
vi.unstubAllGlobals();
});
......@@ -135,6 +142,8 @@ describe('sandbox provider config', () => {
vi.stubEnv('AGENT_SANDBOX_SEALOS_BASEURL', 'https://devbox.example.com');
vi.stubEnv('AGENT_SANDBOX_SEALOS_TOKEN', 'sealos-token');
vi.stubEnv('AGENT_SANDBOX_SEALOS_WORK_DIRECTORY', '/home/devbox/workspace');
vi.stubEnv('AGENT_SANDBOX_WS_MAX_MESSAGE_BYTES', '67108864');
vi.stubEnv('AGENT_SANDBOX_WS_MAX_FRAME_BYTES', '16777216');
const { getSandboxAdapterConfig } = await loadSandboxConfigModule();
......@@ -162,7 +171,9 @@ describe('sandbox provider config', () => {
FASTGPT_WORKDIR: '/home/devbox/workspace',
IDE_AGENT_ENABLED: 'true',
IDE_AGENT_BIND_ADDR: '0.0.0.0:1318',
FASTGPT_IDE_MAX_FILE_BYTES: '536870912'
FASTGPT_IDE_MAX_FILE_BYTES: '536870912',
FASTGPT_IDE_WS_MAX_MESSAGE_BYTES: '67108864',
FASTGPT_IDE_WS_MAX_FRAME_BYTES: '16777216'
}
});
......
......@@ -91,7 +91,11 @@ describe('sandbox runtime profile', () => {
sessionId: 'session-1',
workDirectory: runtimeProfile.workDirectory,
ideAgentBindAddr: '0.0.0.0:1318',
ideAgentMaxFileBytes: 10 * 1024 * 1024
ideAgentMaxFileBytes: 10 * 1024 * 1024,
ideAgentWsLimits: {
maxMessageBytes: 64 * 1024 * 1024,
maxFrameBytes: 16 * 1024 * 1024
}
}),
metadata: { teamId: 'team-1' }
})
......@@ -101,7 +105,9 @@ describe('sandbox runtime profile', () => {
FASTGPT_WORKDIR: '/custom/devbox/workspace',
IDE_AGENT_ENABLED: 'true',
IDE_AGENT_BIND_ADDR: '0.0.0.0:1318',
FASTGPT_IDE_MAX_FILE_BYTES: '10485760'
FASTGPT_IDE_MAX_FILE_BYTES: '10485760',
FASTGPT_IDE_WS_MAX_MESSAGE_BYTES: '67108864',
FASTGPT_IDE_WS_MAX_FRAME_BYTES: '16777216'
},
metadata: {
teamId: 'team-1'
......
......@@ -5,7 +5,9 @@ import type {
SandboxGetTicketBody,
SandboxGetTicketResponse,
SandboxGetHtmlPreviewLinkBody,
SandboxGetHtmlPreviewLinkResponse
SandboxGetHtmlPreviewLinkResponse,
SandboxUploadBody,
SandboxUploadResponse
} from '@fastgpt/global/openapi/core/ai/sandbox/api';
import { parseContentDispositionFilename } from '@fastgpt/global/common/file/tools';
import { POST } from '@/web/common/api/request';
......@@ -25,6 +27,7 @@ type SandboxDownloadClientBody = SandboxClientBody<SandboxDownloadBody>;
type SandboxCheckExistClientBody = SandboxClientBody<SandboxCheckExistBody>;
type SandboxGetTicketClientBody = SandboxClientBody<SandboxGetTicketBody>;
type SandboxGetHtmlPreviewLinkClientBody = SandboxClientBody<SandboxGetHtmlPreviewLinkBody>;
type SandboxUploadClientBody = SandboxClientBody<SandboxUploadBody>;
/**
* share 模式下后端 schema 要求只传 outLinkAuthData,真实 appId 由鉴权解析。
......@@ -63,13 +66,16 @@ export const getSandboxProxyWsUrl = ({
/**
* 下载文件或目录(强制下载)
*/
export const downloadSandbox = async (data: SandboxDownloadClientBody) => {
const response = await fetch('/api/core/ai/sandbox/download', {
const fetchSandboxDownloadResponse = (data: SandboxDownloadClientBody) =>
fetch('/api/core/ai/sandbox/download', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(normalizeSandboxRequest(data))
});
export const downloadSandbox = async (data: SandboxDownloadClientBody) => {
const response = await fetchSandboxDownloadResponse(data);
if (!response.ok) {
throw new Error('Download failed');
}
......@@ -91,6 +97,19 @@ export const downloadSandbox = async (data: SandboxDownloadClientBody) => {
};
/**
* 读取沙盒文件原始字节,供大文件绕过 ide-agent WebSocket JSON RPC 读取。
*/
export const readSandboxFile = async (data: SandboxDownloadClientBody) => {
const response = await fetchSandboxDownloadResponse(data);
if (!response.ok) {
throw new Error('Read failed');
}
return new Uint8Array(await response.arrayBuffer());
};
/**
* 检查沙盒是否存在
*/
export const checkSandboxExist = async (data: SandboxCheckExistClientBody) =>
......@@ -107,3 +126,19 @@ export const getHtmlPreviewLink = (data: SandboxGetHtmlPreviewLinkClientBody) =>
export const getSandboxTicket = async (data: SandboxGetTicketClientBody) =>
POST<SandboxGetTicketResponse>('/core/ai/sandbox/getTicket', normalizeSandboxRequest(data));
/**
* 通过主站 API 复用 sandbox provider 文件上传能力,避免走 ide-agent WebSocket base64。
*/
export const uploadSandboxFile = async ({
file,
...data
}: SandboxUploadClientBody & { file: File }) => {
const formData = new FormData();
formData.append('file', file);
formData.append('data', JSON.stringify(normalizeSandboxRequest(data)));
return POST<SandboxUploadResponse>('/core/ai/sandbox/upload', formData, {
timeout: 10 * 60 * 1000
});
};
......@@ -87,6 +87,9 @@ const EditorContent = ({
const renderFileContent = () => {
if (!activeFile) return null;
const editorReadOnly = !canWrite || !!activeFile.readOnly;
if (activeFile.isLoading) return null;
// 非媒体文件 UTF-8 解码失败 → 走兜底(如 xlsx/zip 等真二进制)
if (activeFile.isUnknown) {
......@@ -190,7 +193,7 @@ const EditorContent = ({
fontFamily: "'Monaco', 'Menlo', 'Consolas', 'Courier New', monospace",
tabSize: 2,
wordWrap: 'on',
readOnly: !canWrite,
readOnly: editorReadOnly,
smoothScrolling: true,
cursorBlinking: 'smooth',
renderLineHighlight: 'line',
......@@ -204,7 +207,7 @@ const EditorContent = ({
// 保存快捷键 Ctrl/Cmd + S
editor.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyS, () => {
if (!canWrite) return;
if (editorReadOnly) return;
saveFile();
});
......@@ -212,7 +215,7 @@ const EditorContent = ({
editor.addCommand(
monaco.KeyMod.CtrlCmd | monaco.KeyMod.Shift | monaco.KeyCode.KeyS,
() => {
if (!canWrite) return;
if (editorReadOnly) return;
openedFilesRef.current?.forEach((file) => {
if (file.isDirty) {
saveFile(file.path);
......@@ -222,7 +225,7 @@ const EditorContent = ({
);
}}
onChange={(value) => {
if (!canWrite) return;
if (editorReadOnly) return;
// 更新当前文件内容
if (activeFilePath && value !== undefined && value !== activeFile?.content) {
setOpenedFiles((prev) =>
......@@ -246,7 +249,9 @@ const EditorContent = ({
</Box>
{activeFile && (
<Flex alignItems={'center'} h={'20px'}>
{activeFile.isDirty ? (
{activeFile.isLoading ? (
<MyIcon name={'common/loading'} w={'12px'} color={'myGray.500'} />
) : activeFile.isDirty ? (
<Flex alignItems={'center'} gap={1} color={'myGray.500'} fontSize={'xs'}>
<MyIcon name={'common/loading'} w={'12px'} />
<Box>{t('common:core.app.saving')}</Box>
......
......@@ -10,9 +10,12 @@ export type OpenedFile = {
language: string;
isBinary: boolean;
isDirty: boolean;
isLoading?: boolean;
// 通过 HTTP stream read 打开的文件,避免编辑保存时再次走 ide-agent WebSocket。
readOnly?: boolean;
// 非媒体文件 UTF-8 解码失败时为 true,前端走「无法预览」兜底
isUnknown?: boolean;
mtime?: number;
etag?: string;
};
type Props = {
......@@ -98,7 +101,7 @@ const FileTabs = ({ openedFiles, activeFilePath, setActiveFilePath, closeFile }:
userSelect={'none'}
>
<MyIcon
name={getIconByFilename(file.name)}
name={file.isLoading ? 'common/loading' : getIconByFilename(file.name)}
fill="none"
w="16px"
h="16px"
......
......@@ -76,6 +76,7 @@ export type FileItem = {
path: string;
type: 'file' | 'directory';
size?: number;
mtime?: number;
};
export type TreeNode = FileItem & {
......@@ -178,12 +179,14 @@ const DroppableRootBox = ({
children,
activeNode,
realOverDestPath,
onContextMenu
onContextMenu,
onPointerDown
}: {
children: React.ReactNode;
activeNode: TreeNode | null;
realOverDestPath: string | null;
onContextMenu: (e: React.MouseEvent) => void;
onPointerDown: (e: React.PointerEvent) => void;
}) => {
const { setNodeRef, isOver } = useDroppable({
id: '.'
......@@ -210,6 +213,7 @@ const DroppableRootBox = ({
}
borderRadius="6px"
onContextMenu={onContextMenu}
onPointerDown={onPointerDown}
>
{children}
</Box>
......@@ -259,10 +263,11 @@ const FileTree = ({
const [activeNode, setActiveNode] = useState<TreeNode | null>(null);
const [activeOverPath, setActiveOverPath] = useState<string | null>(null);
const [selectedPaths, setSelectedPaths] = useState<Set<string>>(new Set());
const effectiveSelectedPaths =
enableMultiSelect && selectedPath && !selectedPaths.has(selectedPath)
? new Set([selectedPath])
: selectedPaths;
const effectiveSelectedPaths = (() => {
if (!enableMultiSelect) return selectedPaths;
if (!selectedPath) return new Set<string>();
return selectedPaths.has(selectedPath) ? selectedPaths : new Set([selectedPath]);
})();
const getOperationSelectedPaths = (basePath: string) => {
if (enableMultiSelect && effectiveSelectedPaths.has(basePath)) {
......@@ -337,8 +342,23 @@ const FileTree = ({
const [isUploading, setIsUploading] = useState(false);
const fileInputRef = useRef<HTMLInputElement>(null);
const treeRootRef = useRef<HTMLDivElement>(null);
const menuRef = useRef<HTMLDivElement>(null);
useEffect(() => {
const handleOutsidePointerDown = (e: PointerEvent) => {
const target = e.target as Node | null;
if (!target) return;
if (treeRootRef.current?.contains(target) || menuRef.current?.contains(target)) return;
setSelectedPath('');
setSelectedPaths(new Set());
};
window.addEventListener('pointerdown', handleOutsidePointerDown, true);
return () => window.removeEventListener('pointerdown', handleOutsidePointerDown, true);
}, [setSelectedPath]);
// 点击任意地方关闭右键菜单(支持捕获阶段并过滤菜单内部点击,防止阻止冒泡导致菜单无法关闭)
useEffect(() => {
if (!contextMenu) return;
......@@ -566,6 +586,14 @@ const FileTree = ({
});
};
const handleTreeBlankPointerDown = (e: React.PointerEvent) => {
const target = e.target as HTMLElement | null;
if (!target || target.closest('[data-file-tree-node="true"]')) return;
setSelectedPath('');
setSelectedPaths(new Set());
};
const handleCtxCreateFile = async () => {
if (!contextMenu) return;
const node = contextMenu.node;
......@@ -789,6 +817,7 @@ const FileTree = ({
return (
<Box
ref={treeRootRef}
flex="1"
w="100%"
h="full"
......@@ -957,6 +986,7 @@ const FileTree = ({
activeNode={activeNode}
realOverDestPath={realOverDestPath}
onContextMenu={handleBlankContextMenu}
onPointerDown={handleTreeBlankPointerDown}
>
{isLoading ? (
<FileTreeSkeleton />
......
......@@ -98,6 +98,7 @@ const FileTreeNode = ({
setDragRef(el);
setDropRef(el);
}}
data-file-tree-node="true"
pl={`${node.level * 16 + 4}px`}
pr={2}
h="28px"
......
import type { NextApiResponse } from 'next';
import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { NextAPI } from '@/service/middleware/entry';
import { type ApiRequestProps } from '@fastgpt/service/type/next';
import {
......@@ -14,7 +16,7 @@ import { SandboxDownloadBodySchema } from '@fastgpt/global/openapi/core/ai/sandb
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import {
isSandboxPathDirectory,
getSandboxFileContent,
resolveSandboxWorkspacePath,
addDirectoryToArchive
} from '@fastgpt/service/core/ai/sandbox/interface/file';
......@@ -59,6 +61,38 @@ export const writeDirectoryArchiveResponse = async ({
}
};
const writeFileStreamResponse = async ({
sandbox,
res,
path
}: {
sandbox: SandboxClient;
res: NextApiResponse;
path: string;
}) => {
const providerPath = resolveSandboxWorkspacePath(path);
const fileInfoMap = await sandbox.provider.getFileInfo([providerPath]).catch(() => undefined);
const fileInfo = fileInfoMap?.get(providerPath);
if (fileInfo?.isDirectory) {
return Promise.reject('Cannot read a directory as a file');
}
const fileName = providerPath.split('/').pop() || 'file';
const encodedFileName = encodeURIComponent(fileName);
res.setHeader('Content-Type', 'application/octet-stream');
res.setHeader(
'Content-Disposition',
`attachment; filename="${encodedFileName}"; filename*=UTF-8''${encodedFileName}`
);
if (typeof fileInfo?.size === 'number') {
res.setHeader('Content-Length', String(fileInfo.size));
}
await pipeline(Readable.from(sandbox.provider.readFileStream(providerPath)), res);
};
async function handler(req: ApiRequestProps, res: NextApiResponse): Promise<void> {
const { sourceType, sourceId, chatId, path, outLinkAuthData } = parseApiInput({
req,
......@@ -110,15 +144,7 @@ async function handler(req: ApiRequestProps, res: NextApiResponse): Promise<void
path
});
} else {
const { content, fileName } = await getSandboxFileContent(sandbox, path, false);
const encodedFileName = encodeURIComponent(fileName);
res.setHeader('Content-Type', 'application/octet-stream');
res.setHeader(
'Content-Disposition',
`attachment; filename="${encodedFileName}"; filename*=UTF-8''${encodedFileName}`
);
res.send(content);
await writeFileStreamResponse({ sandbox, res, path });
}
}
......
import { NextAPI } from '@/service/middleware/entry';
import { type ApiRequestProps } from '@fastgpt/service/type/next';
import {
authSandboxSession,
buildSandboxClientQueryFromChatSource
} from '@/service/core/sandbox/auth';
import { multer } from '@fastgpt/service/common/file/multer';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import {
SandboxUploadBodySchema,
SandboxUploadResponseSchema,
type SandboxUploadResponse
} from '@fastgpt/global/openapi/core/ai/sandbox/api';
import { getAgentSandboxMaxFileBytes } from '@fastgpt/service/core/ai/sandbox/interface/config';
import { getSandboxClient } from '@fastgpt/service/core/ai/sandbox/interface/runtime';
import { resolveSandboxWorkspacePath } from '@fastgpt/service/core/ai/sandbox/interface/file';
import { Readable } from 'node:stream';
async function handler(req: ApiRequestProps): Promise<SandboxUploadResponse> {
const contentType = req.headers['content-type'] ?? '';
if (!contentType.includes('multipart/form-data')) {
return Promise.reject('Content-Type must be multipart/form-data');
}
const filepaths: string[] = [];
try {
const maxFileBytes = getAgentSandboxMaxFileBytes();
const form = await multer.resolveFormData({
request: req,
maxFileSize: Math.ceil(maxFileBytes / 1024 / 1024)
});
if (form.fileMetadata.path) {
filepaths.push(form.fileMetadata.path);
}
const { sourceType, sourceId, chatId, path, outLinkAuthData } = parseApiInput({
req: { body: form.data },
bodySchema: SandboxUploadBodySchema
}).body;
if (form.fileMetadata.size > maxFileBytes) {
return Promise.reject(
`File is too large (${form.fileMetadata.size} bytes > ${maxFileBytes} bytes)`
);
}
const {
uid,
sourceType: resolvedSourceType,
sourceId: resolvedSourceId
} = await authSandboxSession({
req,
sourceType,
sourceId,
chatId,
outLinkAuthData,
per: WritePermissionVal
});
const sandbox = await getSandboxClient(
buildSandboxClientQueryFromChatSource({
sourceType: resolvedSourceType,
sourceId: resolvedSourceId,
userId: uid,
chatId
}),
{
failedArchivePolicy: 'clearAndContinue'
}
);
const providerPath = resolveSandboxWorkspacePath(path);
const [writeResult] = await sandbox.provider.writeFiles([
{
path: providerPath,
data: Readable.toWeb(form.getReadStream()) as ReadableStream<Uint8Array>
}
]);
if (!writeResult || writeResult.error) {
return Promise.reject(
`Failed to upload file: ${writeResult?.error?.message || 'unknown error'}`
);
}
return SandboxUploadResponseSchema.parse({
path,
bytesWritten: writeResult.bytesWritten || form.fileMetadata.size
});
} finally {
multer.clearDiskTempFiles(filepaths);
}
}
export default NextAPI(handler);
export const config = {
api: {
bodyParser: false
}
};
......@@ -8,6 +8,7 @@ import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import jwt from 'jsonwebtoken';
import { z } from 'zod';
import { authAgentSandboxProxy } from '@/service/core/sandbox/auth';
import { IntSchema } from '@fastgpt/global/common/zod';
import {
SandboxChannelSchema,
SandboxTicketPermissionSchema
......@@ -23,6 +24,16 @@ const VerifyTicketQuerySchema = z.object({
ticket: z.string()
});
const SandboxVerifyTicketResponseSchema = z.object({
sandbox_url: z.string().min(1),
agent_token: z.string(),
ws_limits: z.object({
max_message_bytes: IntSchema.min(1),
max_frame_bytes: IntSchema.min(1)
})
});
type SandboxVerifyTicketResponse = z.infer<typeof SandboxVerifyTicketResponseSchema>;
const BaseSandboxTicketClaimsSchema = z.object({
userId: z.string(),
chatId: z.string(),
......@@ -73,7 +84,7 @@ async function readIdeAgentPassword(sandbox: SandboxClient) {
/**
* 校验 proxy ticket,并返回 IDE Agent 的代理连接地址和一次性 agent 口令。
*/
async function handler(req: ApiRequestProps) {
async function handler(req: ApiRequestProps): Promise<SandboxVerifyTicketResponse> {
const secret = authAgentSandboxProxy(req);
const { ticket } = parseApiInput({
......@@ -102,10 +113,14 @@ async function handler(req: ApiRequestProps) {
const endpoint = await sandbox.provider.getEndpoint(getIdeAgentPort());
return {
return SandboxVerifyTicketResponseSchema.parse({
sandbox_url: endpoint.url,
agent_token: agentPassword
};
agent_token: agentPassword,
ws_limits: {
max_message_bytes: serviceEnv.AGENT_SANDBOX_WS_MAX_MESSAGE_BYTES,
max_frame_bytes: serviceEnv.AGENT_SANDBOX_WS_MAX_FRAME_BYTES
}
});
}
export default NextAPI(handler);
import { EventEmitter } from 'node:events';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import type { NextApiResponse } from 'next';
import type { SandboxClient } from '@fastgpt/service/core/ai/sandbox/interface/runtime';
const mocks = vi.hoisted(() => ({
addDirectoryToArchive: vi.fn()
......@@ -21,7 +19,6 @@ vi.mock('@fastgpt/service/core/ai/sandbox/interface/runtime', () => ({
vi.mock('@fastgpt/service/core/ai/sandbox/interface/file', () => ({
addDirectoryToArchive: mocks.addDirectoryToArchive,
getSandboxFileContent: vi.fn(),
isSandboxPathDirectory: vi.fn()
}));
......
import { ChatSourceTypeEnum } from '@fastgpt/global/core/chat/constants';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { Readable } from 'node:stream';
const mocks = vi.hoisted(() => ({
authSandboxSession: vi.fn(),
buildSandboxClientQueryFromChatSource: vi.fn(),
clearDiskTempFiles: vi.fn(),
getAgentSandboxMaxFileBytes: vi.fn(),
getReadStream: vi.fn(),
getSandboxClient: vi.fn(),
resolveFormData: vi.fn(),
writeFiles: vi.fn()
}));
vi.mock('@/service/middleware/entry', () => ({
NextAPI: vi.fn((handler) => handler)
}));
vi.mock('@/service/core/sandbox/auth', () => ({
authSandboxSession: mocks.authSandboxSession,
buildSandboxClientQueryFromChatSource: mocks.buildSandboxClientQueryFromChatSource
}));
vi.mock('@fastgpt/service/common/file/multer', () => ({
multer: {
resolveFormData: mocks.resolveFormData,
clearDiskTempFiles: mocks.clearDiskTempFiles
}
}));
vi.mock('@fastgpt/service/core/ai/sandbox/interface/config', () => ({
getAgentSandboxMaxFileBytes: mocks.getAgentSandboxMaxFileBytes
}));
vi.mock('@fastgpt/service/core/ai/sandbox/interface/runtime', () => ({
getSandboxClient: mocks.getSandboxClient
}));
import handler from '@/pages/api/core/ai/sandbox/upload';
const createReq = () =>
({
headers: {
'content-type': 'multipart/form-data; boundary=test'
}
}) as any;
describe('sandbox upload API', () => {
beforeEach(() => {
vi.clearAllMocks();
mocks.getAgentSandboxMaxFileBytes.mockReturnValue(10 * 1024 * 1024);
mocks.getReadStream.mockReturnValue(Readable.from([new Uint8Array([1, 2, 3])]));
mocks.resolveFormData.mockResolvedValue({
data: {
appId: '507f1f77bcf86cd799439011',
chatId: 'chat-1',
path: 'uploads/a.txt'
},
fileMetadata: {
path: '/tmp/upload-a.txt',
size: 3
},
getReadStream: mocks.getReadStream
});
mocks.authSandboxSession.mockResolvedValue({
uid: 'user-1',
sourceType: ChatSourceTypeEnum.app,
sourceId: '507f1f77bcf86cd799439011'
});
mocks.buildSandboxClientQueryFromChatSource.mockReturnValue({ sandboxId: 'sandbox-1' });
mocks.writeFiles.mockResolvedValue([{ bytesWritten: 3, error: null }]);
mocks.getSandboxClient.mockResolvedValue({ provider: { writeFiles: mocks.writeFiles } });
});
it('uploads multipart file through sandbox provider after write auth', async () => {
const req = createReq();
await expect(handler(req)).resolves.toEqual({
path: 'uploads/a.txt',
bytesWritten: 3
});
expect(mocks.resolveFormData).toHaveBeenCalledWith({
request: req,
maxFileSize: 10
});
expect(mocks.authSandboxSession).toHaveBeenCalledWith({
req,
sourceType: ChatSourceTypeEnum.app,
sourceId: '507f1f77bcf86cd799439011',
chatId: 'chat-1',
outLinkAuthData: undefined,
per: WritePermissionVal
});
expect(mocks.buildSandboxClientQueryFromChatSource).toHaveBeenCalledWith({
sourceType: ChatSourceTypeEnum.app,
sourceId: '507f1f77bcf86cd799439011',
userId: 'user-1',
chatId: 'chat-1'
});
expect(mocks.getSandboxClient).toHaveBeenCalledWith(
{ sandboxId: 'sandbox-1' },
{ failedArchivePolicy: 'clearAndContinue' }
);
expect(mocks.writeFiles).toHaveBeenCalledTimes(1);
const [[writeEntry]] = mocks.writeFiles.mock.calls[0];
expect(writeEntry.path).toBe('/workspace/uploads/a.txt');
expect(writeEntry.data).toBeInstanceOf(ReadableStream);
expect(mocks.clearDiskTempFiles).toHaveBeenCalledWith(['/tmp/upload-a.txt']);
});
});
......@@ -755,6 +755,10 @@ export class OpenSandboxAdapter extends BaseSandboxAdapter {
return results;
}
override readFileStream(path: string): AsyncIterable<Uint8Array> {
return this.sandbox.files.readBytesStream(this.normalizePath(path));
}
// ==================== Command Execution ====================
async execute(command: string, options?: ExecuteOptions): Promise<ExecuteResult> {
const maxBytes = options?.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
......
......@@ -145,14 +145,20 @@ export class DevboxApi {
});
}
/** GET /api/v1/devbox/{name}/files/download */
async downloadFile(name: string, params: DownloadFileParams): Promise<ArrayBuffer> {
private buildDownloadFileUrl(name: string, params: DownloadFileParams): string {
const queryParams: Record<string, string> = { path: params.path };
if (params.filename) queryParams.filename = params.filename;
if (params.timeoutSeconds != null) queryParams.timeoutSeconds = String(params.timeoutSeconds);
if (params.container) queryParams.container = params.container;
const url = this.url(`/api/v1/devbox/${name}/files/download`, queryParams);
return this.url(`/api/v1/devbox/${name}/files/download`, queryParams);
}
private async fetchDownloadFileResponse(
name: string,
params: DownloadFileParams
): Promise<Response> {
const url = this.buildDownloadFileUrl(name, params);
const res = await fetch(url, {
headers: { Authorization: `Bearer ${this.token}` }
});
......@@ -165,6 +171,44 @@ export class DevboxApi {
url
);
}
return res;
}
/** GET /api/v1/devbox/{name}/files/download */
async downloadFile(name: string, params: DownloadFileParams): Promise<ArrayBuffer> {
const res = await this.fetchDownloadFileResponse(name, params);
return res.arrayBuffer();
}
/**
* GET /api/v1/devbox/{name}/files/download as the native HTTP response stream.
*
* This intentionally bypasses `arrayBuffer()` so callers can pipe large files without buffering
* the full payload in Node memory.
*/
downloadFileStream(name: string, params: DownloadFileParams): AsyncIterable<Uint8Array> {
return this.readDownloadFileStream(name, params);
}
private async *readDownloadFileStream(
name: string,
params: DownloadFileParams
): AsyncIterable<Uint8Array> {
const res = await this.fetchDownloadFileResponse(name, params);
const body = res.body;
if (!body) return;
const reader = body.getReader();
try {
while (true) {
const { done, value } = await reader.read();
if (done) return;
if (value) {
yield value;
}
}
} finally {
reader.releaseLock();
}
}
}
......@@ -382,6 +382,12 @@ export class SealosDevboxAdapter extends BaseSandboxAdapter {
return results;
}
override readFileStream(path: string): AsyncIterable<Uint8Array> {
return this.api.downloadFileStream(this._id, {
path: this.normalizePath(path)
});
}
override async writeFileStream(path: string, stream: ReadableStream<Uint8Array>): Promise<void> {
const normalizedPath = this.normalizePath(path);
const results = await this.writeFiles([
......
......@@ -582,6 +582,31 @@ describe('OpenSandboxAdapter', () => {
});
});
describe('readFileStream', () => {
it('should stream file through OpenSandbox native readBytesStream', async () => {
const adapter = makeAdapter();
async function* streamChunks() {
yield new TextEncoder().encode('native ');
yield new TextEncoder().encode('stream');
}
const mockReadBytesStream = vi.fn(() => streamChunks());
(adapter as any).sandbox = {
files: {
readBytesStream: mockReadBytesStream
}
};
const received: Uint8Array[] = [];
for await (const chunk of adapter.readFileStream('test.txt')) {
received.push(chunk);
}
expect(new TextDecoder().decode(Buffer.concat(received))).toBe('native stream');
expect(mockReadBytesStream).toHaveBeenCalledWith('/workspace/test.txt');
});
});
describe('writeFiles', () => {
it('should slice Uint8Array with byte offset and pass clean ArrayBuffer to SDK', async () => {
const adapter = makeAdapter();
......
......@@ -579,4 +579,42 @@ describe('SealosDevboxAdapter', () => {
})
);
});
it('should stream file through SealosDevbox native download response body', async () => {
const chunks = [new TextEncoder().encode('native '), new TextEncoder().encode('stream')];
const arrayBuffer = vi.fn();
const fetchMock = vi.fn(async () => ({
ok: true,
body: new ReadableStream<Uint8Array>({
start(controller) {
for (const chunk of chunks) {
controller.enqueue(chunk);
}
controller.close();
}
}),
arrayBuffer
}));
vi.stubGlobal('fetch', fetchMock);
const adapter = new SealosDevboxAdapter(CONFIG, {
workingDir: '/home/devbox/workspace'
});
const received: Uint8Array[] = [];
for await (const chunk of adapter.readFileStream('test.txt')) {
received.push(chunk);
}
expect(new TextDecoder().decode(Buffer.concat(received))).toBe('native stream');
expect(arrayBuffer).not.toHaveBeenCalled();
expect(fetchMock).toHaveBeenCalledWith(
'https://devbox-server.example.com/api/v1/devbox/devbox-1/files/download?path=%2Fhome%2Fdevbox%2Fworkspace%2Ftest.txt',
expect.objectContaining({
headers: {
Authorization: 'Bearer test-token'
}
})
);
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment