Skip to content

Shucai Chen / Upgrade-lufa-fastgpt

Go to a project
Commit 91ea5278 by 赵月辉
Options

fix: 修复 fastgpt-merged 项目无法启动的问题 

1. pnpm 版本降级 11.7.0→10.6.5 兼容 Node 20
2. 修复 husky prepare 脚本路径
3. 用 FastGPT-main 版本替换过时的 worker 模块(countGptMessagesTokens/htmlStr2Md/readFile)
4. 新增缺失源文件(tokenWorkerConfig.ts, count.ts, base64ImageUpload.ts 等)
5. 添加 OpenTelemetry 依赖到 service
6. 修复 lodash→lodash-es 导入
7. 重写 tool/api.ts 使用 runPluginToolStream 替代废弃的 @fastgpt-sdk/plugin
8. 修复 Turbopack 打包 vm2/proxy-agent 问题
9. 补全缺失环境变量
10. 移除 pnpm-workspace.yaml 中无效的 pro/ 引用和 allowBuilds 配置
parent 5ce0d118
Showing with 1115 additions and 297 deletions
fastgpt-merged/package.json
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
"private": true, "private": true,
"scripts": { "scripts": {
"dev:pro": "turbo run dev:pro --filter=@fastgpt/app", "dev:pro": "turbo run dev:pro --filter=@fastgpt/app",
"prepare": "husky install", "prepare": "cd .. && husky install fastgpt-merged/.husky",
"gen:theme-typings": "chakra-cli tokens packages/web/styles/theme.ts --out node_modules/.pnpm/node_modules/@chakra-ui/styled-system/dist/theming.types.d.ts", "gen:theme-typings": "chakra-cli tokens packages/web/styles/theme.ts --out node_modules/.pnpm/node_modules/@chakra-ui/styled-system/dist/theming.types.d.ts",
"gen:deploy": "node ./deploy/init.mjs", "gen:deploy": "node ./deploy/init.mjs",
"postinstall": "pnpm gen:theme-typings && pnpm run build:sdks", "postinstall": "pnpm gen:theme-typings && pnpm run build:sdks",
...@@ -55,5 +55,5 @@ ...@@ -55,5 +55,5 @@
"node": ">=20.19.0", "node": ">=20.19.0",
"pnpm": ">=10" "pnpm": ">=10"
}, },
"packageManager": "pnpm@11.7.0" "packageManager": "pnpm@10.6.5"
} }
fastgpt-merged/packages/service/core/app/plugin/controller.ts
...@@ -45,7 +45,7 @@ import { splitCombinePluginId } from '@fastgpt/global/core/app/plugin/utils'; ...@@ -45,7 +45,7 @@ import { splitCombinePluginId } from '@fastgpt/global/core/app/plugin/utils';
import { getMCPParentId, getMCPToolRuntimeNode } from '@fastgpt/global/core/app/mcpTools/utils'; import { getMCPParentId, getMCPToolRuntimeNode } from '@fastgpt/global/core/app/mcpTools/utils';
import { AppTypeEnum } from '@fastgpt/global/core/app/constants'; import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
import { getMCPChildren } from '../mcp'; import { getMCPChildren } from '../mcp';
import { cloneDeep } from 'lodash'; import { cloneDeep } from 'lodash-es';
import { UserError } from '@fastgpt/global/common/error/utils'; import { UserError } from '@fastgpt/global/common/error/utils';
type ChildAppType = SystemPluginTemplateItemType & { type ChildAppType = SystemPluginTemplateItemType & {
......
fastgpt-merged/packages/service/core/app/tool/api.ts
import { RunToolWithStream } from '@fastgpt-sdk/plugin';
import { PluginSourceEnum } from '@fastgpt/global/core/app/plugin/constants'; import { PluginSourceEnum } from '@fastgpt/global/core/app/plugin/constants';
import { pluginClient, BASE_URL, TOKEN } from '../../../thirdProvider/fastgptPlugin'; import {
pluginClient,
PLUGIN_BASE_URL,
runPluginToolStream
} from '../../../thirdProvider/fastgptPlugin';
export async function APIGetSystemToolList() { export async function APIGetSystemToolList() {
// 检查插件服务是否可用 // 检查插件服务是否可用
if (!BASE_URL) { if (!PLUGIN_BASE_URL) {
console.log('Plugin service not configured, returning empty tool list'); console.log('Plugin service not configured, returning empty tool list');
return []; return [];
} }
...@@ -33,13 +36,6 @@ export async function APIGetSystemToolList() { ...@@ -33,13 +36,6 @@ export async function APIGetSystemToolList() {
} }
} }
const runToolInstance = BASE_URL
? new RunToolWithStream({
baseUrl: BASE_URL,
token: TOKEN
})
: null;
export const APIRunSystemTool = async (params: { export const APIRunSystemTool = async (params: {
toolId: string; toolId: string;
inputs: Record<string, any>; inputs: Record<string, any>;
...@@ -65,8 +61,19 @@ export const APIRunSystemTool = async (params: { ...@@ -65,8 +61,19 @@ export const APIRunSystemTool = async (params: {
}; };
onMessage: (message: { type: string; content: string }) => void; onMessage: (message: { type: string; content: string }) => void;
}) => { }) => {
if (!runToolInstance) { if (!PLUGIN_BASE_URL) {
throw new Error('Plugin service not configured'); throw new Error('Plugin service not configured');
} }
return runToolInstance.run(params);
return runPluginToolStream({
pluginId: params.toolId,
input: params.inputs,
systemVar: params.systemVar,
onMessage: (message) => {
params.onMessage({
type: 'stream',
content: JSON.stringify(message)
});
}
});
}; };
fastgpt-merged/packages/service/package.json
...@@ -26,6 +26,10 @@ ...@@ -26,6 +26,10 @@
"@modelcontextprotocol/sdk": "catalog:", "@modelcontextprotocol/sdk": "catalog:",
"@node-rs/jieba": "catalog:", "@node-rs/jieba": "catalog:",
"@opentelemetry/api": "^1.9.0", "@opentelemetry/api": "^1.9.0",
"@opentelemetry/api-logs": "^0.203.0",
"@opentelemetry/exporter-logs-otlp-http": "^0.203.0",
"@opentelemetry/resources": "^2.0.1",
"@opentelemetry/sdk-logs": "^0.203.0",
"@t3-oss/env-core": "catalog:", "@t3-oss/env-core": "catalog:",
"@xmldom/xmldom": "^0.8.10", "@xmldom/xmldom": "^0.8.10",
"@zilliz/milvus2-sdk-node": "2.4.10", "@zilliz/milvus2-sdk-node": "2.4.10",
......
fastgpt-merged/packages/service/worker/countGptMessagesTokens/count.ts 0 → 100644
import {
type ChatCompletionContentPart,
type ChatCompletionCreateParams,
type ChatCompletionMessageParam,
type ChatCompletionTool
} from '@fastgpt/global/core/ai/llm/type';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
import o200kTokenizer from 'gpt-tokenizer/encoding/o200k_base';
export type CountGptMessagesTokensParams = {
messages: ChatCompletionMessageParam[];
tools?: ChatCompletionTool[];
functionCall?: ChatCompletionCreateParams.Function[];
};
type TokenizerApi = {
countTokens: (
input: string,
options?: {
disallowedSpecial?: Set<string> | 'all';
allowedSpecial?: Set<string> | 'all';
}
) => number;
};
const tokenizer: TokenizerApi = o200kTokenizer;
const noDisallowedSpecial = { disallowedSpecial: new Set<string>() };
/**
* FastGPT 的 worker token 计数统一使用 GPT 现代模型的 o200k_base 编码。
* 该路径只做上下文预算和缺 usage 时的近似兜底；供应商返回 usage 时仍以 usage 为准。
*/
export const GPT_TOKENIZER_ENCODING = 'o200k_base';
type CountableContentPart = ChatCompletionContentPart | { type: 'refusal'; refusal: string };
/**
* 将多模态 content part 转成可计数文本。
*
* 这里不尝试复刻各家模型对图片、音频、文件的精确计费规则，只把会进入上下文或
* 明显影响输入规模的字段纳入估算；真实计费仍以模型供应商返回的 usage 为准。
*/
const contentPartToText = (part: CountableContentPart) => {
if (part.type === 'text') return part.text;
if (part.type === 'image_url') return part.image_url.url;
if (part.type === 'input_audio') return part.input_audio.data;
if (part.type === 'file')
return [part.file.filename, part.file.file_id, part.file.file_data].filter(Boolean).join(' ');
if (part.type === 'file_url') return [part.name, part.url].filter(Boolean).join(' ');
if (part.type === 'refusal') return part.refusal;
return '';
};
/**
* 统一把 OpenAI chat content 规整为字符串。
*
* 字符串 content 直接计数；数组 content 按 part 拼接，保持和旧方案一致的“近似预算”
* 语义，避免在不同消息类型间引入额外分隔符导致历史 token 预算明显漂移。
*/
const contentToText = (content: ChatCompletionMessageParam['content'] = '') => {
if (!content) return '';
if (typeof content === 'string') return content;
return (content as CountableContentPart[]).map(contentPartToText).join('');
};
const countTextTokens = (text: string) => {
try {
return tokenizer.countTokens(text, noDisallowedSpecial);
} catch {
// tokenizer 对极少数非法 special token 组合可能抛错，退回字符数保证计费链路不断。
return text.length;
}
};
/**
* 统计普通 prompt 文本 token 数。
*
* 该函数只在 token worker 内执行，用于知识库裁剪、embedding/rerank 兜底计费等
* 近似场景，统一按 o200k_base 估算。
*/
export const countPromptTokensInWorker = (
prompt: string | ChatCompletionContentPart[] | null | undefined = '',
role: '' | `${ChatCompletionRequestMessageRoleEnum}` = ''
) => {
const promptText =
typeof prompt === 'string' || !prompt ? prompt || '' : prompt.map(contentPartToText).join('');
const text = `${role}\n${promptText}`.trim();
// 兼容旧实现：只有传入 role 时才补 chat message 的固定结构开销。
const supplementaryToken = role ? 4 : 0;
return countTextTokens(text) + supplementaryToken;
};
const countToolsTokens = (tools?: ChatCompletionTool[] | ChatCompletionCreateParams.Function[]) => {
if (!tools || tools.length === 0) return 0;
// 旧方案也是把工具 schema 规整成紧凑文本后估算，避免格式化 JSON 的空白影响预算。
const toolText = JSON.stringify(tools)
.replace(/"/g, '')
.replace(/\n/g, '')
.replace(/( ){2,}/g, ' ');
return countTextTokens(toolText);
};
const getAssistantCallText = (message: ChatCompletionMessageParam) => {
if (message.role !== ChatCompletionRequestMessageRoleEnum.Assistant) return '';
// assistant 的 tool/function call 参数会进入模型上下文，需要和普通 content 一起计入。
const toolCallsText =
message.tool_calls
?.map((item) => `${item?.function?.name} ${item?.function?.arguments}`.trim())
?.join('') || '';
const functionCall = message.function_call;
const functionCallText = `${functionCall?.name || ''} ${functionCall?.arguments || ''}`.trim();
return `${toolCallsText}${functionCallText}`;
};
/**
* 在 token worker 内同步统计 Chat messages token 数。
*
* 这里保持旧实现的消息常数近似规则，只替换为更快的 GPT tokenizer。
* 主线程只通过 worker 调用该函数，避免主进程加载 tokenizer rank 常驻内存。
*/
export const countGptMessagesTokensInWorker = ({
messages,
tools,
functionCall
}: CountGptMessagesTokensParams) => {
return (
messages.reduce((sum, item, index) => {
// 只有最后一条消息的 reasoning_content 会继续影响后续上下文预算。
const reasoningText = index === messages.length - 1 ? item.reasoning_content || '' : '';
const contentPrompt = contentToText(item.content);
const callPrompt = getAssistantCallText(item);
const text = `${item.role}\n${reasoningText}${contentPrompt}${callPrompt}`.trim();
// 每条带 role 的 chat message 保留旧实现的固定结构开销，降低切换 tokenizer 的行为差异。
return sum + countTextTokens(text) + (item.role ? 4 : 0);
}, 0) +
countToolsTokens(tools) +
countToolsTokens(functionCall)
);
};
fastgpt-merged/packages/service/worker/countGptMessagesTokens/index.ts
/* Only the token of gpt-3.5-turbo is used */
import { Tiktoken } from 'tiktoken/lite';
import cl100k_base from './cl100k_base.json';
import { import {
type ChatCompletionMessageParam, type ChatCompletionMessageParam,
type ChatCompletionContentPart,
type ChatCompletionCreateParams, type ChatCompletionCreateParams,
type ChatCompletionTool type ChatCompletionTool
} from '@fastgpt/global/core/ai/type'; } from '@fastgpt/global/core/ai/llm/type';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
import { parentPort } from 'worker_threads'; import { parentPort } from 'worker_threads';
import { getLogger, LogCategories } from '../../common/logger';
import { countGptMessagesTokensInWorker, countPromptTokensInWorker } from './count';
const enc = new Tiktoken(cl100k_base.bpe_ranks, cl100k_base.special_tokens, cl100k_base.pat_str); const logger = getLogger(LogCategories.INFRA.WORKER);
/* count messages tokens */ type CountGptMessagesTokensWorkerPayload = {
id: string;
messages?: ChatCompletionMessageParam[];
messageGroups?: ChatCompletionMessageParam[][];
prompts?: (string | null | undefined)[];
tools?: ChatCompletionTool[];
functionCall?: ChatCompletionCreateParams.Function[];
};
/**
* Token 计数 worker 入口。
*
* 单条 messages、批量 messageGroups、批量 prompts 共用同一个 worker 文件，减少 worker
* 类型数量和初始化成本。批量请求在 worker 内同步 map，避免主线程为大量短文本反复
* postMessage，也保证返回顺序和输入顺序一致。
*/
parentPort?.on( parentPort?.on(
'message', 'message',
({ ({
id, id,
messages, messages,
messageGroups,
prompts,
tools, tools,
functionCall functionCall
}: { }: CountGptMessagesTokensWorkerPayload) => {
id: string;
messages: ChatCompletionMessageParam[];
tools?: ChatCompletionTool[];
functionCall?: ChatCompletionCreateParams.Function[];
}) => {
try { try {
/* count one prompt tokens */ const data = (() => {
const countPromptTokens = ( // 上下文裁剪会频繁计算多组 messages，批量放进一次 worker 消息能降低 IPC 开销。
prompt: string | ChatCompletionContentPart[] | null | undefined = '', if (messageGroups) {
role: '' | `${ChatCompletionRequestMessageRoleEnum}` = '' return messageGroups.map((messages) => countGptMessagesTokensInWorker({ messages }));
) => {
const promptText = (() => {
if (!prompt) return '';
if (typeof prompt === 'string') return prompt;
let promptText = '';
prompt.forEach((item) => {
if (item.type === 'text') {
promptText += item.text;
} else if (item.type === 'image_url') {
promptText += item.image_url.url;
}
});
return promptText;
})();
const text = `${role}\n${promptText}`.trim();
try {
const encodeText = enc.encode(text);
const supplementaryToken = role ? 4 : 0;
return encodeText.length + supplementaryToken;
} catch (error) {
return text.length;
} }
};
const countToolsTokens = (
tools?: ChatCompletionTool[] | ChatCompletionCreateParams.Function[]
) => {
if (!tools || tools.length === 0) return 0;
const toolText = tools
? JSON.stringify(tools)
.replace('"', '')
.replace('\n', '')
.replace(/( ){2,}/g, ' ')
: '';
return enc.encode(toolText).length;
};
const total = // embedding/rerank 等路径多为纯文本 prompt，走轻量分支可少做 chat message 拼装。
messages.reduce((sum, item, index) => { if (prompts) {
// Evaluates the text of toolcall and functioncall return prompts.map((prompt) => countPromptTokensInWorker(prompt));
const functionCallPrompt = (() => { }
let prompt = '';
if (item.role === ChatCompletionRequestMessageRoleEnum.Assistant) {
const toolCalls = item.tool_calls;
prompt +=
toolCalls
?.map((item) => `${item?.function?.name} ${item?.function?.arguments}`.trim())
?.join('') || '';
const functionCall = item.function_call;
prompt += `${functionCall?.name} ${functionCall?.arguments}`.trim();
}
return prompt;
})();
const contentPrompt = (() => {
if (!item.content) return '';
if (typeof item.content === 'string') return item.content;
return item.content
.map((item) => {
if (item.type === 'text') return item.text;
return '';
})
.join('');
})();
// Only the last message computed reasoning_text
const reasoningText = index === messages.length - 1 ? item.reasoning_text || '' : '';
return ( return countGptMessagesTokensInWorker({
sum + messages: messages || [],
countPromptTokens(`${reasoningText}${contentPrompt}${functionCallPrompt}`, item.role) tools,
); functionCall
}, 0) + });
countToolsTokens(tools) + })();
countToolsTokens(functionCall);
parentPort?.postMessage({ parentPort?.postMessage({
id, id,
type: 'success', type: 'success',
data: total data
}); });
} catch (error) { } catch (error) {
console.log(error); logger.error('Token count worker failed', { error });
parentPort?.postMessage({ parentPort?.postMessage({
id, id,
type: 'success', type: 'error',
data: 0 data: error instanceof Error ? error.message : String(error)
}); });
} }
} }
......
fastgpt-merged/packages/service/worker/env.ts 0 → 100644
import { createEnv } from '@t3-oss/env-core';
import z from 'zod';
const IntSchema = z.coerce.number<number>().int().nonnegative();
export const workerEnv = createEnv({
server: {
MAX_HTML_TRANSFORM_CHARS: IntSchema.default(1000000)
},
emptyStringAsUndefined: true,
runtimeEnv: process.env
});
fastgpt-merged/packages/service/worker/htmlStr2Md/index.ts
import { parentPort } from 'worker_threads'; import { parentPort } from 'worker_threads';
import { html2md } from './utils'; import { html2md } from './utils';
import { workerResponse } from '../controller'; import {
createWorkerUploadFileHandler,
handleWorkerUploadFileResponse,
isWorkerUploadFileResponse
} from '../utils/uploadFile';
parentPort?.on('message', (params: { html: string }) => { type IncomingMessage = {
try { id: string;
const md = html2md(params?.html || ''); html: string;
uploadImages?: boolean;
type?: 'uploadFileResult' | 'uploadFileError';
requestId?: string;
data?: any;
};
parentPort?.on('message', async (params: IncomingMessage) => {
const { id, html, requestId, data, type } = params;
workerResponse({ if (isWorkerUploadFileResponse(type)) {
parentPort, handleWorkerUploadFileResponse({
status: 'success', taskId: id,
data: md type,
requestId,
data
}); });
} catch (error) { return;
workerResponse({ }
parentPort,
status: 'error', const uploadFileHandler = createWorkerUploadFileHandler({
data: error taskId: id,
parentPort
});
try {
const md = await html2md(html || '', {
uploadFile: params.uploadImages ? uploadFileHandler.uploadFile : undefined
}); });
parentPort?.postMessage({ id, type: 'success', data: md });
} catch (error) {
parentPort?.postMessage({ id, type: 'error', data: error });
} finally {
uploadFileHandler.cleanup();
} }
}); });
fastgpt-merged/packages/service/worker/htmlStr2Md/utils.ts
import TurndownService from 'turndown'; import TurndownService from 'turndown';
import { type ImageType } from '../readFile/type'; import { simpleMarkdownText } from '@fastgpt/global/common/string/markdown';
import { matchMdImg } from '@fastgpt/global/common/string/markdown'; import { getLogger, LogCategories } from '../../common/logger';
import { getNanoid } from '@fastgpt/global/common/string/tools'; import { workerEnv } from '../env';
// @ts-ignore import { gfm } from 'joplin-turndown-plugin-gfm';
const turndownPluginGfm = require('joplin-turndown-plugin-gfm'); import { uploadBase64Image } from '../utils/base64ImageUpload';
import { type UploadFileHandler } from '../readFile/type';
const processBase64Images = (htmlContent: string) => { import { batchRun } from '@fastgpt/global/common/system/utils';
const base64Regex = /src="data:([^;]+);base64,([^"]+)"/g;
const images: ImageType[] = []; const MAX_HTML_SIZE = workerEnv.MAX_HTML_TRANSFORM_CHARS;
const logger = getLogger(LogCategories.INFRA.WORKER);
const processedHtml = htmlContent.replace(base64Regex, (match, mime, base64Data) => { const htmlBase64UploadConcurrency = 5;
const uuid = `IMAGE_${getNanoid(12)}_IMAGE`;
images.push({ const htmlBase64SrcRegex = /\bsrc\s*=\s*(["'])data:([^;]+);base64,([A-Za-z0-9+/=]+)\1/gi;
uuid,
base64: base64Data, /**
mime * HTML 转 markdown 前实时处理 base64 图片。
}); *
return `src="${uuid}"`; * 有 uploadFile 时上传为对象存储 key；没有 uploadFile 时删除 src，避免大体积 base64
}); * 进入 turndown 或被 worker 结果回传。
*/
const processBase64Images = async (
htmlContent: string,
options: {
uploadFile?: UploadFileHandler;
} = {}
) => {
const matches = Array.from(htmlContent.matchAll(htmlBase64SrcRegex));
if (matches.length === 0) return htmlContent;
const replacements = await batchRun(
matches,
async (match) => {
const [, quote, mime, base64Data] = match;
if (!options.uploadFile) {
return `src=${quote}${quote}`;
}
try {
const { key } = await uploadBase64Image({
mime,
base64: base64Data,
uploadFile: options.uploadFile
});
return `src=${quote}${key}${quote}`;
} catch (error) {
logger.warn('Failed to upload parsed HTML base64 image', { mime, error });
return `src=${quote}${quote}`;
}
},
htmlBase64UploadConcurrency
);
let result = '';
let lastIndex = 0;
for (const [matchIndex, match] of matches.entries()) {
const [fullMatch] = match;
const index = match.index ?? 0;
result += htmlContent.slice(lastIndex, index);
result += replacements[matchIndex];
lastIndex = index + fullMatch.length;
}
return { processedHtml, images }; return result + htmlContent.slice(lastIndex);
}; };
export const html2md = ( export const html2md = async (
html: string html: string,
): { options: {
uploadFile?: UploadFileHandler;
} = {}
): Promise<{
rawText: string; rawText: string;
imageList: ImageType[]; }> => {
} => {
const turndownService = new TurndownService({ const turndownService = new TurndownService({
headingStyle: 'atx', headingStyle: 'atx',
bulletListMarker: '-', bulletListMarker: '-',
...@@ -41,7 +88,7 @@ export const html2md = ( ...@@ -41,7 +88,7 @@ export const html2md = (
try { try {
turndownService.remove(['i', 'script', 'iframe', 'style']); turndownService.remove(['i', 'script', 'iframe', 'style']);
turndownService.use(turndownPluginGfm.gfm); turndownService.use(gfm);
// add custom handling for media tag // add custom handling for media tag
turndownService.addRule('media', { turndownService.addRule('media', {
...@@ -62,19 +109,28 @@ export const html2md = ( ...@@ -62,19 +109,28 @@ export const html2md = (
}); });
// Base64 img to id, otherwise it will occupy memory when going to md // Base64 img to id, otherwise it will occupy memory when going to md
const { processedHtml, images } = processBase64Images(html); const processedHtml = await processBase64Images(html, {
uploadFile: options.uploadFile
});
// if html is too large, return the original html
if (processedHtml.length > MAX_HTML_SIZE) {
return { rawText: processedHtml };
}
const md = turndownService.turndown(processedHtml); const md = turndownService.turndown(processedHtml);
const { text, imageList } = matchMdImg(md);
return { return {
rawText: text, rawText: simpleMarkdownText(md)
imageList: [...images, ...imageList]
}; };
} catch (error) { } catch (error) {
console.log('html 2 markdown error', error); if (options.uploadFile) {
throw error;
}
logger.error('HTML to markdown conversion failed', { error });
return { return {
rawText: '', rawText: ''
imageList: []
}; };
} }
}; };
fastgpt-merged/packages/service/worker/readFile/extension/rawText.ts
import iconv from 'iconv-lite'; import iconv from 'iconv-lite';
import { type ReadRawTextByBuffer, type ReadFileResponse } from '../type'; import { type ReadRawTextByBuffer, type ReadFileResponse, type UploadFileHandler } from '../type';
import { matchMdImg } from '@fastgpt/global/common/string/markdown'; import { parseMarkdownBase64Images } from '@fastgpt/global/common/string/markdown';
import { uploadBase64Image } from '../../utils/base64ImageUpload';
const hasNonAsciiByte = (buffer: Buffer) => {
for (let i = 0; i < buffer.length; i++) {
if (buffer[i] > 0x7f) return true;
}
return false;
};
const rawEncodingList = [ const rawEncodingList = [
'ascii', 'ascii',
...@@ -18,30 +26,48 @@ const rawEncodingList = [ ...@@ -18,30 +26,48 @@ const rawEncodingList = [
]; ];
// 加载源文件内容 // 加载源文件内容
export const readFileRawText = async ({ export const readFileRawText = async (
buffer, { buffer, encoding }: ReadRawTextByBuffer,
encoding options: {
}: ReadRawTextByBuffer): Promise<ReadFileResponse> => { uploadFile?: UploadFileHandler;
} = {}
): Promise<ReadFileResponse> => {
const content = (() => { const content = (() => {
try { try {
if (rawEncodingList.includes(encoding)) { const normalizedEncoding = encoding?.toLowerCase?.() || '';
return buffer.toString(encoding as BufferEncoding);
if (rawEncodingList.includes(normalizedEncoding)) {
// `ascii` 只适用于 0x00~0x7F 字节，若含非 ASCII 字节则优先按 utf-8 解码，避免中文乱码
if (normalizedEncoding === 'ascii' && hasNonAsciiByte(buffer)) {
return buffer.toString('utf-8');
}
return buffer.toString(normalizedEncoding as BufferEncoding);
} }
if (encoding) { if (normalizedEncoding) {
return iconv.decode(buffer, encoding); return iconv.decode(buffer, normalizedEncoding);
} }
return buffer.toString('utf-8'); return buffer.toString('utf-8');
} catch (error) { } catch {
return buffer.toString('utf-8'); return buffer.toString('utf-8');
} }
})(); })();
const { text, imageList } = matchMdImg(content); const rawText = await parseMarkdownBase64Images(content, {
controller: (image) => {
if (image.type !== 'base64') return Promise.resolve({ key: '' });
return uploadBase64Image({
mime: image.mime,
base64: image.base64,
uploadFile: options.uploadFile
});
}
});
return { return {
rawText: text, rawText
imageList
}; };
}; };
fastgpt-merged/packages/service/worker/readFile/extension/utils.ts 0 → 100644
/**
* 清理表格二维数组中的全空行和全空列，保留原始行列顺序。
* 这里只判断单元格文本是否有有效内容，不做类型转换或业务格式化。
*/
export const filterEmptyTableData = (data: unknown[][]) => {
const filteredRows = data.filter((row) => row.some((cell) => String(cell ?? '').trim() !== ''));
const maxColumnLength = Math.max(0, ...filteredRows.map((row) => row.length));
const columnIndexes = Array.from({ length: maxColumnLength }, (_, index) => index).filter(
(index) => filteredRows.some((row) => String(row[index] ?? '').trim() !== '')
);
return filteredRows.map((row) => columnIndexes.map((index) => row[index] ?? ''));
};
/**
* 转义 Markdown table 单元格中的结构字符，避免单元格内容破坏表格列结构。
*/
export const formatMarkdownTableCell = (cell: unknown) => {
return String(cell ?? '')
.replace(/\r\n|\r|\n/g, '\\n')
.replace(/\|/g, '\\|');
};
export const formatMarkdownTableRow = (row: unknown[]) => {
return `| ${row.map(formatMarkdownTableCell).join(' | ')} |`;
};
fastgpt-merged/packages/service/worker/readFile/extension/xlsx.ts
import { CUSTOM_SPLIT_SIGN } from '@fastgpt/global/common/string/textSplitter'; import { CUSTOM_SPLIT_SIGN } from '../../../common/string/textSplitter';
import { type ReadRawTextByBuffer, type ReadFileResponse } from '../type'; import { type ReadRawTextByBuffer, type ReadFileResponse } from '../type';
import xlsx from 'node-xlsx'; import XLSX from 'xlsx';
import Papa from 'papaparse'; import { filterEmptyTableData, formatMarkdownTableRow } from './utils';
export const readXlsxRawText = async ({ export const readXlsxRawText = async ({
buffer buffer
}: ReadRawTextByBuffer): Promise<ReadFileResponse> => { }: ReadRawTextByBuffer): Promise<ReadFileResponse> => {
const result = xlsx.parse(buffer, { const workbook = XLSX.read(buffer, {
skipHidden: false, type: 'buffer',
defval: '' cellDates: true
}); });
const result = workbook.SheetNames.map((name) => {
const worksheet = workbook.Sheets[name];
const data = XLSX.utils.sheet_to_json<unknown[]>(worksheet, {
header: 1,
defval: '',
blankrows: true,
raw: false
});
const merges = worksheet['!merges'] ?? [];
const sheetRange = worksheet['!ref'] ? XLSX.utils.decode_range(worksheet['!ref']) : undefined;
const startRow = sheetRange?.s.r ?? 0;
const startColumn = sheetRange?.s.c ?? 0;
if (merges.length > 0) {
// 合并单元格只有左上角存值；!merges 使用 Excel 绝对坐标，
// 但 sheet_to_json 生成的二维数组从 !ref 起点开始，所以填充前要扣掉起始偏移。
// 必须先补齐合并区域，再做空行空列过滤，否则会丢失用户在 Excel 中表达的结构语义。
for (const merge of merges) {
const startDataRow = merge.s.r - startRow;
const startDataColumn = merge.s.c - startColumn;
const endDataRow = merge.e.r - startRow;
const endDataColumn = merge.e.c - startColumn;
const value = data[startDataRow]?.[startDataColumn] ?? '';
if (String(value).trim() === '') continue;
for (let rowIndex = startDataRow; rowIndex <= endDataRow; rowIndex++) {
if (rowIndex < 0) continue;
data[rowIndex] ??= [];
for (let columnIndex = startDataColumn; columnIndex <= endDataColumn; columnIndex++) {
if (columnIndex < 0) continue;
data[rowIndex][columnIndex] = value;
}
}
}
}
return {
name,
data
};
});
const filteredResult = result.map(({ name, data }) => ({
name,
data: filterEmptyTableData(data)
}));
const format2Csv = result.map(({ name, data }) => { const format2Csv = result.map(({ name, data }) => {
return { return {
title: `#${name}`, title: `#${name}`,
...@@ -20,17 +70,14 @@ export const readXlsxRawText = async ({ ...@@ -20,17 +70,14 @@ export const readXlsxRawText = async ({
const rawText = format2Csv.map((item) => item.csvText).join('\n'); const rawText = format2Csv.map((item) => item.csvText).join('\n');
const formatText = result const formatText = filteredResult
.map(({ data }) => { .map(({ data }) => {
const header = data[0]; const header = data[0];
if (!header) return; if (!header) return;
const formatText = `| ${header.join(' | ')} | const formatText = `${formatMarkdownTableRow(header)}
| ${header.map(() => '---').join(' | ')} | | ${header.map(() => '---').join(' | ')} |
${data ${data.slice(1).map(formatMarkdownTableRow).join('\n')}`;
.slice(1)
.map((row) => `| ${row.map((cell) => String(cell).replace(/\n/g, '\\n')).join(' | ')} |`)
.join('\n')}`;
return formatText; return formatText;
}) })
......
fastgpt-merged/packages/service/worker/readFile/parseOffice.ts
import { getNanoid } from '@fastgpt/global/common/string/tools';
import fs from 'fs';
import decompress from 'decompress';
import { DOMParser } from '@xmldom/xmldom'; import { DOMParser } from '@xmldom/xmldom';
import { clearDirFiles } from '../../common/file/utils'; import { type Entry, fromBuffer, type ZipFile } from 'yauzl';
import { addLog } from '../../common/system/log';
const DEFAULTDECOMPRESSSUBLOCATION = '/tmp'; const powerPointXmlPathRegex = /^ppt\/(?:notesSlides\/notesSlide|slides\/slide)\d+\.xml$/;
const powerPointSlidePathRegex = /^ppt\/slides\/slide\d+\.xml$/;
function getNewFileName(ext: string) { const maxPowerPointEntries = 10000;
return `${DEFAULTDECOMPRESSSUBLOCATION}/${getNanoid()}.${ext}`; const maxPowerPointXmlFileBytes = 10 * 1024 * 1024;
} const maxPowerPointXmlBytes = 100 * 1024 * 1024;
const parseString = (xml: string) => {
let parser = new DOMParser();
return parser.parseFromString(xml, 'text/xml');
};
const parsePowerPoint = async ({ const parsePowerPoint = async ({
filepath, buffer,
decompressPath,
encoding encoding
}: { }: {
filepath: string; buffer: Buffer;
decompressPath: string;
encoding: BufferEncoding; encoding: BufferEncoding;
}) => { }) => {
// Files regex that hold our content of interest const decodeXml = (data: Buffer) => {
const allFilesRegex = /ppt\/(notesSlides|slides)\/(notesSlide|slide)\d+.xml/g; try {
const slidesRegex = /ppt\/slides\/slide\d+.xml/g; return data.toString(encoding);
} catch {
/** The decompress location which contains the filename in it */ // 上游按整个压缩包探测编码时可能得到 Node.js 不支持的编码名。
return data.toString('utf-8');
const files = await decompress(filepath, decompressPath, { }
filter: (x) => !!x.path.match(allFilesRegex) };
const zip = await new Promise<ZipFile>((resolve, reject) => {
fromBuffer(
buffer,
{
lazyEntries: true,
decodeStrings: true,
validateEntrySizes: true,
strictFileNames: true
},
(error, zipFile) => {
if (error) {
reject(error);
return;
}
resolve(zipFile);
}
);
}); });
// Verify if atleast the slides xml files exist in the extracted files list. const files = await new Promise<{ path: string; content: string }[]>((resolve, reject) => {
if ( const result: { path: string; content: string }[] = [];
files.length == 0 || let entriesRead = 0;
!files.map((file) => file.path).some((filename) => filename.match(slidesRegex)) let totalXmlBytes = 0;
) { let settled = false;
return Promise.reject('解析 PPT 失败');
} const fail = (error: unknown) => {
if (settled) return;
settled = true;
zip.close();
reject(error);
};
const readEntryContent = (entry: Entry) => {
zip.openReadStream(entry, (error, stream) => {
if (error) {
fail(error);
return;
}
const chunks: Buffer[] = [];
let entryBytes = 0;
stream.on('data', (chunk: Buffer) => {
entryBytes += chunk.length;
totalXmlBytes += chunk.length;
// 元数据可被伪造，必须按流中真实输出字节中止解压。
if (entryBytes > maxPowerPointXmlFileBytes || totalXmlBytes > maxPowerPointXmlBytes) {
stream.destroy(new Error('解析 PPT 失败'));
return;
}
chunks.push(chunk);
});
stream.once('error', fail);
stream.once('end', () => {
if (settled) return;
result.push({
path: entry.fileName,
content: decodeXml(Buffer.concat(chunks, entryBytes))
});
zip.readEntry();
});
});
};
zip.once('error', fail);
zip.on('entry', (entry: Entry) => {
entriesRead += 1;
if (entriesRead > maxPowerPointEntries) {
fail('解析 PPT 失败');
return;
}
// Returning an array of all the xml contents read using fs.readFileSync const createdOnUnix = entry.versionMadeBy >>> 8 === 3;
const xmlContentArray = await Promise.all( const unixMode = entry.externalFileAttributes >>> 16;
files.map(async (file) => { const unixFileType = unixMode & 0xf000;
try { const isRegularFile =
return await fs.promises.readFile(`${decompressPath}/${file.path}`, encoding); !entry.fileName.endsWith('/') &&
} catch (err) { (!createdOnUnix || unixFileType === 0 || unixFileType === 0x8000);
return await fs.promises.readFile(`${decompressPath}/${file.path}`, 'utf-8');
// yauzl 在 entry 事件前校验原始路径；这里只读取锚定 OOXML 路径下的普通文件。
if (!isRegularFile || !powerPointXmlPathRegex.test(entry.fileName)) {
zip.readEntry();
return;
} }
})
);
let responseArr: string[] = []; if (
entry.uncompressedSize > maxPowerPointXmlFileBytes ||
totalXmlBytes + entry.uncompressedSize > maxPowerPointXmlBytes
) {
fail('解析 PPT 失败');
return;
}
readEntryContent(entry);
});
zip.once('end', () => {
if (settled) return;
settled = true;
if (!result.some((file) => powerPointSlidePathRegex.test(file.path))) {
reject('解析 PPT 失败');
return;
}
resolve(result);
});
xmlContentArray.forEach((xmlContent) => { if (zip.entryCount > maxPowerPointEntries) {
/** Find text nodes with a:p tags */ fail('解析 PPT 失败');
const xmlParagraphNodesList = parseString(xmlContent).getElementsByTagName('a:p'); return;
}
zip.readEntry();
});
const sortedFiles = files.sort((a, b) => {
const getSlideNumber = (path: string) => {
const match = path.match(/\d+/);
return match ? parseInt(match[0]) : 0;
};
return getSlideNumber(a.path) - getSlideNumber(b.path);
});
const parser = new DOMParser();
/** Store all the text content to respond */ return sortedFiles
responseArr.push( .map(({ content }) => {
Array.from(xmlParagraphNodesList) const xmlParagraphNodesList = parser
// Filter paragraph nodes than do not have any text nodes which are identifiable by a:t tag .parseFromString(content, 'text/xml')
.getElementsByTagName('a:p');
return Array.from(xmlParagraphNodesList)
.filter((paragraphNode) => paragraphNode.getElementsByTagName('a:t').length != 0) .filter((paragraphNode) => paragraphNode.getElementsByTagName('a:t').length != 0)
.map((paragraphNode) => { .map((paragraphNode) => {
/** Find text nodes with a:t tags */
const xmlTextNodeList = paragraphNode.getElementsByTagName('a:t'); const xmlTextNodeList = paragraphNode.getElementsByTagName('a:t');
return Array.from(xmlTextNodeList) return Array.from(xmlTextNodeList)
.filter((textNode) => textNode.childNodes[0] && textNode.childNodes[0].nodeValue) .filter((textNode) => textNode.childNodes[0] && textNode.childNodes[0].nodeValue)
.map((textNode) => textNode.childNodes[0].nodeValue) .map((textNode) => textNode.childNodes[0].nodeValue)
.join(''); .join('');
}) })
.join('\n') .join('\n');
); })
}); .join('\n');
return responseArr.join('\n');
}; };
/**
* 解析受支持的 Office 文件文本。
* PPTX 归档按 entry 流式读取，只解压固定 OOXML 路径下且满足大小限制的普通 XML 文件。
*/
export const parseOffice = async ({ export const parseOffice = async ({
buffer, buffer,
encoding, encoding,
...@@ -89,43 +176,10 @@ export const parseOffice = async ({ ...@@ -89,43 +176,10 @@ export const parseOffice = async ({
encoding: BufferEncoding; encoding: BufferEncoding;
extension: string; extension: string;
}) => { }) => {
// Prepare file for processing switch (extension) {
// create temp file subdirectory if it does not exist case 'pptx':
if (!fs.existsSync(DEFAULTDECOMPRESSSUBLOCATION)) { return parsePowerPoint({ buffer, encoding });
fs.mkdirSync(DEFAULTDECOMPRESSSUBLOCATION, { recursive: true }); default:
} return Promise.reject('只能读取 .pptx 文件');
// temp file name
const filepath = getNewFileName(extension);
const decompressPath = `${DEFAULTDECOMPRESSSUBLOCATION}/${getNanoid()}`;
// const decompressPath = `${DEFAULTDECOMPRESSSUBLOCATION}/test`;
// write new file
try {
fs.writeFileSync(filepath, buffer, {
encoding
});
} catch (err) {
fs.writeFileSync(filepath, buffer, {
encoding: 'utf-8'
});
} }
const text = await (async () => {
try {
switch (extension) {
case 'pptx':
return parsePowerPoint({ filepath, decompressPath, encoding });
default:
return Promise.reject('只能读取 .pptx 文件');
}
} catch (error) {
addLog.error(`Load ppt error`, { error });
}
return '';
})();
fs.unlinkSync(filepath);
clearDirFiles(decompressPath);
return text;
}; };
fastgpt-merged/packages/service/worker/readFile/type.d.ts
...@@ -12,6 +12,17 @@ export type ImageType = { ...@@ -12,6 +12,17 @@ export type ImageType = {
mime: string; mime: string;
}; };
export type UploadedFileResult = {
key: string;
previewUrl?: string;
};
export type UploadFileHandler = (data: {
name: string;
mime: string;
buffer: ArrayBuffer;
}) => Promise<UploadedFileResult>;
export type ReadFileResponse = { export type ReadFileResponse = {
rawText: string; rawText: string;
formatText?: string; formatText?: string;
......
fastgpt-merged/packages/service/worker/tokenWorkerConfig.ts 0 → 100644
import { availableParallelism, cpus } from 'os';
/**
* Token 计算 worker 数量跟随当前运行环境可用 CPU 数，最多保留 4 个。
*
* tokenizer 会在每个 worker 内各自加载一份编码表，worker 过多会放大常驻内存；
* 这里固定为 min(cpu, 4)，不再暴露配置项，避免部署环境误配过多 worker。
* availableParallelism 会优先考虑容器 CPU 配额，拿不到时再回退到物理 CPU 数。
*/
export const getTokenWorkerCount = () => {
const availableCpu = availableParallelism?.() || cpus().length || 1;
return Math.max(1, Math.min(availableCpu, 4));
};
fastgpt-merged/packages/service/worker/utils/base64ImageUpload.ts 0 → 100644
import { type UploadFileHandler } from '../readFile/type';
import { resolveMimeExtension } from '../../common/s3/utils/mime';
import { getLogger, LogCategories } from '../../common/logger';
const logger = getLogger(LogCategories.INFRA.WORKER);
const MAX_PARSED_IMAGE_BUFFER_SIZE = 40 * 1024 * 1024;
export class ParsedImageTooLargeError extends Error {
constructor(size: number, maxSize: number) {
super(`Parsed image too large. Size: ${size} bytes, maximum allowed: ${maxSize} bytes`);
this.name = 'ParsedImageTooLargeError';
}
}
const getBase64DecodedSize = (base64: string) => {
const normalizedBase64 = base64.replace(/\s/g, '');
const padding = normalizedBase64.endsWith('==') ? 2 : normalizedBase64.endsWith('=') ? 1 : 0;
return Math.max(0, Math.floor((normalizedBase64.length * 3) / 4) - padding);
};
const toTransferableArrayBuffer = (buffer: Buffer): ArrayBuffer => {
if (
buffer.buffer instanceof ArrayBuffer &&
buffer.byteOffset === 0 &&
buffer.byteLength === buffer.buffer.byteLength
) {
return buffer.buffer;
}
const imageArrayBuffer = new Uint8Array(buffer.byteLength);
imageArrayBuffer.set(buffer);
return imageArrayBuffer.buffer;
};
export const uploadBase64Image = async ({
mime,
base64,
uploadFile
}: {
mime: string;
base64: string;
uploadFile?: UploadFileHandler;
}) => {
if (!uploadFile) {
logger.warn('Missing image upload handler when parsing document image', { mime });
throw new Error('Missing imageKeyOptions.prefix for parsed document image upload');
}
const decodedSize = getBase64DecodedSize(base64);
if (decodedSize > MAX_PARSED_IMAGE_BUFFER_SIZE) {
throw new ParsedImageTooLargeError(decodedSize, MAX_PARSED_IMAGE_BUFFER_SIZE);
}
const imageBuffer = Buffer.from(base64, 'base64');
const filename = `${crypto.randomUUID()}${resolveMimeExtension(mime)}`;
return uploadFile({
name: filename,
mime,
buffer: toTransferableArrayBuffer(imageBuffer)
}).catch((error) => {
logger.warn('Failed to upload parsed document image from worker', {
filename,
mime,
error
});
throw error;
});
};
fastgpt-merged/packages/service/worker/utils/uploadFile.ts 0 → 100644
import type { MessagePort } from 'worker_threads';
import { type UploadFileHandler, type UploadedFileResult } from '../readFile/type';
type WorkerUploadFileResponse = {
id: string;
type?: string;
requestId?: string;
data?: UploadedFileResult | any;
};
type PendingUploadFileRequest = {
taskId: string;
resolve: (value: UploadedFileResult) => void;
reject: (error: any) => void;
};
const pendingUploadFileRequests = new Map<string, PendingUploadFileRequest>();
export const isWorkerUploadFileResponse = (type?: string) =>
type === 'uploadFileResult' || type === 'uploadFileError';
/**
* 处理 worker uploadFile 的主线程回包。
*
* 多个 worker 入口共用同一套 requestId -> Promise 映射，按 taskId 防串扰。
*/
export const handleWorkerUploadFileResponse = ({
taskId,
type,
requestId,
data
}: {
taskId: string;
type?: string;
requestId?: string;
data?: any;
}) => {
if (!isWorkerUploadFileResponse(type) || !requestId) return false;
const pending = pendingUploadFileRequests.get(requestId);
if (!pending || pending.taskId !== taskId) return true;
pendingUploadFileRequests.delete(requestId);
if (type === 'uploadFileError') {
pending.reject(data);
} else {
pending.resolve(data as UploadedFileResult);
}
return true;
};
export const cleanupWorkerUploadFileRequests = (taskId: string, reason: Error) => {
for (const [requestId, pending] of pendingUploadFileRequests.entries()) {
if (pending.taskId !== taskId) continue;
pending.reject(reason);
pendingUploadFileRequests.delete(requestId);
}
};
/**
* 为 worker 内单个任务创建 uploadFile handler。
*
* 返回的 handler 会向主线程发送 `uploadFile` 中间事件，并等待同 requestId 的结果回包；
* cleanup 必须在任务结束时调用，避免 dangling promise。
*/
export const createWorkerUploadFileHandler = ({
taskId,
parentPort
}: {
taskId: string;
parentPort?: MessagePort | null;
}): {
uploadFile: UploadFileHandler;
cleanup: () => void;
} => {
const uploadFile: UploadFileHandler = (data) =>
new Promise((resolve, reject) => {
const requestId = crypto.randomUUID();
pendingUploadFileRequests.set(requestId, { taskId, resolve, reject });
try {
parentPort?.postMessage(
{
id: taskId,
type: 'uploadFile',
requestId,
data
},
[data.buffer]
);
} catch (error) {
pendingUploadFileRequests.delete(requestId);
reject(error);
}
});
return {
uploadFile,
cleanup: () =>
cleanupWorkerUploadFileRequests(
taskId,
new Error('Worker upload request cancelled before completion')
)
};
};
/**
* 兼容需要在 worker 入口内部监听回包的任务，例如 readFile 会忽略主 message handler 中的回包。
*/
export const createWorkerUploadFileHandlerWithListener = ({
taskId,
parentPort,
enabled
}: {
taskId: string;
parentPort?: MessagePort | null;
enabled: boolean;
}): {
uploadFile?: UploadFileHandler;
cleanup: () => void;
} => {
if (!enabled) return { cleanup: () => {} };
const onMessage = ({ id, type, requestId, data }: WorkerUploadFileResponse) => {
if (id !== taskId) return;
handleWorkerUploadFileResponse({
taskId,
type,
requestId,
data
});
};
parentPort?.on('message', onMessage);
const bridge = createWorkerUploadFileHandler({ taskId, parentPort });
return {
uploadFile: bridge.uploadFile,
cleanup: () => {
parentPort?.off('message', onMessage);
bridge.cleanup();
}
};
};
fastgpt-merged/pnpm-workspace.yaml
...@@ -6,27 +6,11 @@ packages: ...@@ -6,27 +6,11 @@ packages:
- projects/marketplace - projects/marketplace
- projects/mcp_server - projects/mcp_server
- projects/volume-manager - projects/volume-manager
- pro/llm_benchmark/content_benchmark
- pro/admin
- pro/sso
- pro/browser-sandbox
- document/ - document/
- scripts/icon - scripts/icon
- sdk/* - sdk/*
allowBuilds:
'@google/genai': set this to true or false
'@parcel/watcher': set this to true or false
core-js: set this to true or false
esbuild: set this to true or false
micromark: set this to true or false
mongodb-memory-server: set this to true or false
msgpackr-extract: set this to true or false
protobufjs: set this to true or false
sharp: set this to true or false
vue-demi: set this to true or false
catalog: catalog:
'@chakra-ui/anatomy': ^2 '@chakra-ui/anatomy': ^2
'@chakra-ui/icons': ^2 '@chakra-ui/icons': ^2
......
fastgpt-merged/projects/app/.env.local
...@@ -10,6 +10,10 @@ DB_MAX_LINK=5 ...@@ -10,6 +10,10 @@ DB_MAX_LINK=5
TOKEN_KEY=dfdasfdas TOKEN_KEY=dfdasfdas
# 文件阅读时的秘钥 # 文件阅读时的秘钥
FILE_TOKEN_KEY=filetokenkey FILE_TOKEN_KEY=filetokenkey
# 密钥加密 key
AES256_SECRET_KEY=fastgptsecret
# Invoke 反向调用 JWT 密钥，至少 32 位
INVOKE_TOKEN_SECRET=fastgpt_invoke_token_secret_32_chars_min
# root key, 最高权限 # root key, 最高权限
ROOT_KEY=fdafasd ROOT_KEY=fdafasd
# openai 基本地址，可用作中转。 # openai 基本地址，可用作中转。
......
fastgpt-merged/projects/app/next.config.ts
...@@ -77,7 +77,9 @@ const nextConfig: NextConfig = { ...@@ -77,7 +77,9 @@ const nextConfig: NextConfig = {
'@zilliz/milvus2-sdk-node', '@zilliz/milvus2-sdk-node',
'@opentelemetry/api-logs', '@opentelemetry/api-logs',
'@mariozechner/pi-agent-core', '@mariozechner/pi-agent-core',
'@mariozechner/pi-ai' '@mariozechner/pi-ai',
'proxy-agent',
'vm2'
], ],
// 优化大库的 barrel exports tree-shaking // 优化大库的 barrel exports tree-shaking
experimental: { experimental: {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment