Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 89b80f75 by Archer Committed by GitHub
Options

Perf worker and env load (#6861) 

* perf: read file worker

* perf: worker pool

* fix: test

* fix: review

* fix: test

* fix: add helm aes secret env

* fix: align required env configuration

* docs: align env default values

* test: make AES tamper case deterministic

* sandbox default env

* chore: centralize environment configuration

* fix: tighten env ownership and validation

* fix: harden env compatibility

* perf: env

* fix: resolve env ci failures

* perf: env

* perf: env

* remove invalid code

* doc

* doc

* doc

* doc

* perf: axios header get

* fix: test

* fix: i18n
parent 4a3401a4
Showing with 1352 additions and 651 deletions
.vscode/settings.json
......@@ -34,5 +34,6 @@
},
"files.associations": {
"*.mdx": "markdown"
}
},
"typescript.tsdk": "node_modules/typescript/lib"
}
deploy/helm/fastgpt/templates/secret-env.yaml
......@@ -7,6 +7,7 @@ stringData:
TOKEN_KEY: "any"
ROOT_KEY: "root_key"
FILE_TOKEN_KEY: "filetoken"
AES256_SECRET_KEY: "fastgptsecret"
MONGODB_URI: "mongodb://{{ .Values.mongodb.auth.rootUser }}:{{ .Values.mongodb.auth.rootPassword }}@{{ include "fastgpt.fullname" . }}-mongodb-headless:27017/fastgpt?authSource=admin"
PG_URL: "postgresql://postgres:{{ .Values.postgresql.auth.rootPassword }}@{{ include "fastgpt.fullname" . }}-postgresql:5432/{{ .Values.postgresql.global.postgresql.auth.database }}"
kind: Secret
......
deploy/templates/docker-compose.prod.yml
......@@ -157,6 +157,7 @@ ${{vec.db}}
condition: service_healthy
restart: always
environment:
# 完整变量请参考: https://github.com/labring/FastGPT/blob/main/projects/app/.env.template
<<: [*x-share-db-config, *x-vec-config, *x-log-config]
HOSTNAME: 0.0.0.0
# ==================== 基础配置 ====================
......@@ -168,8 +169,8 @@ ${{vec.db}}
DEFAULT_ROOT_PSW: *x-default-root-psw
# 数据库最大连接数
DB_MAX_LINK: 5
# 自动同步索引（0 表示不同步）
SYNC_INDEX: 1
# 自动同步索引
SYNC_INDEX: true
TOKEN_KEY: fastgpt
# 文件阅读时的密钥
FILE_TOKEN_KEY: filetokenkey
......@@ -207,30 +208,6 @@ ${{vec.db}}
# ==================== 日志与监控 ====================
# 传递给 OTLP 收集器的服务名称
LOG_OTEL_SERVICE_NAME: fastgpt-client
# ==================== 安全与运行限制 ====================
# 启动 IP 限流（true）；部分接口启用 IP 限流策略以防止异常请求
USE_IP_LIMIT: false
# 工作流最大运行次数，避免极端死循环
WORKFLOW_MAX_RUN_TIMES: 1000
# 循环最大运行次数，避免极端死循环
WORKFLOW_MAX_LOOP_TIMES: 100
# 服务器接收请求的最大大小（MB）
SERVICE_REQUEST_MAX_CONTENT_LENGTH: 10
# 启用内网 IP 检查
CHECK_INTERNAL_IP: false
# ==================== 上传与账号策略 ====================
# 最大上传文件大小（MB）
UPLOAD_FILE_MAX_SIZE: 1000
# 最大上传文件数量
UPLOAD_FILE_MAX_AMOUNT: 1000
# LLM 请求追踪保留时长（小时）
LLM_REQUEST_TRACKING_RETENTION_HOURS: 6
# ==================== 功能开关与特殊配置 ====================
# 自定义跨域；不配置时默认允许所有跨域（逗号分割）
ALLOWED_ORIGINS:
# HTML 转 Markdown 最大字符数（超过后不执行转换）
MAX_HTML_TRANSFORM_CHARS: 1000000
volumes:
- ./config.json:/app/data/config.json
fastgpt-code-sandbox:
......
document/content/self-host/config/meta.en.json
{
"title": "Configuration",
"description": "FastGPT self-hosting configuration",
"pages": ["model", "object-storage", "json", "signoz"]
"pages": ["model", "object-storage", "json", "env", "signoz"]
}
document/content/self-host/config/meta.json
{
"title": "配置说明",
"description": "FastGPT 自部署配置",
"pages": ["model", "object-storage", "json", "signoz"]
"pages": ["model", "object-storage", "json", "env", "signoz"]
}
document/content/self-host/upgrading/4-15/4150.mdx
......@@ -3,12 +3,38 @@ title: 'V4.15.0(进行中)'
description: 'FastGPT V4.15.0 更新说明'
---
## 升级指南
### 镜像变更
- 更新 fastgpt-plugin 镜像
- 更新 AIProxy 镜像 tag: v0.5.6
### 环境变量变更
`fastgpt-app`, `fastgpt-pro` 可增加文件解析并发线程数
```bash
# 文件解析 worker 并发数
PARSE_FILE_WORKERS=10
# 文件解析超时时间（秒）
PARSE_FILE_TIMEOUT_SECONDS=600
# HTML 转 Markdown worker 并发数
HTML_TO_MARKDOWN_WORKERS=10
# 文本切块 worker 并发数
TEXT_TO_CHUNKS_WORKERS=10
# 自动同步 mongo 数据库索引, 改成 boolean 字符串值，而不是 0 和 1
SYNC_INDEX=true
```
## 🚀 新增内容
1. 新增循环节点，弃用旧的批量执行。
2. 全局变量输入框支持输入 object 类型数据。
3. 工具调用模式下，如果开启了虚拟机功能，用户对话框上传的文件会直接注入到虚拟机中。
4. 第三方知识库接入钉钉知识库。
5. 增加文件解析/HTML转Markdown/文本切块 worker pool，避免并发太高导致资源耗尽，可通过环境变量调整其 pool 数量。
6. 模型思考配置。
## ⚙️ 优化
......@@ -18,6 +44,7 @@ description: 'FastGPT V4.15.0 更新说明'
4. 无创建权限时，隐藏模板功能。
5. 加强第三方知识库请求的 SSRF 防护。
6. codex-sandbox 加强 AST 检查，防止绕过安全检查。
7. 站点同步限流错误提示，重复提示。
## 🐛 修复
......@@ -27,4 +54,5 @@ description: 'FastGPT V4.15.0 更新说明'
1. 重新调整代码结构，升级 nextjs 最新版，切换至 turbopack 构建，提高构建速度；升级容器默认 node 至 24。
2. 优化 Agent tool 声明和运行，统一所有 tool 的声明和运行方式。
3. 文件上传内容从 system prompt 中放到 user message 中，提高 cache 命中率
3. 文件上传内容从 system prompt 中放到 user message 中，提高 cache 命中率。
4. 服务端 env 加载全部使用`@t3-oss/env-core`，增加更多类型检查。其余服务，也采用集中导出 env 的方式进行环境变量使用。
document/content/toc.en.mdx
......@@ -75,6 +75,7 @@ description: FastGPT Toc
- [/en/openapi/index](/en/openapi/index)
- [/en/openapi/intro](/en/openapi/intro)
- [/en/openapi/share](/en/openapi/share)
- [/en/self-host/config/env](/en/self-host/config/env)
- [/en/self-host/config/json](/en/self-host/config/json)
- [/en/self-host/config/model/intro](/en/self-host/config/model/intro)
- [/en/self-host/config/model/minimax](/en/self-host/config/model/minimax)
......
document/content/toc.mdx
......@@ -75,6 +75,7 @@ description: FastGPT 文档目录
- [/openapi/index](/openapi/index)
- [/openapi/intro](/openapi/intro)
- [/openapi/share](/openapi/share)
- [/self-host/config/env](/self-host/config/env)
- [/self-host/config/json](/self-host/config/json)
- [/self-host/config/model/intro](/self-host/config/model/intro)
- [/self-host/config/model/minimax](/self-host/config/model/minimax)
......
document/data/doc-last-modified.json
......@@ -145,6 +145,8 @@
"content/openapi/intro.mdx": "2026-04-26T21:08:47+08:00",
"content/openapi/share.en.mdx": "2026-04-26T21:08:47+08:00",
"content/openapi/share.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/config/env.en.mdx": "2026-04-30T00:20:04+08:00",
"content/self-host/config/env.mdx": "2026-04-30T00:20:04+08:00",
"content/self-host/config/json.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/config/json.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/config/model/intro.en.mdx": "2026-04-26T21:08:47+08:00",
......@@ -233,8 +235,8 @@
"content/self-host/upgrading/4-14/41415.mdx": "2026-04-26T21:28:27+08:00",
"content/self-host/upgrading/4-14/41416.en.mdx": "2026-04-26T21:28:27+08:00",
"content/self-host/upgrading/4-14/41416.mdx": "2026-04-26T22:41:57+08:00",
"content/self-host/upgrading/4-14/41417.mdx": "2026-04-28T18:03:38+08:00",
"content/self-host/upgrading/4-14/41418.mdx": "2026-05-06T10:37:28+08:00",
"content/self-host/upgrading/4-14/41417.mdx": "2026-05-06T13:57:16+08:00",
"content/self-host/upgrading/4-14/41418.mdx": "2026-05-06T13:57:16+08:00",
"content/self-host/upgrading/4-14/4142.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/4142.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/4143.en.mdx": "2026-04-26T21:08:47+08:00",
......@@ -255,7 +257,7 @@
"content/self-host/upgrading/4-14/41481.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/4149.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/4149.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-15/4150.mdx": "2026-04-29T20:39:24+08:00",
"content/self-host/upgrading/4-15/4150.mdx": "2026-05-06T14:03:59+08:00",
"content/self-host/upgrading/outdated/40.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/40.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/41.en.mdx": "2026-04-26T21:08:47+08:00",
......@@ -396,8 +398,8 @@
"content/self-host/upgrading/outdated/499.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/upgrade-intruction.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/upgrade-intruction.mdx": "2026-04-26T21:08:47+08:00",
"content/toc.en.mdx": "2026-04-29T20:39:24+08:00",
"content/toc.mdx": "2026-05-06T10:37:28+08:00",
"content/toc.en.mdx": "2026-04-29T23:22:43+08:00",
"content/toc.mdx": "2026-04-29T23:22:43+08:00",
"content/use-cases/app-cases/dalle3.en.mdx": "2026-04-26T21:08:47+08:00",
"content/use-cases/app-cases/dalle3.mdx": "2026-04-26T21:08:47+08:00",
"content/use-cases/app-cases/english_essay_correction_bot.en.mdx": "2026-04-26T21:08:47+08:00",
......
packages/global/common/axios/utils.ts 0 → 100644
import type { AxiosHeaderValue } from 'axios';
export const getAxiosHeaderValue = (header?: AxiosHeaderValue): string | undefined => {
if (header === null || header === undefined || typeof header === 'boolean') return;
if (Array.isArray(header)) {
return header[0];
}
if (typeof header === 'number') {
return String(header);
}
if (typeof header === 'string') {
return header;
}
return;
};
export const getAxiosContentType = (header?: AxiosHeaderValue): string | undefined => {
return getAxiosHeaderValue(header)?.toLowerCase()?.split(';')?.[0]?.trim();
};
packages/global/common/file/constants.ts
......@@ -4,9 +4,6 @@ export enum BucketNameEnum {
chat = 'chat'
}
export const EndpointUrl = `${process.env.FILE_DOMAIN || process.env.FE_DOMAIN || ''}${process.env.NEXT_PUBLIC_BASE_URL || ''}`;
export const ReadFileBaseUrl = `${EndpointUrl}/api/common/file/read`;
export const documentFileType = '.txt, .docx, .csv, .xlsx, .pdf, .md, .html, .pptx';
/** 图片数据集创建/追加图片（multer 直传）与 ImageDataset、InsertImageModal 的 fileType 一致 */
......
packages/global/common/string/url.ts 0 → 100644
export const stripUrlTrailingSlash = (value?: string) => value?.replace(/\/+$/, '') || '';
packages/global/common/system/constants.ts
......@@ -9,3 +9,4 @@ export const DEFAULT_USER_AVATAR = '/imgs/avatar/BlueAvatar.svg';
export const isDevEnv = process.env.NODE_ENV === 'development';
export const isProduction = process.env.NODE_ENV === 'production';
export const isTestEnv = process.env.NODE_ENV === 'test';
export const isPhaseProductionBuild = process.env.NEXT_PHASE === 'phase-production-build';
packages/global/common/system/network.ts 0 → 100644
import dns from 'dns/promises';
import ipaddr from 'ipaddr.js';
import { isIPv6 } from 'net';
export const PRIVATE_URL_TEXT = 'Request to private network not allowed';
export type InternalAddressCheckerOptions = {
checkInternalIp: () => boolean;
nodeEnv?: string;
hostname?: string;
port?: string | number;
};
// 云厂商元数据服务 IP（除 169.254.0.0/16 段外的特殊地址）
// 预先归一化为 ipaddr.js 的 normalizedString 形式以便比对
const METADATA_IPS = new Set<string>(
[
'100.100.100.200', // 阿里云
'fd00:ec2::254' // AWS IPv6
].map((ip) => ipaddr.parse(ip).toNormalizedString().toLowerCase())
);
// 云厂商元数据服务主机名（归一化：小写、去尾部点）
const METADATA_HOSTNAMES = new Set<string>([
'metadata.google.internal',
'metadata',
'metadata.tencentyun.com',
'kubernetes.default.svc',
'kubernetes.default',
'kubernetes'
]);
const LOCALHOST_HOSTNAMES = new Set<string>(['localhost']);
/**
* 把 URL hostname 尝试解析成 ipaddr.js 的地址对象
* - 处理 IPv6 方括号
* - 处理 IPv4-mapped IPv6 (::ffff:a.b.c.d / ::ffff:xxxx:xxxx) -> 解包为 IPv4
* - 处理十进制/十六进制/八进制/短点分 IPv4 字面量
* 非 IP 字面量返回 null
*/
const parseHostAsIP = (rawHostname: string): ipaddr.IPv4 | ipaddr.IPv6 | null => {
const host = rawHostname.replace(/^\[|\]$/g, '').replace(/\.$/, '');
if (!host) return null;
// ipaddr.process 会自动把 IPv4-mapped IPv6 解包为 IPv4，处理常规字面量
if (ipaddr.isValid(host)) {
try {
return ipaddr.process(host);
} catch {
return null;
}
}
// ipaddr.js 不支持十进制/十六进制/八进制 IPv4 短写，手动兜底
const numeric = parseNumericIPv4(host);
if (numeric) return ipaddr.parse(numeric) as ipaddr.IPv4;
return null;
};
/**
* 解析 inet_aton 兼容的 IPv4 字面量：十进制 2852039166、十六进制 0xa9fea9fe、
* 八进制、1-4 段形式（含 dec/hex/oct 混合）。返回标准点分十进制或 null
*/
const parseNumericIPv4 = (host: string): string | null => {
const parts = host.split('.');
if (parts.length === 0 || parts.length > 4) return null;
const nums: number[] = [];
for (const part of parts) {
if (!part) return null;
let n: number;
if (/^0x[0-9a-f]+$/i.test(part)) n = parseInt(part, 16);
else if (/^0[0-7]+$/.test(part)) n = parseInt(part, 8);
else if (/^\d+$/.test(part)) n = parseInt(part, 10);
else return null;
if (!Number.isFinite(n) || n < 0) return null;
nums.push(n);
}
const maxLast = [0xffffffff, 0xffffff, 0xffff, 0xff][parts.length - 1];
if (nums[nums.length - 1] > maxLast) return null;
for (let i = 0; i < nums.length - 1; i++) if (nums[i] > 0xff) return null;
let ipInt = 0;
for (let i = 0; i < nums.length - 1; i++) ipInt = (ipInt + nums[i]) * 256;
ipInt += nums[nums.length - 1];
if (ipInt > 0xffffffff) return null;
return [(ipInt >>> 24) & 0xff, (ipInt >>> 16) & 0xff, (ipInt >>> 8) & 0xff, ipInt & 0xff].join(
'.'
);
};
const normalizeDomain = (rawHostname: string): string =>
rawHostname
.replace(/^\[|\]$/g, '')
.replace(/\.$/, '')
.toLowerCase();
/**
* ipaddr.js range() 返回的所有非 'unicast' 分类都视为内部地址。
* 主要范围：private / loopback / linkLocal / uniqueLocal / reserved /
* multicast / broadcast / unspecified / carrierGradeNat 等
*/
const isInternalIPAddress = (addr: ipaddr.IPv4 | ipaddr.IPv6): boolean => {
return addr.range() !== 'unicast';
};
/**
* 元数据端点：
* - 169.254.0.0/16 link-local 段全部视为元数据
* - 显式列表里的 IP（阿里云 100.100.100.200、AWS IPv6 fd00:ec2::254）
*/
const isMetadataIPAddress = (addr: ipaddr.IPv4 | ipaddr.IPv6): boolean => {
if (addr.kind() === 'ipv4' && addr.range() === 'linkLocal') return true;
return METADATA_IPS.has(addr.toNormalizedString().toLowerCase());
};
export const createInternalAddressChecker = (options: InternalAddressCheckerOptions) => {
const isDevEnv = (options.nodeEnv ?? process.env.NODE_ENV) === 'development';
const serviceLocalPort = `${options.port ?? process.env.PORT ?? 3000}`;
const hostname = options.hostname ?? process.env.HOSTNAME;
const serviceLocalHost =
hostname && isIPv6(hostname)
? `[${hostname}]:${serviceLocalPort}`
: `${hostname || 'localhost'}:${serviceLocalPort}`;
/**
* 对已解析出的 IP 复检（防 DNS rebinding TOCTOU）。
* 调用方先用 isInternalAddress(url) 通过预检，再用 dns.lookup 拿到将要连接的 IP，
* 在真正建连前用此函数二次校验，确保两次解析的 IP 都在策略允许范围内。
*/
const isInternalResolvedIP = (rawIP: string): boolean => {
if (isDevEnv) return false;
if (!ipaddr.isValid(rawIP)) return false;
const addr = ipaddr.process(rawIP);
if (isMetadataIPAddress(addr)) return true;
const range = addr.range();
if (range === 'loopback' || range === 'unspecified') return true;
if (options.checkInternalIp() && isInternalIPAddress(addr)) return true;
return false;
};
const isInternalAddress = async (url: string): Promise<boolean> => {
if (isDevEnv) return false;
let parsedUrl: URL;
try {
parsedUrl = new URL(url);
} catch {
return false;
}
const hostDomain = normalizeDomain(parsedUrl.hostname);
const localHost = serviceLocalHost.split(':')[0].toLowerCase();
// 1. localhost / 本机
if (LOCALHOST_HOSTNAMES.has(hostDomain) || hostDomain === localHost) {
return true;
}
// 2. 云元数据主机名
if (METADATA_HOSTNAMES.has(hostDomain)) {
return true;
}
// 3. IP 字面量（含各种编码变体）
const ip = parseHostAsIP(parsedUrl.hostname);
if (ip) {
if (isMetadataIPAddress(ip)) return true;
// loopback/unspecified 等始终阻止（这些是显而易见的错误配置或攻击）
const range = ip.range();
if (range === 'loopback' || range === 'unspecified') return true;
if (options.checkInternalIp()) return isInternalIPAddress(ip);
return false;
}
// 4. 域名：解析 DNS；元数据命中始终阻止，私有段受 CHECK_INTERNAL_IP 控制
try {
const [v4Res, v6Res] = await Promise.allSettled([
dns.resolve4(hostDomain),
dns.resolve6(hostDomain)
]);
const resolvedIPs = [
...(v4Res.status === 'fulfilled' ? v4Res.value : []),
...(v6Res.status === 'fulfilled' ? v6Res.value : [])
];
for (const raw of resolvedIPs) {
if (!ipaddr.isValid(raw)) continue;
const addr = ipaddr.process(raw);
if (isMetadataIPAddress(addr)) return true;
const r = addr.range();
if (r === 'loopback' || r === 'unspecified') return true;
if (options.checkInternalIp() && isInternalIPAddress(addr)) return true;
}
return false;
} catch {
return false;
}
};
return { isInternalAddress, isInternalResolvedIP };
};
packages/global/common/system/types/index.ts
import type { SubPlanType } from '../../../support/wallet/sub/type';
import { StandSubPlanLevelMapType } from '../../../support/wallet/sub/type';
import type {
LLMModelItemType,
EmbeddingModelItemType,
......@@ -7,7 +6,6 @@ import type {
STTModelType,
RerankModelItemType
} from '../../../core/ai/model.schema';
import { SubTypeEnum } from '../../../support/wallet/sub/constants';
export type NavbarItemType = {
id: string;
......@@ -153,7 +151,7 @@ export type FastGPTFeConfigsType = {
export type SystemEnvType = {
openapiPrefix?: string;
tokenWorkers: number; // token count max worker
tokenWorkers: number; // token count max worker (min 10, max 1000)
datasetParseMaxProcess: number;
vectorMaxProcess: number;
......
packages/global/common/zod/index.ts 0 → 100644
import z from 'zod';
import { stripUrlTrailingSlash } from '../string/url';
const truthyBoolStrs = ['true', '1', 'yes', 'y'];
export const BoolSchema = z
.string()
.transform((val) => truthyBoolStrs.includes(val.toLowerCase()))
.pipe(z.boolean());
export const NumSchema = z.coerce.number<number>();
export const IntSchema = NumSchema.int().nonnegative();
export const UrlSchema = z.string().url().transform(stripUrlTrailingSlash);
packages/global/package.json
......@@ -24,6 +24,7 @@
"next": "catalog:",
"openai": "6.34.0",
"openapi-types": "^12.1.3",
"ipaddr.js": "^2.3.0",
"timezones-list": "^3.0.2",
"lodash": "catalog:",
"zod": "catalog:",
......
packages/global/test/common/axios/utils.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import { getAxiosContentType, getAxiosHeaderValue } from '@fastgpt/global/common/axios/utils';
describe('getAxiosHeaderValue', () => {
it('should normalize axios header values to strings', () => {
expect(getAxiosHeaderValue('text/plain')).toBe('text/plain');
expect(getAxiosHeaderValue(['text/plain', 'text/html'])).toBe('text/plain');
expect(getAxiosHeaderValue(123)).toBe('123');
});
it('should ignore unset and boolean header values', () => {
expect(getAxiosHeaderValue(undefined)).toBe(undefined);
expect(getAxiosHeaderValue(null)).toBe(undefined);
expect(getAxiosHeaderValue(true)).toBe(undefined);
expect(getAxiosHeaderValue(false)).toBe(undefined);
});
});
describe('getAxiosContentType', () => {
it('should extract and normalize the content type', () => {
expect(getAxiosContentType('Image/PNG; charset=utf-8')).toBe('image/png');
expect(getAxiosContentType(['Text/HTML; charset=UTF-8'])).toBe('text/html');
});
});
packages/service/common/api/plusRequest.ts
......@@ -9,6 +9,7 @@ import { UserError } from '@fastgpt/global/common/error/utils';
import { createProxyAxios } from './axios';
import { getLogger, LogCategories } from '../logger';
import { assertRelativePath } from '../security/network';
import { serviceEnv } from '../../env';
const logger = getLogger(LogCategories.HTTP.ERROR);
......@@ -76,7 +77,7 @@ const instance = createProxyAxios(
headers: {
'content-type': 'application/json',
'Cache-Control': 'no-cache',
rootkey: process.env.ROOT_KEY
rootkey: serviceEnv.ROOT_KEY
}
},
false
......
packages/service/common/cache/index.ts
......@@ -4,6 +4,7 @@ import type { SystemCacheKeyEnum } from './type';
import { randomUUID } from 'node:crypto';
import { initCache } from './init';
import { isProduction } from '@fastgpt/global/common/system/constants';
import { serviceEnv } from '../../env';
const cachePrefix = `VERSION_KEY:`;
......@@ -50,7 +51,7 @@ export const getCachedData = async <T extends SystemCacheKeyEnum>(key: T, id?: s
if (!global.systemCache) initCache();
const versionKey = await getVersionKey(key, id);
const isDisableCache = process.env.DISABLE_CACHE === 'true';
const isDisableCache = serviceEnv.DISABLE_CACHE;
const item = global.systemCache[key];
......
packages/service/common/file/image/controller.ts
......@@ -13,9 +13,11 @@ import { getS3AvatarSource } from '../../s3/sources/avatar';
import { isS3ObjectKey } from '../../s3/utils';
import path from 'path';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { serviceEnv } from '../../../env';
export const maxImgSize = 1024 * 1024 * 12;
const base64MimeRegex = /data:image\/([^\)]+);base64/;
const imageRouteBase = serviceEnv.NEXT_PUBLIC_BASE_URL;
export async function uploadMongoImg({
base64Img,
......@@ -59,7 +61,7 @@ export async function uploadMongoImg({
})
);
return `${process.env.NEXT_PUBLIC_BASE_URL || ''}${imageBaseUrl}${String(_id)}.${extension}`;
return `${imageRouteBase}${imageBaseUrl}${String(_id)}.${extension}`;
}
export const copyAvatarImage = async ({
......@@ -121,7 +123,7 @@ export const copyAvatarImage = async ({
ordered: true
}
);
return `${process.env.NEXT_PUBLIC_BASE_URL || ''}${imageBaseUrl}${String(newImage._id)}.${image.metadata?.mime?.split('/')[1]}`;
return `${imageRouteBase}${imageBaseUrl}${String(newImage._id)}.${image.metadata?.mime?.split('/')[1]}`;
}
return imageUrl;
......
packages/service/common/file/image/utils.ts
import { axios } from '../../api/axios';
import { serverRequestBaseUrl } from '../../api/serverRequest';
import { retryFn } from '@fastgpt/global/common/system/utils';
import { getContentTypeFromHeader } from '../utils';
import { getAxiosContentType } from '@fastgpt/global/common/axios/utils';
import { getLogger, LogCategories } from '../../logger';
import { serviceEnv } from '../../../env';
const logger = getLogger(LogCategories.MODULE.DATASET.FILE);
......@@ -109,7 +110,7 @@ export const getImageBase64 = async (url: string) => {
const buffer = Buffer.from(response.data);
const base64 = buffer.toString('base64');
const headerContentType = getContentTypeFromHeader(response.headers['content-type']);
const headerContentType = getAxiosContentType(response?.headers?.['content-type']);
// 检测图片类型的优先级策略
const imageType = (() => {
......@@ -140,11 +141,12 @@ export const getImageBase64 = async (url: string) => {
};
export const addEndpointToImageUrl = (text: string) => {
const baseURL = process.env.FE_DOMAIN;
const subRoute = process.env.NEXT_PUBLIC_BASE_URL || '';
const baseURL = serviceEnv.FE_DOMAIN;
const subRoute = serviceEnv.NEXT_PUBLIC_BASE_URL;
if (!baseURL) return text;
const escapedSubRoute = subRoute.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const regex = new RegExp(
`(?<!https?:\\/\\/[^\\s]*)(?:${subRoute}\\/api\\/system\\/img\\/[^\\s.]*\\.[^\\s]*)`,
`(?<!https?:\\/\\/[^\\s]*)(?:${escapedSubRoute}\\/api\\/system\\/img\\/[^\\s.]*\\.[^\\s]*)`,
'g'
);
// 匹配 ${subRoute}/api/system/img/xxx.xx 的图片链接，并追加 baseURL
......
packages/service/common/file/utils.ts
......@@ -20,10 +20,6 @@ export const removeFilesByPaths = (paths: string[]) => {
});
};
export const getContentTypeFromHeader = (header: string): string | undefined => {
return header?.toLowerCase()?.split(';')?.[0]?.trim();
};
export const clearDirFiles = (dirPath: string) => {
if (!fs.existsSync(dirPath)) {
return;
......
packages/service/common/logger/client.ts
import { configureLoggerFromEnv, disposeLogger, getLogger } from '@fastgpt-sdk/otel/logger';
export async function configureLogger() {
const { env } = await import('../../env');
const { serviceEnv } = await import('../../env');
await configureLoggerFromEnv({
env,
env: serviceEnv,
defaultCategory: ['system'],
defaultServiceName: 'fastgpt-client',
sensitiveProperties: ['fastgpt']
......
packages/service/common/metrics/client.ts
......@@ -3,12 +3,12 @@ import {
disposeMetrics as disposeOtelMetrics,
getMeter
} from '@fastgpt-sdk/otel/metrics';
import { env } from '../../env';
import { serviceEnv } from '../../env';
import { startRuntimeMetrics, stopRuntimeMetrics } from './runtime';
export async function configureMetrics() {
await configureMetricsFromEnv({
env,
env: serviceEnv,
defaultServiceName: 'fastgpt-client',
defaultMeterName: 'fastgpt-client'
});
......
packages/service/common/middle/cors.ts
import type { NextApiResponse, NextApiRequest } from 'next';
import NextCors from 'nextjs-cors';
import { serviceEnv } from '../../env';
export async function withNextCors(req: NextApiRequest, res: NextApiResponse) {
const methods = ['GET', 'HEAD', 'PUT', 'PATCH', 'POST', 'DELETE'];
const allowedOrigins = process.env.ALLOWED_ORIGINS?.split(',');
const allowedOrigins = serviceEnv.ALLOWED_ORIGINS?.split(',');
const origin = req.headers.origin;
await NextCors(req, res, {
......
packages/service/common/middle/reqFrequencyLimit.ts
......@@ -4,6 +4,7 @@ import { authFrequencyLimit } from '../system/frequencyLimit/utils';
import { addSeconds } from 'date-fns';
import { type NextApiResponse } from 'next';
import { jsonRes } from '../response';
import { serviceEnv } from '../../env';
// unit: times/s
// how to use?
......@@ -21,7 +22,7 @@ export function useIPFrequencyLimit({
}) {
return async (req: ApiRequestProps, res: NextApiResponse) => {
const ip = requestIp.getClientIp(req);
if (!ip || (process.env.USE_IP_LIMIT !== 'true' && !force)) {
if (!ip || (!serviceEnv.USE_IP_LIMIT && !force)) {
return;
}
try {
......
packages/service/common/middle/tracks/processor.ts
......@@ -2,10 +2,11 @@ import { delay } from '@fastgpt/global/common/system/utils';
import { TrackModel } from './schema';
import { TrackEnum } from '@fastgpt/global/common/middle/tracks/constants';
import { getLogger, LogCategories } from '../../logger';
import { serviceEnv } from '../../../env';
const logger = getLogger(LogCategories.EVENT.TRACK);
const batchUpdateTime = Number(process.env.TRACK_BATCH_UPDATE_TIME || 10000);
const batchUpdateTime = serviceEnv.TRACK_BATCH_UPDATE_TIME;
const getCurrentTenMinuteBoundary = () => {
const now = new Date();
......
packages/service/common/mongo/index.ts
......@@ -8,6 +8,7 @@ import type {
PipelineStage
} from 'mongoose';
import mongoose, { Mongoose } from 'mongoose';
import { serviceEnv } from '../../env';
const logger = getLogger(LogCategories.INFRA.MONGO);
......@@ -21,8 +22,8 @@ export type {
PipelineStage
};
export const MONGO_URL = process.env.MONGODB_URI as string;
export const MONGO_LOG_URL = (process.env.MONGODB_LOG_URI ?? process.env.MONGODB_URI) as string;
export const MONGO_URL = serviceEnv.MONGODB_URI;
export const MONGO_LOG_URL = serviceEnv.MONGODB_LOG_URI ?? serviceEnv.MONGODB_URI;
export const connectionMongo = (() => {
if (!global.mongodb) {
......@@ -156,8 +157,8 @@ export const getMongoLogModel = <T>(name: string, schema: mongoose.Schema): Mode
const syncMongoIndex = async (model: Model<any>) => {
if (
process.env.NODE_ENV === 'test' ||
process.env.SYNC_INDEX === '0' ||
process.env.NEXT_PHASE === 'phase-production-build' ||
!serviceEnv.SYNC_INDEX ||
!MONGO_URL
) {
return;
......
packages/service/common/mongo/init.ts
import { delay } from '@fastgpt/global/common/system/utils';
import { getLogger, LogCategories } from '../logger';
import type { Mongoose } from 'mongoose';
import { serviceEnv } from '../../env';
const logger = getLogger(LogCategories.INFRA.MONGO);
const maxConnecting = Math.max(30, Number(process.env.DB_MAX_LINK || 20));
const maxConnecting = Math.max(30, serviceEnv.DB_MAX_LINK);
/**
* connect MongoDB and init data
......
packages/service/common/redis/index.ts
import { getLogger, LogCategories } from '../logger';
import Redis from 'ioredis';
import type { RedisOptions } from 'ioredis';
import { serviceEnv } from '../../env';
const logger = getLogger(LogCategories.INFRA.REDIS);
const REDIS_URL = process.env.REDIS_URL ?? 'redis://localhost:6379';
const REDIS_URL = serviceEnv.REDIS_URL;
// Base Redis options for connection reliability
const REDIS_BASE_OPTION = {
......
packages/service/common/s3/config/constants.ts
......@@ -4,11 +4,11 @@ import type {
IOssStorageOptions,
IStorageOptions
} from '@fastgpt-sdk/storage';
import { env } from '../../../env';
import { serviceEnv } from '../../../env';
export const S3Buckets = {
public: env.STORAGE_PUBLIC_BUCKET,
private: env.STORAGE_PRIVATE_BUCKET
public: serviceEnv.STORAGE_PUBLIC_BUCKET,
private: serviceEnv.STORAGE_PRIVATE_BUCKET
} as const;
export const getSystemMaxFileSize = () => global.feConfigs.uploadFileMaxSize || 1024; // MB, 默认 1024MB;
......@@ -21,11 +21,11 @@ type BucketStorageOptions = {
externalEndpoint?: string;
};
const storageRegion = env.STORAGE_REGION;
const storageExternalEndpoint = env.STORAGE_EXTERNAL_ENDPOINT;
const storageS3Endpoint = env.STORAGE_S3_ENDPOINT;
export const storageDownloadMode = env.STORAGE_EXTERNAL_ENDPOINT ? 'presigned' : 'proxy';
const storagePublicAccessExtraSubPath = env.STORAGE_PUBLIC_ACCESS_EXTRA_SUB_PATH;
const storageRegion = serviceEnv.STORAGE_REGION;
const storageExternalEndpoint = serviceEnv.STORAGE_EXTERNAL_ENDPOINT;
const storageS3Endpoint = serviceEnv.STORAGE_S3_ENDPOINT;
export const storageDownloadMode = serviceEnv.STORAGE_EXTERNAL_ENDPOINT ? 'presigned' : 'proxy';
const storagePublicAccessExtraSubPath = serviceEnv.STORAGE_PUBLIC_ACCESS_EXTRA_SUB_PATH;
const bucketStorageOptions = {
publicBucket: S3Buckets.public,
......@@ -34,13 +34,13 @@ const bucketStorageOptions = {
} satisfies BucketStorageOptions;
const awsCompatibleSharedOptions = {
forcePathStyle: env.STORAGE_S3_FORCE_PATH_STYLE,
maxRetries: env.STORAGE_S3_MAX_RETRIES,
forcePathStyle: serviceEnv.STORAGE_S3_FORCE_PATH_STYLE,
maxRetries: serviceEnv.STORAGE_S3_MAX_RETRIES,
publicAccessExtraSubPath: storagePublicAccessExtraSubPath
};
export function createDefaultStorageOptions() {
const vendor = env.STORAGE_VENDOR as IStorageOptions['vendor'];
const vendor = serviceEnv.STORAGE_VENDOR as IStorageOptions['vendor'];
switch (vendor) {
case 'minio': {
......@@ -49,8 +49,8 @@ export function createDefaultStorageOptions() {
endpoint: storageS3Endpoint,
region: storageRegion,
credentials: {
accessKeyId: env.STORAGE_ACCESS_KEY_ID,
secretAccessKey: env.STORAGE_SECRET_ACCESS_KEY
accessKeyId: serviceEnv.STORAGE_ACCESS_KEY_ID,
secretAccessKey: serviceEnv.STORAGE_SECRET_ACCESS_KEY
},
...bucketStorageOptions,
...awsCompatibleSharedOptions
......@@ -63,8 +63,8 @@ export function createDefaultStorageOptions() {
endpoint: storageS3Endpoint,
region: storageRegion,
credentials: {
accessKeyId: env.STORAGE_ACCESS_KEY_ID,
secretAccessKey: env.STORAGE_SECRET_ACCESS_KEY
accessKeyId: serviceEnv.STORAGE_ACCESS_KEY_ID,
secretAccessKey: serviceEnv.STORAGE_SECRET_ACCESS_KEY
},
...bucketStorageOptions,
...awsCompatibleSharedOptions
......@@ -76,13 +76,13 @@ export function createDefaultStorageOptions() {
vendor: 'cos',
region: storageRegion,
credentials: {
accessKeyId: env.STORAGE_ACCESS_KEY_ID,
secretAccessKey: env.STORAGE_SECRET_ACCESS_KEY
accessKeyId: serviceEnv.STORAGE_ACCESS_KEY_ID,
secretAccessKey: serviceEnv.STORAGE_SECRET_ACCESS_KEY
},
protocol: env.STORAGE_COS_PROTOCOL,
useAccelerate: env.STORAGE_COS_USE_ACCELERATE,
domain: env.STORAGE_COS_CNAME_DOMAIN,
proxy: env.STORAGE_COS_PROXY,
protocol: serviceEnv.STORAGE_COS_PROTOCOL,
useAccelerate: serviceEnv.STORAGE_COS_USE_ACCELERATE,
domain: serviceEnv.STORAGE_COS_CNAME_DOMAIN,
proxy: serviceEnv.STORAGE_COS_PROXY,
...bucketStorageOptions
} satisfies Omit<ICosStorageOptions, 'bucket'> & BucketStorageOptions;
}
......@@ -90,16 +90,16 @@ export function createDefaultStorageOptions() {
case 'oss': {
return {
vendor: 'oss',
endpoint: env.STORAGE_OSS_ENDPOINT,
endpoint: serviceEnv.STORAGE_OSS_ENDPOINT,
region: storageRegion,
credentials: {
accessKeyId: env.STORAGE_ACCESS_KEY_ID,
secretAccessKey: env.STORAGE_SECRET_ACCESS_KEY
accessKeyId: serviceEnv.STORAGE_ACCESS_KEY_ID,
secretAccessKey: serviceEnv.STORAGE_SECRET_ACCESS_KEY
},
cname: env.STORAGE_OSS_CNAME,
internal: env.STORAGE_OSS_INTERNAL,
secure: env.STORAGE_OSS_SECURE,
enableProxy: env.STORAGE_OSS_ENABLE_PROXY,
cname: serviceEnv.STORAGE_OSS_CNAME,
internal: serviceEnv.STORAGE_OSS_INTERNAL,
secure: serviceEnv.STORAGE_OSS_SECURE,
enableProxy: serviceEnv.STORAGE_OSS_ENABLE_PROXY,
...bucketStorageOptions
} satisfies Omit<IOssStorageOptions, 'bucket'> & BucketStorageOptions;
}
......
packages/service/common/s3/security/token.ts
import jwt from 'jsonwebtoken';
import { differenceInSeconds } from 'date-fns';
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import { EndpointUrl } from '@fastgpt/global/common/file/constants';
import type { UploadConstraints } from '../contracts/type';
import path from 'path';
import { env } from '../../../env';
import { serviceEnv } from '../../../env';
/* ==================== 路由与类型 ==================== */
const FileApiPath = {
......@@ -49,8 +48,6 @@ type SignS3UploadTokenParams = {
};
/* ==================== 通用工具函数 ==================== */
const getTokenSecret = () => env.FILE_TOKEN_KEY;
const getExpiresIn = (expiredTime: Date) => {
return Math.max(1, differenceInSeconds(expiredTime, new Date()));
};
......@@ -62,8 +59,10 @@ const isNonEmptyString = (val: unknown): val is string => typeof val === 'string
const isStringArray = (val: unknown): val is string[] =>
Array.isArray(val) && val.every(isNonEmptyString);
const endpointUrl = `${serviceEnv.FILE_DOMAIN || serviceEnv.FE_DOMAIN || ''}${serviceEnv.NEXT_PUBLIC_BASE_URL}`;
const buildFileApiUrl = (apiPath: string, token: string, query = '') => {
return `${EndpointUrl}${apiPath}/${token}${query}`;
return `${endpointUrl}${apiPath}/${token}${query}`;
};
const parsePayload = <T>(payload: unknown, checker: (value: unknown) => value is T): T => {
......@@ -74,14 +73,14 @@ const parsePayload = <T>(payload: unknown, checker: (value: unknown) => value is
};
const signToken = <T extends object>(payload: T, expiredTime: Date) => {
return jwt.sign(payload, getTokenSecret(), {
return jwt.sign(payload, serviceEnv.FILE_TOKEN_KEY, {
expiresIn: getExpiresIn(expiredTime)
});
};
const verifyToken = <T>(token: string, checker: (value: unknown) => value is T) => {
return new Promise<T>((resolve, reject) => {
jwt.verify(token, getTokenSecret(), (err, payload) => {
jwt.verify(token, serviceEnv.FILE_TOKEN_KEY, (err, payload) => {
if (err) {
return reject(ERROR_ENUM.unAuthFile);
}
......
packages/service/common/s3/validation/upload.ts
......@@ -4,7 +4,7 @@ import path from 'node:path';
import type { UploadConstraints } from '../contracts/type';
import { DEFAULT_CONTENT_TYPE, resolveMimeType } from '../utils/mime';
import { normalizeAllowedExtensions, normalizeFileExtension } from '../utils/uploadConstraints';
import { env } from '../../../env';
import { serviceEnv } from '../../../env';
const defaultInspectBytes = 8192;
const officeZipInspectBytes = 64 * 1024;
......@@ -173,7 +173,7 @@ export async function validateUploadFile({
uploadConstraints
});
if (env.SKIP_FILE_TYPE_CHECK) {
if (serviceEnv.SKIP_FILE_TYPE_CHECK) {
return {
filename: normalizedFileName,
contentType: expectedMime
......
packages/service/common/secret/aes256gcm.ts
import crypto from 'crypto';
import { AES256_SECRET_KEY } from './constants';
import { serviceEnv } from '../../env';
const AES256_SECRET_KEY = serviceEnv.AES256_SECRET_KEY;
export const encryptSecret = (text: string) => {
const iv = crypto.randomBytes(16);
const key = crypto.scryptSync(AES256_SECRET_KEY, 'salt', 32);
......
packages/service/common/secret/constants.ts deleted 100644 → 0
export const AES256_SECRET_KEY = process.env.AES256_SECRET_KEY || 'fastgptkey';
packages/service/common/security/fileUrlValidator.ts
import { env } from '../../env';
import { serviceEnv } from '../../env';
const appendHost = ({
list,
......@@ -22,20 +22,10 @@ const appendHost = ({
const systemWhiteList = (() => {
const list: string[] = [];
appendHost({ list, value: env.STORAGE_S3_ENDPOINT, allowRawHost: true });
appendHost({ list, value: env.STORAGE_EXTERNAL_ENDPOINT });
if (process.env.FE_DOMAIN) {
try {
const urlData = new URL(process.env.FE_DOMAIN);
list.push(urlData.hostname);
} catch (error) {}
}
if (process.env.PRO_URL) {
try {
const urlData = new URL(process.env.PRO_URL);
list.push(urlData.hostname);
} catch (error) {}
}
appendHost({ list, value: serviceEnv.STORAGE_S3_ENDPOINT, allowRawHost: true });
appendHost({ list, value: serviceEnv.STORAGE_EXTERNAL_ENDPOINT });
appendHost({ list, value: serviceEnv.FE_DOMAIN });
appendHost({ list, value: serviceEnv.PRO_URL });
return list;
})();
......
packages/service/common/string/utils.ts
import { WorkerNameEnum, runWorker } from '../../worker/utils';
import { serviceEnv } from '../../env';
import { WorkerNameEnum, getWorkerController } from '../../worker/utils';
import { type ImageType } from '../../worker/readFile/type';
export const htmlToMarkdown = async (html?: string | null) => {
const md = await runWorker<{
rawText: string;
imageList: ImageType[];
}>(WorkerNameEnum.htmlStr2Md, { html: html || '' });
const workerController = getWorkerController<
{ html: string },
{
rawText: string;
imageList: ImageType[];
}
>({
name: WorkerNameEnum.htmlStr2Md,
maxReservedThreads: serviceEnv.HTML_TO_MARKDOWN_WORKERS,
taskTimeoutMs: 300000,
maxTasksPerWorker: 100
});
const md = await workerController.run({ html: html || '' });
return md.rawText;
};
packages/service/common/system/constants.ts
export const FastGPTProUrl = process.env.PRO_URL ? `${process.env.PRO_URL}/api` : '';
export const FastGPTPluginUrl = process.env.PLUGIN_BASE_URL ? `${process.env.PLUGIN_BASE_URL}` : '';
import { serviceEnv } from '../../env';
export const FastGPTProUrl = serviceEnv.PRO_URL ? `${serviceEnv.PRO_URL}/api` : '';
export const FastGPTPluginUrl = serviceEnv.PLUGIN_BASE_URL ?? '';
// @ts-ignore
export const isFastGPTProService = () => !!global.systemConfig;
......@@ -8,7 +10,7 @@ export const isProVersion = () => {
};
export const serviceRequestMaxContentLength =
Number(process.env.SERVICE_REQUEST_MAX_CONTENT_LENGTH || 10) * 1024 * 1024; // 10MB
serviceEnv.SERVICE_REQUEST_MAX_CONTENT_LENGTH * 1024 * 1024;
export const InitialErrorEnum = {
S3_ERROR: 's3_error',
......
packages/service/common/system/tools.ts
import { type FastGPTConfigFileType } from '@fastgpt/global/common/system/types';
import { isIPv6 } from 'net';
import { getLogger, LogCategories } from '../logger';
import { serviceEnv } from '../../env';
const logger = getLogger(LogCategories.ERROR);
......@@ -19,8 +20,8 @@ export const initFastGPTConfig = (config?: FastGPTConfigFileType) => {
!!config.systemEnv.customPdfParse?.textinAppId ||
!!config.systemEnv.customPdfParse?.doc2xKey;
config.feConfigs.customPdfParsePrice = config.systemEnv.customPdfParse?.price || 0;
config.feConfigs.uploadFileMaxSize = Number(process.env.UPLOAD_FILE_MAX_SIZE || 1000);
config.feConfigs.uploadFileMaxAmount = Number(process.env.UPLOAD_FILE_MAX_AMOUNT || 1000);
config.feConfigs.uploadFileMaxSize = serviceEnv.UPLOAD_FILE_MAX_SIZE;
config.feConfigs.uploadFileMaxAmount = serviceEnv.UPLOAD_FILE_MAX_AMOUNT;
global.feConfigs = config.feConfigs;
global.systemEnv = config.systemEnv;
......
packages/service/common/system/utils.ts
import ipaddr from 'ipaddr.js';
import { isIPv6 } from 'net';
import dns from 'dns/promises';
import {
createInternalAddressChecker,
PRIVATE_URL_TEXT
} from '@fastgpt/global/common/system/network';
import { serviceEnv } from '../../env';
const isDevEnv = process.env.NODE_ENV === 'development';
const SERVICE_LOCAL_PORT = `${process.env.PORT || 3000}`;
const SERVICE_LOCAL_HOST =
process.env.HOSTNAME && isIPv6(process.env.HOSTNAME)
? `[${process.env.HOSTNAME}]:${SERVICE_LOCAL_PORT}`
: `${process.env.HOSTNAME || 'localhost'}:${SERVICE_LOCAL_PORT}`;
const { isInternalAddress } = createInternalAddressChecker({
checkInternalIp: () => serviceEnv.CHECK_INTERNAL_IP
});
// 云厂商元数据服务 IP（除 169.254.0.0/16 段外的特殊地址）
// 预先归一化为 ipaddr.js 的 normalizedString 形式以便比对
const METADATA_IPS = new Set<string>(
[
'100.100.100.200', // 阿里云
'fd00:ec2::254' // AWS IPv6
].map((ip) => ipaddr.parse(ip).toNormalizedString().toLowerCase())
);
// 云厂商元数据服务主机名（归一化：小写、去尾部点）
const METADATA_HOSTNAMES = new Set<string>([
'metadata.google.internal',
'metadata',
'metadata.tencentyun.com',
'kubernetes.default.svc',
'kubernetes.default',
'kubernetes'
]);
const LOCALHOST_HOSTNAMES = new Set<string>(['localhost']);
/**
* 把 URL hostname 尝试解析成 ipaddr.js 的地址对象
* - 处理 IPv6 方括号
* - 处理 IPv4-mapped IPv6 (::ffff:a.b.c.d / ::ffff:xxxx:xxxx) → 解包为 IPv4
* - 处理十进制/十六进制/八进制/短点分 IPv4 字面量
* 非 IP 字面量返回 null
*/
const parseHostAsIP = (rawHostname: string): ipaddr.IPv4 | ipaddr.IPv6 | null => {
const host = rawHostname.replace(/^\[|\]$/g, '').replace(/\.$/, '');
if (!host) return null;
// ipaddr.process 会自动把 IPv4-mapped IPv6 解包为 IPv4，处理常规字面量
if (ipaddr.isValid(host)) {
try {
return ipaddr.process(host);
} catch {
return null;
}
}
// ipaddr.js 不支持十进制/十六进制/八进制 IPv4 短写，手动兜底
const numeric = parseNumericIPv4(host);
if (numeric) return ipaddr.parse(numeric) as ipaddr.IPv4;
return null;
};
/**
* 解析 inet_aton 兼容的 IPv4 字面量：十进制 2852039166、十六进制 0xa9fea9fe、
* 八进制、1-4 段形式（含 dec/hex/oct 混合）。返回标准点分十进制或 null
*/
const parseNumericIPv4 = (host: string): string | null => {
const parts = host.split('.');
if (parts.length === 0 || parts.length > 4) return null;
const nums: number[] = [];
for (const part of parts) {
if (!part) return null;
let n: number;
if (/^0x[0-9a-f]+$/i.test(part)) n = parseInt(part, 16);
else if (/^0[0-7]+$/.test(part)) n = parseInt(part, 8);
else if (/^\d+$/.test(part)) n = parseInt(part, 10);
else return null;
if (!Number.isFinite(n) || n < 0) return null;
nums.push(n);
}
const maxLast = [0xffffffff, 0xffffff, 0xffff, 0xff][parts.length - 1];
if (nums[nums.length - 1] > maxLast) return null;
for (let i = 0; i < nums.length - 1; i++) if (nums[i] > 0xff) return null;
let ipInt = 0;
for (let i = 0; i < nums.length - 1; i++) ipInt = (ipInt + nums[i]) * 256;
ipInt += nums[nums.length - 1];
if (ipInt > 0xffffffff) return null;
return [(ipInt >>> 24) & 0xff, (ipInt >>> 16) & 0xff, (ipInt >>> 8) & 0xff, ipInt & 0xff].join(
'.'
);
};
const normalizeDomain = (rawHostname: string): string =>
rawHostname
.replace(/^\[|\]$/g, '')
.replace(/\.$/, '')
.toLowerCase();
/**
* ipaddr.js range() 返回的所有非 'unicast' 分类都视为内部地址。
* 主要范围：private / loopback / linkLocal / uniqueLocal / reserved /
* multicast / broadcast / unspecified / carrierGradeNat 等
*/
const isInternalIPAddress = (addr: ipaddr.IPv4 | ipaddr.IPv6): boolean => {
return addr.range() !== 'unicast';
};
/**
* 元数据端点：
* - 169.254.0.0/16 link-local 段全部视为元数据
* - 显式列表里的 IP（阿里云 100.100.100.200、AWS IPv6 fd00:ec2::254）
*/
const isMetadataIPAddress = (addr: ipaddr.IPv4 | ipaddr.IPv6): boolean => {
if (addr.kind() === 'ipv4' && addr.range() === 'linkLocal') return true;
return METADATA_IPS.has(addr.toNormalizedString().toLowerCase());
};
export const isInternalAddress = async (url: string): Promise<boolean> => {
if (isDevEnv) return false;
let parsedUrl: URL;
try {
parsedUrl = new URL(url);
} catch {
return false;
}
const hostDomain = normalizeDomain(parsedUrl.hostname);
const localHost = SERVICE_LOCAL_HOST.split(':')[0].toLowerCase();
// 1. localhost / 本机
if (LOCALHOST_HOSTNAMES.has(hostDomain) || hostDomain === localHost) {
return true;
}
// 2. 云元数据主机名
if (METADATA_HOSTNAMES.has(hostDomain)) {
return true;
}
// 3. IP 字面量（含各种编码变体）
const ip = parseHostAsIP(parsedUrl.hostname);
const checkFullInternal = process.env.CHECK_INTERNAL_IP === 'true';
if (ip) {
if (isMetadataIPAddress(ip)) return true;
// loopback/unspecified 等始终阻止（这些是显而易见的错误配置或攻击）
const range = ip.range();
if (range === 'loopback' || range === 'unspecified') return true;
if (checkFullInternal) return isInternalIPAddress(ip);
return false;
}
// 4. 域名：解析 DNS；元数据命中始终阻止，私有段受 CHECK_INTERNAL_IP 控制
try {
const [v4Res, v6Res] = await Promise.allSettled([
dns.resolve4(hostDomain),
dns.resolve6(hostDomain)
]);
const resolvedIPs = [
...(v4Res.status === 'fulfilled' ? v4Res.value : []),
...(v6Res.status === 'fulfilled' ? v6Res.value : [])
];
for (const raw of resolvedIPs) {
if (!ipaddr.isValid(raw)) continue;
const addr = ipaddr.process(raw);
if (isMetadataIPAddress(addr)) return true;
const r = addr.range();
if (r === 'loopback' || r === 'unspecified') return true;
if (checkFullInternal && isInternalIPAddress(addr)) return true;
}
return false;
} catch {
return false;
}
};
export const PRIVATE_URL_TEXT = 'Request to private network not allowed';
export { isInternalAddress, PRIVATE_URL_TEXT };
/**
* 用于"保存配置 URL"或"调用前校验"的统一安全检查:
......
packages/service/common/tracing/client.ts
......@@ -7,7 +7,7 @@ import {
getTracer
} from '@fastgpt-sdk/otel/tracing';
import { withContext } from '../logger';
import { env } from '../../env';
import { serviceEnv } from '../../env';
type SpanAttributeValue = string | number | boolean;
type SpanStatusLike = {
......@@ -33,8 +33,8 @@ const DEFAULT_PRODUCTION_TRACING_SAMPLE_RATIO = 0.01;
const DEFAULT_NON_PRODUCTION_TRACING_SAMPLE_RATIO = 1;
function getDefaultTracingSampleRatio() {
if (typeof env.TRACING_OTEL_SAMPLE_RATIO === 'number') {
return env.TRACING_OTEL_SAMPLE_RATIO;
if (typeof serviceEnv.TRACING_OTEL_SAMPLE_RATIO === 'number') {
return serviceEnv.TRACING_OTEL_SAMPLE_RATIO;
}
return process.env.NODE_ENV === 'production'
......@@ -61,7 +61,7 @@ function normalizeAttributes(attributes?: Record<string, unknown>) {
export async function configureTracing() {
await configureTracingFromEnv({
env,
env: serviceEnv,
defaultServiceName: 'fastgpt-client',
defaultTracerName: 'fastgpt-client',
defaultSampleRatio: getDefaultTracingSampleRatio()
......
packages/service/common/vectorDB/constants.ts
import { serviceEnv } from '../../env';
export const DatasetVectorDbName = 'fastgpt';
export const DatasetVectorTableName = 'modeldata';
export const PG_ADDRESS = process.env.PG_URL;
export const OPENGAUSS_ADDRESS = process.env.OPENGAUSS_URL;
export const OCEANBASE_ADDRESS = process.env.OCEANBASE_URL;
export const SEEKDB_ADDRESS = process.env.SEEKDB_URL;
export const MILVUS_ADDRESS = process.env.MILVUS_ADDRESS;
export const MILVUS_TOKEN = process.env.MILVUS_TOKEN;
export const PG_ADDRESS = serviceEnv.PG_URL;
export const OPENGAUSS_ADDRESS = serviceEnv.OPENGAUSS_URL;
export const OCEANBASE_ADDRESS = serviceEnv.OCEANBASE_URL;
export const SEEKDB_ADDRESS = serviceEnv.SEEKDB_URL;
export const MILVUS_ADDRESS = serviceEnv.MILVUS_ADDRESS;
export const MILVUS_TOKEN = serviceEnv.MILVUS_TOKEN;
export const VectorVQ = (() => {
if (process.env.VECTOR_VQ_LEVEL === '32') {
if (serviceEnv.VECTOR_VQ_LEVEL === 32) {
return 32;
}
if (process.env.VECTOR_VQ_LEVEL === '16') {
if (serviceEnv.VECTOR_VQ_LEVEL === 16) {
return 16;
}
if (process.env.VECTOR_VQ_LEVEL === '8') {
if (serviceEnv.VECTOR_VQ_LEVEL === 8) {
return 8;
}
if (process.env.VECTOR_VQ_LEVEL === '4') {
if (serviceEnv.VECTOR_VQ_LEVEL === 4) {
return 4;
}
if (process.env.VECTOR_VQ_LEVEL === '2') {
if (serviceEnv.VECTOR_VQ_LEVEL === 2) {
return 2;
}
return 32;
......@@ -51,9 +53,9 @@ export const VectorVQ = (() => {
* ```
*/
export const OceanBaseIndexConfig = (() => {
const level = process.env.VECTOR_VQ_LEVEL;
const level = serviceEnv.VECTOR_VQ_LEVEL;
if (level === '1') {
if (level === 1) {
return {
type: 'hnsw_bq' as const,
distance: 'cosine' as const,
......@@ -63,7 +65,7 @@ export const OceanBaseIndexConfig = (() => {
};
}
if (level === '8') {
if (level === 8) {
return {
type: 'hnsw_sq' as const,
distance: 'inner_product' as const,
......
packages/service/common/vectorDB/oceanbase/controller.ts
......@@ -7,6 +7,7 @@ import mysql, {
import { getLogger, LogCategories } from '../../logger';
import { OCEANBASE_ADDRESS, SEEKDB_ADDRESS } from '../constants';
import { delay } from '@fastgpt/global/common/system/utils';
import { serviceEnv } from '../../../env';
const logger = getLogger(LogCategories.INFRA.VECTOR);
......@@ -50,7 +51,7 @@ export class ObClass {
global.obClient = mysql.createPool({
uri: address,
waitForConnections: true,
connectionLimit: Number(process.env.DB_MAX_LINK || 20),
connectionLimit: serviceEnv.DB_MAX_LINK,
connectTimeout: 20000,
idleTimeout: 60000,
queueLimit: 0,
......
packages/service/common/vectorDB/opengauss/controller.ts
......@@ -3,6 +3,7 @@ import { getLogger, LogCategories } from '../../logger';
import { Pool } from 'pg';
import type { QueryResultRow } from 'pg';
import { OPENGAUSS_ADDRESS } from '../constants';
import { serviceEnv } from '../../../env';
const logger = getLogger(LogCategories.INFRA.VECTOR);
......@@ -13,7 +14,7 @@ export const connectOg = async (): Promise<Pool> => {
const pool = new Pool({
connectionString: OPENGAUSS_ADDRESS,
max: Number(process.env.DB_MAX_LINK || 30),
max: serviceEnv.DB_MAX_LINK,
min: 15,
keepAlive: true,
idleTimeoutMillis: 1800000,
......
packages/service/common/vectorDB/pg/controller.ts
......@@ -3,6 +3,7 @@ import { getLogger, LogCategories } from '../../logger';
import { Pool } from 'pg';
import type { QueryResultRow } from 'pg';
import { PG_ADDRESS } from '../constants';
import { serviceEnv } from '../../../env';
const logger = getLogger(LogCategories.INFRA.POSTGRES);
......@@ -14,7 +15,7 @@ export const connectPg = async (): Promise<Pool> => {
const pool = new Pool({
connectionString: PG_ADDRESS,
// 连接池配置
max: Number(process.env.DB_MAX_LINK || 30), // 增加到 30，支持更高并发
max: serviceEnv.DB_MAX_LINK, // 支持通过统一 env 配置并发
min: 15, // 调整为 max 的 50%
keepAlive: true,
......
packages/service/core/agentSkills/sandboxConfig.ts
......@@ -11,7 +11,7 @@ import type {
import { createSandbox, type ISandbox, type OpenSandboxVolume } from '@fastgpt-sdk/sandbox-adapter';
import type { OpenSandboxConfigType, SandboxProviderType } from '@fastgpt-sdk/sandbox-adapter';
import type { OpenSandboxAdapter } from '@fastgpt-sdk/sandbox-adapter';
import { env } from '../../env';
import { serviceEnv } from '../../env';
type SandboxRuntime = 'kubernetes' | 'docker';
......@@ -74,24 +74,30 @@ function toOpenSandboxCreateConfig(
* Get sandbox provider configuration from environment variables
*/
export function getSandboxProviderConfig(): SandboxProviderConfig {
const provider = (env.AGENT_SANDBOX_PROVIDER ?? 'opensandbox') as SandboxProviderType;
const runtime = (env.AGENT_SANDBOX_OPENSANDBOX_RUNTIME ?? 'kubernetes') as SandboxRuntime;
const provider = serviceEnv.AGENT_SANDBOX_PROVIDER;
const runtime = serviceEnv.AGENT_SANDBOX_OPENSANDBOX_RUNTIME;
switch (provider) {
case 'opensandbox':
return {
provider,
baseUrl: env.AGENT_SANDBOX_OPENSANDBOX_BASEURL ?? 'http://127.0.0.1:8080',
apiKey: env.AGENT_SANDBOX_OPENSANDBOX_API_KEY,
baseUrl: serviceEnv.AGENT_SANDBOX_OPENSANDBOX_BASEURL,
apiKey: serviceEnv.AGENT_SANDBOX_OPENSANDBOX_API_KEY,
runtime,
useServerProxy: env.AGENT_SANDBOX_OPENSANDBOX_USE_SERVER_PROXY
useServerProxy: serviceEnv.AGENT_SANDBOX_OPENSANDBOX_USE_SERVER_PROXY
};
case 'sealosdevbox':
return {
provider,
baseUrl: env.AGENT_SANDBOX_SEALOS_BASEURL ?? env.AGENT_SANDBOX_OPENSANDBOX_BASEURL ?? '',
token: env.AGENT_SANDBOX_SEALOS_TOKEN ?? env.AGENT_SANDBOX_OPENSANDBOX_API_KEY ?? '',
baseUrl:
serviceEnv.AGENT_SANDBOX_SEALOS_BASEURL ??
serviceEnv.AGENT_SANDBOX_OPENSANDBOX_BASEURL ??
'',
token:
serviceEnv.AGENT_SANDBOX_SEALOS_TOKEN ??
serviceEnv.AGENT_SANDBOX_OPENSANDBOX_API_KEY ??
'',
runtime
};
......@@ -109,14 +115,14 @@ export function getSandboxProviderConfig(): SandboxProviderConfig {
export function getSandboxDefaults(): SandboxDefaults {
return {
defaultImage: {
repository: env.AGENT_SANDBOX_OPENSANDBOX_IMAGE_REPO ?? 'fastgpt-agent-sandbox',
tag: env.AGENT_SANDBOX_OPENSANDBOX_IMAGE_TAG ?? 'latest'
repository: serviceEnv.AGENT_SANDBOX_OPENSANDBOX_IMAGE_REPO,
tag: serviceEnv.AGENT_SANDBOX_OPENSANDBOX_IMAGE_TAG
},
workDirectory: '/home/sandbox/workspace',
// workDirectory: env.AGENT_SANDBOX_OPENSANDBOX_WORK_DIRECTORY ?? '/home/sandbox/workspace',
// workDirectory: serviceEnv.AGENT_SANDBOX_OPENSANDBOX_WORK_DIRECTORY ?? '/home/sandbox/workspace',
targetPort: 44772,
entrypoint: '/home/sandbox/entrypoint.sh'
// entrypoint: env.AGENT_SANDBOX_OPENSANDBOX_ENTRYPOINT ?? '/home/sandbox/entrypoint.sh'
// entrypoint: serviceEnv.AGENT_SANDBOX_OPENSANDBOX_ENTRYPOINT ?? '/home/sandbox/entrypoint.sh'
};
}
......@@ -125,10 +131,10 @@ export function getSandboxDefaults(): SandboxDefaults {
*/
export function getSkillSizeLimits(): SkillSizeLimits {
return {
maxUploadBytes: env.AGENT_SKILL_MAX_UPLOAD_SIZE ?? 50 * 1024 * 1024,
maxUncompressedBytes: env.AGENT_SKILL_MAX_UNCOMPRESSED_SIZE ?? 200 * 1024 * 1024,
maxDownloadBytes: env.AGENT_SKILL_MAX_DOWNLOAD_SIZE ?? 200 * 1024 * 1024,
maxSandboxPackageBytes: env.AGENT_SKILL_MAX_SANDBOX_SIZE ?? 200 * 1024 * 1024
maxUploadBytes: serviceEnv.AGENT_SKILL_MAX_UPLOAD_SIZE,
maxUncompressedBytes: serviceEnv.AGENT_SKILL_MAX_UNCOMPRESSED_SIZE,
maxDownloadBytes: serviceEnv.AGENT_SKILL_MAX_DOWNLOAD_SIZE,
maxSandboxPackageBytes: serviceEnv.AGENT_SKILL_MAX_SANDBOX_SIZE
};
}
......@@ -273,7 +279,7 @@ export function getVolumeManagerConfig(): VolumeManagerConfig {
AGENT_SANDBOX_VOLUME_MANAGER_URL,
AGENT_SANDBOX_VOLUME_MANAGER_TOKEN,
AGENT_SANDBOX_VOLUME_MANAGER_MOUNT_PATH
} = env;
} = serviceEnv;
if (
!AGENT_SANDBOX_VOLUME_MANAGER_URL ||
!AGENT_SANDBOX_VOLUME_MANAGER_TOKEN ||
......
packages/service/core/agentSkills/sandboxController.ts
......@@ -31,7 +31,7 @@ import { SandboxTypeEnum } from '@fastgpt/global/core/agentSkills/constants';
import { SandboxStatusEnum } from '@fastgpt/global/core/ai/sandbox/constants';
import { getSandboxClient, type SandboxClient } from '../ai/sandbox/controller';
import { getLogger, LogCategories } from '../../common/logger';
import { env } from '../../env';
import { serviceEnv } from '../../env';
import type { SandboxStatusItemType } from '@fastgpt/global/core/chat/type';
const addLog = getLogger(LogCategories.MODULE.AI.AGENT);
......@@ -212,7 +212,7 @@ export async function createEditDebugSandbox(
// Check active edit-debug sandbox count limit
const maxEditDebug =
global.feConfigs?.limit?.agentSandboxMaxEditDebug ?? env.AGENT_SANDBOX_MAX_EDIT_DEBUG;
global.feConfigs?.limit?.agentSandboxMaxEditDebug ?? serviceEnv.AGENT_SANDBOX_MAX_EDIT_DEBUG;
if (maxEditDebug !== undefined) {
const activeCount = await MongoSandboxInstance.countDocuments({
status: SandboxStatusEnum.running,
......
packages/service/core/ai/config.ts
import OpenAI from '@fastgpt/global/core/ai';
import { type OpenaiAccountType } from '@fastgpt/global/support/user/team/type';
import { serviceEnv } from '../../env';
const aiProxyBaseUrl = process.env.AIPROXY_API_ENDPOINT
? `${process.env.AIPROXY_API_ENDPOINT}/v1`
const aiProxyBaseUrl = serviceEnv.AIPROXY_API_ENDPOINT
? `${serviceEnv.AIPROXY_API_ENDPOINT}/v1`
: undefined;
const openaiBaseUrl = aiProxyBaseUrl || process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';
const openaiBaseKey = process.env.AIPROXY_API_TOKEN || process.env.CHAT_API_KEY || '';
export const openaiBaseUrl = aiProxyBaseUrl || serviceEnv.OPENAI_BASE_URL;
export const openaiBaseKey = aiProxyBaseUrl
? serviceEnv.AIPROXY_API_TOKEN || serviceEnv.CHAT_API_KEY
: serviceEnv.CHAT_API_KEY;
// 代理走 packages/service/common/proxy/index.ts 里的 EnvHttpProxyAgent + setGlobalDispatcher
export const getAIApi = (props?: { userKey?: OpenaiAccountType; timeout?: number }) => {
......
packages/service/core/ai/config/utils.ts
......@@ -23,6 +23,7 @@ import { refreshVersionKey } from '../../../common/cache';
import { SystemCacheKeyEnum } from '../../../common/cache/type';
import { getLogger, LogCategories } from '../../../common/logger';
import { getRuntimeResolvedPriceTiers } from '@fastgpt/global/core/ai/pricing';
import { serviceEnv } from '../../../env';
export const loadSystemModels = async (init = false, language = 'en') => {
if (!init && global.systemModelList) return;
......@@ -76,7 +77,7 @@ export const loadSystemModels = async (init = false, language = 'en') => {
if (model.isDefaultDatasetImageModel) {
_systemDefaultModel.datasetImageLLM = model;
}
if (model.model === process.env.HELPER_BOT_MODEL) {
if (model.model === serviceEnv.HELPER_BOT_MODEL) {
_systemDefaultModel.helperBotLLM = model;
}
} else if (model.type === ModelTypeEnum.embedding) {
......
packages/service/core/ai/llm/utils.ts
......@@ -15,6 +15,7 @@ import { getS3ChatSource } from '../../../common/s3/sources/chat';
import { isInternalAddress } from '../../../common/system/utils';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { getLogger, LogCategories } from '../../../common/logger';
import { serviceEnv } from '../../../env';
const logger = getLogger(LogCategories.MODULE.AI.LLM);
......@@ -166,7 +167,7 @@ export const loadRequestMessages = async ({
// If imgUrl is a local path, load image from local, and set url to base64
if (
imgUrl.startsWith('/') ||
process.env.MULTIPLE_DATA_TO_BASE64 !== 'false' ||
serviceEnv.MULTIPLE_DATA_TO_BASE64 ||
(await isInternalAddress(imgUrl))
) {
try {
......
packages/service/core/ai/record/schema.ts
import { getMongoLogModel, Schema } from '../../../common/mongo';
import type { LLMRequestRecordSchemaType } from '@fastgpt/global/openapi/core/ai/api';
import { serviceEnv } from '../../../env';
export const LLMRequestRecordCollectionName = 'llm_request_records';
const expiredHours = process.env.LLM_REQUEST_TRACKING_RETENTION_HOURS
? Number(process.env.LLM_REQUEST_TRACKING_RETENTION_HOURS)
: 6;
const expiredHours = serviceEnv.LLM_REQUEST_TRACKING_RETENTION_HOURS;
const LLMRequestRecordSchema = new Schema({
requestId: {
......
packages/service/core/ai/sandbox/config.ts
import { env } from '../../../env';
import { serviceEnv } from '../../../env';
import type {
OpenSandboxConfigType,
OpenSandboxConnectionConfig
......@@ -13,12 +13,12 @@ export type SealosConnectionConfig = {
};
export const getSealosConnectionConfig = (sandboxId: string): SealosConnectionConfig => {
if (!env.AGENT_SANDBOX_SEALOS_BASEURL || !env.AGENT_SANDBOX_SEALOS_TOKEN) {
if (!serviceEnv.AGENT_SANDBOX_SEALOS_BASEURL || !serviceEnv.AGENT_SANDBOX_SEALOS_TOKEN) {
throw new Error('AGENT_SANDBOX_SEALOS_BASEURL / AGENT_SANDBOX_SEALOS_TOKEN required');
}
return {
baseUrl: env.AGENT_SANDBOX_SEALOS_BASEURL,
token: env.AGENT_SANDBOX_SEALOS_TOKEN,
baseUrl: serviceEnv.AGENT_SANDBOX_SEALOS_BASEURL,
token: serviceEnv.AGENT_SANDBOX_SEALOS_TOKEN,
sandboxId
};
};
......@@ -29,15 +29,15 @@ export const getOpenSandboxConnectionConfig = ({
}: {
sessionId: string;
}): OpenSandboxConnectionConfig => {
if (!env.AGENT_SANDBOX_OPENSANDBOX_BASEURL) {
if (!serviceEnv.AGENT_SANDBOX_OPENSANDBOX_BASEURL) {
throw new Error('AGENT_SANDBOX_OPENSANDBOX_BASEURL is required');
}
return {
sessionId,
useServerProxy: env.AGENT_SANDBOX_OPENSANDBOX_USE_SERVER_PROXY,
baseUrl: env.AGENT_SANDBOX_OPENSANDBOX_BASEURL,
apiKey: env.AGENT_SANDBOX_OPENSANDBOX_API_KEY,
runtime: env.AGENT_SANDBOX_OPENSANDBOX_RUNTIME
useServerProxy: serviceEnv.AGENT_SANDBOX_OPENSANDBOX_USE_SERVER_PROXY,
baseUrl: serviceEnv.AGENT_SANDBOX_OPENSANDBOX_BASEURL,
apiKey: serviceEnv.AGENT_SANDBOX_OPENSANDBOX_API_KEY,
runtime: serviceEnv.AGENT_SANDBOX_OPENSANDBOX_RUNTIME
};
};
......@@ -48,13 +48,13 @@ export const buildOpenSandboxCreateConfig = (
createConfig?: OpenSandboxConfigType;
} = {}
): OpenSandboxConfigType => {
if (!env.AGENT_SANDBOX_OPENSANDBOX_IMAGE_REPO && !opts.createConfig?.image) {
if (!serviceEnv.AGENT_SANDBOX_OPENSANDBOX_IMAGE_REPO && !opts.createConfig?.image) {
throw new Error('AGENT_SANDBOX_OPENSANDBOX_IMAGE_REPO is required for opensandbox provider');
}
return {
image: {
repository: env.AGENT_SANDBOX_OPENSANDBOX_IMAGE_REPO,
tag: env.AGENT_SANDBOX_OPENSANDBOX_IMAGE_TAG
repository: serviceEnv.AGENT_SANDBOX_OPENSANDBOX_IMAGE_REPO,
tag: serviceEnv.AGENT_SANDBOX_OPENSANDBOX_IMAGE_TAG
},
...(opts.resourceLimits ? { resourceLimits: opts.resourceLimits } : {}),
...opts.createConfig,
......@@ -73,10 +73,10 @@ export type VolumeManagerResult = {
storage: SandboxStorageType;
};
const vmConfig = {
enable: env.AGENT_SANDBOX_ENABLE_VOLUME,
url: env.AGENT_SANDBOX_VOLUME_MANAGER_URL!,
token: env.AGENT_SANDBOX_VOLUME_MANAGER_TOKEN,
mountPath: env.AGENT_SANDBOX_VOLUME_MANAGER_MOUNT_PATH
enable: serviceEnv.AGENT_SANDBOX_ENABLE_VOLUME,
url: serviceEnv.AGENT_SANDBOX_VOLUME_MANAGER_URL!,
token: serviceEnv.AGENT_SANDBOX_VOLUME_MANAGER_TOKEN,
mountPath: serviceEnv.AGENT_SANDBOX_VOLUME_MANAGER_MOUNT_PATH
};
export const buildVolumeConfig = (claimName: string, mountPath: string): VolumeManagerResult => {
return {
......
packages/service/core/ai/sandbox/controller.ts
......@@ -3,7 +3,7 @@ import {
SandboxStatusEnum,
SANDBOX_SUSPEND_MINUTES
} from '@fastgpt/global/core/ai/sandbox/constants';
import { env } from '../../../env';
import { serviceEnv } from '../../../env';
import { MongoSandboxInstance } from './schema';
import {
createSandbox,
......@@ -58,7 +58,7 @@ export class SandboxClient {
this.userId = props.userId;
this.chatId = props.chatId;
const providerName = env.AGENT_SANDBOX_PROVIDER;
const providerName = serviceEnv.AGENT_SANDBOX_PROVIDER;
if (providerName === 'sealosdevbox') {
const config = getSealosConnectionConfig(this.sandboxId);
......@@ -76,11 +76,11 @@ export class SandboxClient {
})
);
} else if (providerName === 'e2b') {
if (!env.AGENT_SANDBOX_E2B_API_KEY) {
if (!serviceEnv.AGENT_SANDBOX_E2B_API_KEY) {
throw new Error('AGENT_SANDBOX_E2B_API_KEY required');
}
this.provider = createSandbox('e2b', {
apiKey: env.AGENT_SANDBOX_E2B_API_KEY,
apiKey: serviceEnv.AGENT_SANDBOX_E2B_API_KEY,
sandboxId: this.sandboxId
});
} else if (!providerName) {
......@@ -88,7 +88,7 @@ export class SandboxClient {
'AGENT_SANDBOX_PROVIDER is not configured. Please set it in your environment variables.'
);
} else {
throw new Error(`Unsupported sandbox provider: ${env.AGENT_SANDBOX_PROVIDER}`);
throw new Error(`Unsupported sandbox provider: ${serviceEnv.AGENT_SANDBOX_PROVIDER}`);
}
}
......
packages/service/core/app/evaluation/mq.ts
import { getQueue, getWorker, QueueNames } from '../../../common/bullmq';
import { type Processor } from 'bullmq';
import { getLogger, LogCategories } from '../../../common/logger';
import { serviceEnv } from '../../../env';
const logger = getLogger(LogCategories.MODULE.APP.EVALUATION);
......@@ -18,7 +19,7 @@ export const evaluationQueue = getQueue<EvaluationJobData>(QueueNames.evaluation
}
});
const concurrency = process.env.EVAL_CONCURRENCY ? Number(process.env.EVAL_CONCURRENCY) : 3;
const concurrency = serviceEnv.EVAL_CONCURRENCY;
export const getEvaluationWorker = (processor: Processor<EvaluationJobData>) => {
return getWorker<EvaluationJobData>(QueueNames.evaluation, processor, {
removeOnFail: {
......
packages/service/core/chat/pushChatLog.ts
......@@ -3,6 +3,7 @@ import { MongoChat } from './chatSchema';
import { axios } from '../../common/api/axios';
import { type AIChatItemType, type UserChatItemType } from '@fastgpt/global/core/chat/type';
import { getLogger, LogCategories } from '../../common/logger';
import { serviceEnv } from '../../env';
const logger = getLogger(LogCategories.MODULE.CHAT.RECORD);
......@@ -26,9 +27,9 @@ export const pushChatLog = ({
appId: string;
metadata?: Metadata;
}) => {
const interval = Number(process.env.CHAT_LOG_INTERVAL);
const url = process.env.CHAT_LOG_URL;
if (!isNaN(interval) && interval > 0 && url) {
const interval = serviceEnv.CHAT_LOG_INTERVAL;
const url = serviceEnv.CHAT_LOG_URL;
if (interval && interval > 0 && url) {
logger.debug('Chat log push scheduled', {
intervalMs: interval,
appId,
......@@ -151,7 +152,7 @@ ${JSON.stringify(item.interactive, null, 2)}
const responseTime =
responseData?.reduce((acc, item) => acc + (item?.runningTime ?? 0), 0) || 0;
const sourceIdPrefix = process.env.CHAT_LOG_SOURCE_ID_PREFIX ?? 'fastgpt-';
const sourceIdPrefix = serviceEnv.CHAT_LOG_SOURCE_ID_PREFIX;
const chatLog: ChatLog = {
title: chat.title,
......
packages/service/core/chat/resume.ts
import { env } from '../../env';
import { serviceEnv } from '../../env';
import { getLogger, LogCategories } from '../../common/logger';
import { FASTGPT_REDIS_PREFIX, getGlobalRedisConnection } from '../../common/redis';
import type { NextApiResponse } from 'next';
......@@ -7,14 +7,15 @@ import { StreamResumeUnavailableReasonEnum } from '@fastgpt/global/core/workflow
const logger = getLogger(LogCategories.MODULE.CHAT.RESUME);
/** 生成中：定期续期（见 env `STREAM_RESUME_TTL_SECONDS`） */
export const STREAM_RESUME_TTL_SECONDS = env.STREAM_RESUME_TTL_SECONDS;
export const STREAM_RESUME_TTL_SECONDS = serviceEnv.STREAM_RESUME_TTL_SECONDS;
/** 流结束后短 TTL（见 env `STREAM_RESUME_POST_COMPLETE_TTL_SECONDS`） */
export const STREAM_RESUME_POST_COMPLETE_TTL_SECONDS = env.STREAM_RESUME_POST_COMPLETE_TTL_SECONDS;
export const STREAM_RESUME_POST_COMPLETE_TTL_SECONDS =
serviceEnv.STREAM_RESUME_POST_COMPLETE_TTL_SECONDS;
/** 当 Redis 已用内存 / maxmemory 达到该阈值时，停止为新请求创建镜像 */
export const STREAM_RESUME_REDIS_MAXMEMORY_RATIO = env.STREAM_RESUME_REDIS_MAXMEMORY_RATIO;
export const STREAM_RESUME_REDIS_MAXMEMORY_RATIO = serviceEnv.STREAM_RESUME_REDIS_MAXMEMORY_RATIO;
/** Redis 内存检测缓存时间，避免每个流请求都去调用 INFO MEMORY */
export const STREAM_RESUME_REDIS_MEMORY_CHECK_INTERVAL_MS =
env.STREAM_RESUME_REDIS_MEMORY_CHECK_INTERVAL_MS;
serviceEnv.STREAM_RESUME_REDIS_MEMORY_CHECK_INTERVAL_MS;
/**
* One active resume request keeps one dedicated blocking Redis connection alive for at most this
* long before the XREAD call returns and the loop re-checks the HTTP socket state.
......
packages/service/core/dataset/apiDataset/dingtalkDataset/api.ts
......@@ -10,6 +10,7 @@ import type { Method } from 'axios';
import { axios, createProxyAxios } from '../../../../common/api/axios';
import { delRedisCache, getRedisCache, setRedisCache } from '../../../../common/redis/cache';
import { getLogger, LogCategories } from '../../../../common/logger';
import { serviceEnv } from '../../../../env';
type DingtalkAccessTokenResponse = {
accessToken: string;
......@@ -86,8 +87,8 @@ type ListAllByNextTokenProps<T> = {
maxResults?: number;
};
const dingtalkBaseUrl = process.env.DINGTALK_BASE_URL || 'https://api.dingtalk.com';
const dingtalkOapiBaseUrl = process.env.DINGTALK_OAPI_BASE_URL || 'https://oapi.dingtalk.com';
const dingtalkBaseUrl = serviceEnv.DINGTALK_BASE_URL;
const dingtalkOapiBaseUrl = serviceEnv.DINGTALK_OAPI_BASE_URL;
const tokenSafeWindowSeconds = 5 * 60;
const dingtalkListPageSize = 100;
const refreshingTokenMap = new Map<string, Promise<string>>();
......
packages/service/core/dataset/apiDataset/feishuDataset/api.ts
......@@ -8,6 +8,7 @@ import { type ParentIdType } from '@fastgpt/global/common/parentFolder/type';
import { type Method } from 'axios';
import { createProxyAxios, axios } from '../../../../common/api/axios';
import { getLogger, LogCategories } from '../../../../common/logger';
import { serviceEnv } from '../../../../env';
type ResponseDataType = {
success: boolean;
......@@ -30,7 +31,7 @@ type FeishuFileListResponse = {
next_page_token: string;
};
const feishuBaseUrl = process.env.FEISHU_BASE_URL || 'https://open.feishu.cn';
const feishuBaseUrl = serviceEnv.FEISHU_BASE_URL;
const logger = getLogger(LogCategories.MODULE.DATASET.API_DATASET);
export const useFeishuDatasetRequest = ({ feishuServer }: { feishuServer: FeishuServerType }) => {
......
packages/service/core/dataset/apiDataset/yuqueDataset/api.ts
......@@ -8,6 +8,7 @@ import { type Method } from 'axios';
import { type ParentIdType } from '@fastgpt/global/common/parentFolder/type';
import { createProxyAxios } from '../../../../common/api/axios';
import { getLogger, LogCategories } from '../../../../common/logger';
import { serviceEnv } from '../../../../env';
type ResponseDataType = {
success: boolean;
......@@ -40,7 +41,7 @@ type YuqueTocListResponse = {
parent_uuid: string;
}[];
const yuqueBaseUrl = process.env.YUQUE_DATASET_BASE_URL || 'https://www.yuque.com';
const yuqueBaseUrl = serviceEnv.YUQUE_DATASET_BASE_URL;
export const useYuqueDatasetRequest = ({ yuqueServer }: { yuqueServer: YuqueServerType }) => {
const logger = getLogger(LogCategories.MODULE.DATASET.API_DATASET);
......
packages/service/core/dataset/read.ts
......@@ -14,7 +14,8 @@ import { text2Chunks } from '../../worker/function';
import { retryFn } from '@fastgpt/global/common/system/utils';
import { getFileMaxSize } from '../../common/file/utils';
import { UserError } from '@fastgpt/global/common/error/utils';
import { getS3DatasetSource, S3DatasetSource } from '../../common/s3/sources/dataset';
import { getAxiosHeaderValue } from '@fastgpt/global/common/axios/utils';
import { getS3DatasetSource } from '../../common/s3/sources/dataset';
import { getFileS3Key, isS3ObjectKey } from '../../common/s3/utils';
import { getLogger, LogCategories } from '../../common/logger';
......@@ -44,7 +45,9 @@ export const readFileRawTextByUrl = async ({
// Check file size
try {
const headResponse = await axios.head(url, { timeout: 10000 });
const contentLength = parseInt(headResponse.headers['content-length'] || '0');
const contentLength = parseInt(
getAxiosHeaderValue(headResponse.headers['content-length']) || '0'
);
if (contentLength > 0 && contentLength > maxFileSize) {
return Promise.reject(
......
packages/service/core/workflow/constants.ts
export const WORKFLOW_MAX_RUN_TIMES = process.env.WORKFLOW_MAX_RUN_TIMES
? parseInt(process.env.WORKFLOW_MAX_RUN_TIMES)
: 500;
import { serviceEnv } from '../../env';
export const WORKFLOW_MAX_RUN_TIMES = serviceEnv.WORKFLOW_MAX_RUN_TIMES;
packages/service/core/workflow/dispatch/abandoned/runLoop.ts
......@@ -14,7 +14,7 @@ import {
import { cloneDeep } from 'lodash';
import { type WorkflowInteractiveResponseType } from '@fastgpt/global/core/workflow/template/system/interactive/type';
import { storeEdges2RuntimeEdges } from '@fastgpt/global/core/workflow/runtime/utils';
import { env } from '../../../../env';
import { serviceEnv } from '../../../../env';
import { getNestedEndOutputValue } from '../loop/service';
import { collectResponseFeedbacks, injectNestedStartInputs, pushSubWorkflowUsage } from '../utils';
......@@ -42,7 +42,7 @@ export const dispatchLoop = async (props: Props): Promise<Response> => {
}
// Max loop times
const maxLength = env.WORKFLOW_MAX_LOOP_TIMES;
const maxLength = serviceEnv.WORKFLOW_MAX_LOOP_TIMES;
if (loopInputArray.length > maxLength) {
return Promise.reject(`Input array length cannot be greater than ${maxLength}`);
}
......
packages/service/core/workflow/dispatch/ai/agent/index.ts
......@@ -44,7 +44,7 @@ import { getContinuePlanQuery, parseUserSystemPrompt } from './sub/plan/prompt';
import type { PlanAgentParamsType } from './sub/plan/constants';
import type { AppFormEditFormType } from '@fastgpt/global/core/app/formEdit/type';
import { getLogger, LogCategories } from '../../../../../common/logger';
import { env } from '../../../../../env';
import { serviceEnv } from '../../../../../env';
import { dispatchPiAgent } from './piAgent';
import { i18nT } from '../../../../../../web/i18n/utils';
......@@ -80,7 +80,7 @@ type Response = DispatchNodeResultType<{
export const dispatchRunAgent = async (props: DispatchAgentModuleProps): Promise<Response> => {
// pi-agent-core engine: bypass Plan+Step orchestration
if (env.AGENT_ENGINE === 'pi') {
if (serviceEnv.AGENT_ENGINE === 'pi') {
return dispatchPiAgent(props);
}
......@@ -218,7 +218,7 @@ export const dispatchRunAgent = async (props: DispatchAgentModuleProps): Promise
// Initialize capabilities — always create sandbox capability (lazy-init, no container yet)
// Skill capability is gated by SHOW_SKILL env: when disabled, we skip skill loading entirely
// (no MongoDB query, no sandbox init), even if existing apps still have skills configured.
if (env.SHOW_SKILL) {
if (serviceEnv.SHOW_SKILL) {
const sandboxSessionId = mode === 'chat' ? chatId : `debug-${runningAppInfo.id}-${nodeId}`;
const useEditDebugSandbox_flag = !!useEditDebugSandbox;
const sandboxMode = useEditDebugSandbox_flag ? 'editDebug' : 'sessionRuntime';
......
packages/service/core/workflow/dispatch/ai/agent/piAgent/index.ts
......@@ -23,7 +23,7 @@ import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/util
import { buildPiModel, getModelApiKey } from './modelBridge';
import { buildAgentTools } from './toolAdapter';
import { getLogger, LogCategories } from '../../../../../../common/logger';
import { env } from '../../../../../../env';
import { serviceEnv } from '../../../../../../env';
import type { DispatchAgentModuleProps } from '..';
type Response = DispatchNodeResultType<{
......@@ -92,7 +92,7 @@ export const dispatchPiAgent = async (props: DispatchAgentModuleProps): Promise<
: userChatInput;
// Initialize capabilities — sandbox skills (lazy-init, gated by SHOW_SKILL)
if (env.SHOW_SKILL) {
if (serviceEnv.SHOW_SKILL) {
const sandboxSessionId = mode === 'chat' ? chatId : `debug-${runningAppInfo.id}-${nodeId}`;
const sandboxMode = useEditDebugSandbox ? 'editDebug' : 'sessionRuntime';
......
packages/service/core/workflow/dispatch/ai/agent/piAgent/modelBridge.ts
import { getLLMModel } from '../../../../../ai/model';
import { openaiBaseUrl, openaiBaseKey } from '../../../../../ai/config';
type Model = import('@mariozechner/pi-ai').Model<'openai-completions'>;
const aiProxyBaseUrl = process.env.AIPROXY_API_ENDPOINT
? `${process.env.AIPROXY_API_ENDPOINT}/v1`
: undefined;
const defaultBaseUrl = aiProxyBaseUrl || process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';
const defaultApiKey = process.env.AIPROXY_API_TOKEN || process.env.CHAT_API_KEY || '';
export function buildPiModel(modelNameOrId?: string, useVision?: boolean): Model {
const cfg = getLLMModel(modelNameOrId);
// requestUrl is the full endpoint (e.g. https://api.deepseek.com/chat/completions).
// pi-ai's openai-completions provider appends /chat/completions automatically,
// so we strip it to get baseUrl.
const rawUrl = cfg?.requestUrl ?? '';
const baseUrl = rawUrl ? rawUrl.replace(/\/chat\/completions$/, '') : defaultBaseUrl;
const apiKey = cfg?.requestAuth || defaultApiKey;
const baseUrl = rawUrl ? rawUrl.replace(/\/chat\/completions$/, '') : openaiBaseUrl;
const apiKey = cfg?.requestAuth || openaiBaseKey;
return {
id: cfg?.model ?? 'gpt-4o',
......@@ -42,5 +37,5 @@ export function buildPiModel(modelNameOrId?: string, useVision?: boolean): Model
export function getModelApiKey(modelNameOrId?: string): string {
const cfg = getLLMModel(modelNameOrId);
return cfg?.requestAuth || defaultApiKey;
return cfg?.requestAuth || openaiBaseKey || '';
}
packages/service/core/workflow/dispatch/ai/agent/sub/file/index.ts
......@@ -16,6 +16,7 @@ import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import { i18nT } from '../../../../../../../../web/i18n/utils';
import { getLogger, LogCategories } from '../../../../../../../common/logger';
import type { OpenaiAccountType } from '@fastgpt/global/support/user/team/type';
import { getAxiosHeaderValue } from '@fastgpt/global/common/axios/utils';
import type { DispatchSubAppResponse } from '../../type';
type FileReadParams = {
......@@ -69,7 +70,7 @@ export const dispatchFileRead = async ({
// Get file name
const filename = (() => {
const contentDisposition = response.headers['content-disposition'];
const contentDisposition = getAxiosHeaderValue(response.headers['content-disposition']);
return parseContentDispositionFilename(contentDisposition) || url;
})();
// Extension
......@@ -77,7 +78,7 @@ export const dispatchFileRead = async ({
// Get encoding
const encoding = (() => {
const contentType = response.headers['content-type'];
const contentType = getAxiosHeaderValue(response.headers['content-type']);
if (contentType) {
const charsetRegex = /charset=([^;]*)/;
const matches = charsetRegex.exec(contentType);
......
packages/service/core/workflow/dispatch/ai/agent/sub/sandbox/lifecycle.ts
......@@ -25,7 +25,7 @@ import {
import { SandboxTypeEnum } from '@fastgpt/global/core/agentSkills/constants';
import { SandboxStatusEnum } from '@fastgpt/global/core/ai/sandbox/constants';
import { getSandboxClient, type SandboxClient } from '../../../../../../ai/sandbox/controller';
import { env } from '../../../../../../../env';
import { serviceEnv } from '../../../../../../../env';
import type {
AgentSkillSchemaType,
AgentSkillsVersionSchemaType,
......@@ -256,7 +256,8 @@ export async function createAgentSandbox(
// Check active session-runtime sandbox count limit
const maxSessionRuntime =
global.feConfigs?.limit?.agentSandboxMaxSessionRuntime ?? env.AGENT_SANDBOX_MAX_SESSION_RUNTIME;
global.feConfigs?.limit?.agentSandboxMaxSessionRuntime ??
serviceEnv.AGENT_SANDBOX_MAX_SESSION_RUNTIME;
if (maxSessionRuntime !== undefined) {
const activeCount = await MongoSandboxInstance.countDocuments({
status: SandboxStatusEnum.running,
......
packages/service/core/workflow/dispatch/loopRun/runLoopRun.ts
......@@ -18,7 +18,7 @@ import {
} from '@fastgpt/global/core/workflow/runtime/utils';
import { LoopRunModeEnum } from '@fastgpt/global/core/workflow/template/system/loopRun/loopRun';
import { env } from '../../../../env';
import { serviceEnv } from '../../../../env';
import { i18nT } from '../../../../../web/i18n/utils';
import { runWorkflow } from '..';
import { collectResponseFeedbacks, getNodeErrResponse, pushSubWorkflowUsage } from '../utils';
......@@ -47,7 +47,7 @@ export const dispatchLoopRun = async (props: Props): Promise<Response> => {
const childrenNodeIdList = params[NodeInputKeyEnum.childrenNodeIdList] ?? [];
const inputArray = params[NodeInputKeyEnum.loopRunInputArray] ?? [];
const maxLength = env.WORKFLOW_MAX_LOOP_TIMES;
const maxLength = serviceEnv.WORKFLOW_MAX_LOOP_TIMES;
const maxIterationsMessage = i18nT('workflow:loop_run_max_iterations_exceeded');
// Surface precheck failures through `errorText` to match the max-iterations
......
packages/service/core/workflow/dispatch/parallelRun/runParallelRun.ts
......@@ -7,7 +7,7 @@ import {
type ModuleDispatchProps
} from '@fastgpt/global/core/workflow/runtime/type';
import { env } from '../../../../env';
import { serviceEnv } from '../../../../env';
import { runWorkflow } from '..';
import { cloneDeep } from 'lodash';
import {
......@@ -49,14 +49,14 @@ export const dispatchParallelRun = async (props: Props): Promise<Response> => {
return Promise.reject('Input value is not an array');
}
const maxLength = env.WORKFLOW_MAX_LOOP_TIMES;
const maxLength = serviceEnv.WORKFLOW_MAX_LOOP_TIMES;
if (loopInputArray.length > maxLength) {
return Promise.reject(`Input array length cannot be greater than ${maxLength}`);
}
const concurrency = clampParallelConcurrency(
userConcurrency,
env.WORKFLOW_PARALLEL_MAX_CONCURRENCY
serviceEnv.WORKFLOW_PARALLEL_MAX_CONCURRENCY
);
const maxRetryAttempts = clampParallelRetryTimes(userRetryTimes);
......
packages/service/core/workflow/dispatch/tools/codeSandbox.ts
......@@ -4,6 +4,7 @@ import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runti
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { codeSandbox } from '../../../../thirdProvider/codeSandbox';
import { serviceEnv } from '../../../../env';
type RunCodeType = ModuleDispatchProps<{
[NodeInputKeyEnum.codeType]: string;
......@@ -29,7 +30,7 @@ export const dispatchCodeSandbox = async (props: RunCodeType): Promise<RunCodeRe
params: { codeType, code, [NodeInputKeyEnum.addInputParam]: customVariables }
} = props;
if (!process.env.CODE_SANDBOX_URL) {
if (!serviceEnv.CODE_SANDBOX_URL) {
return {
error: {
[NodeOutputKeyEnum.error]: 'Can not find CODE_SANDBOX_URL in env'
......
packages/service/core/workflow/utils/file.ts
......@@ -18,6 +18,7 @@ import { addDays } from 'date-fns';
import { replaceS3KeyToPreviewUrl } from '../../dataset/utils';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { getUserFilesPrompt, injectUserQueryPrompt } from '../../ai/llm/agentLoop/prompt';
import { getAxiosHeaderValue } from '@fastgpt/global/common/axios/utils';
type GetFileProps = {
requestOrigin?: string;
......@@ -121,7 +122,7 @@ export const getFileInfoFromUrl = async ({ teamId, url }: { teamId: string; url:
// Get file name
const { filename, extension, imageParsePrefix } = (() => {
if (isChatExternalUrl) {
const contentDisposition = response.headers['content-disposition'] || '';
const contentDisposition = getAxiosHeaderValue(response.headers['content-disposition']) || '';
const matchFilename = parseContentDispositionFilename(contentDisposition);
const filename = matchFilename || urlObj.pathname.split('/').pop() || 'file';
const extension = path.extname(filename).replace('.', '');
......@@ -141,7 +142,7 @@ export const getFileInfoFromUrl = async ({ teamId, url }: { teamId: string; url:
filename,
extension,
imageParsePrefix,
contentType: response.headers['content-type'],
contentType: getAxiosHeaderValue(response.headers['content-type']),
stream: response.data
};
};
......
packages/service/package.json
......@@ -24,7 +24,7 @@
"@modelcontextprotocol/sdk": "catalog:",
"@node-rs/jieba": "catalog:",
"@opentelemetry/api": "^1.9.0",
"@t3-oss/env-core": "0.13.10",
"@t3-oss/env-core": "catalog:",
"@xmldom/xmldom": "^0.8.10",
"@zilliz/milvus2-sdk-node": "2.4.10",
"axios": "catalog:",
......@@ -43,7 +43,6 @@
"https-proxy-agent": "^7.0.6",
"iconv-lite": "^0.6.3",
"ioredis": "^5.6.0",
"ipaddr.js": "^2.3.0",
"joplin-turndown-plugin-gfm": "^1.0.12",
"json5": "catalog:",
"jsonpath-plus": "^10.3.0",
......
packages/service/support/outLink/wechat/mq.ts
......@@ -7,7 +7,7 @@ import { MongoOutLink } from '../../../support/outLink/schema';
import { outlinkInvokeChat } from '../../../support/outLink/runtime/utils';
import { delRedisCache, getRedisCache, setRedisCache } from '../../../common/redis/cache';
import { groupMessagesByUser } from './messageParser';
import { env } from '../../../env';
import { serviceEnv } from '../../../env';
import { batchRun, retryFn } from '@fastgpt/global/common/system/utils';
const logger = getLogger(LogCategories.MODULE.OUTLINK.WECHAT);
......@@ -231,7 +231,7 @@ async function shouldContinuePolling(shareId: string): Promise<boolean> {
export const initWechatPollWorker = async () => {
const pollWorker = getWorker<WechatPollJobData>(QueueNames.wechatPoll, processWechatPollJob, {
// poll job 主要阻塞在 getUpdates 长轮询 I/O（~30s），不吃 CPU
concurrency: env.WECHAT_CHANNEL_CONCURRENCY,
concurrency: serviceEnv.WECHAT_CHANNEL_CONCURRENCY,
lockDuration: POLL_LOCK_MS, // 120s 防止 job 被误判为 stalled
stalledInterval: 30_000, // 30s 检查下是否活跃
removeOnComplete: { count: 0 },
......@@ -264,7 +264,7 @@ export const initWechatPollWorker = async () => {
});
getWorker<WechatReplyJobData>(QueueNames.wechatReply, processWechatReplyJob, {
concurrency: env.WECHAT_CHANNEL_CONCURRENCY,
concurrency: serviceEnv.WECHAT_CHANNEL_CONCURRENCY,
lockDuration: REPLY_LOCK_MS,
stalledInterval: 60_000,
removeOnComplete: { count: 0 },
......
packages/service/support/permission/auth/common.ts
......@@ -8,6 +8,7 @@ import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import { authUserSession } from '../../../support/user/session';
import { authOpenApiKey } from '../../../support/openapi/auth';
import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
import { serviceEnv } from '../../../env';
export const authCert = async (props: AuthModeType) => {
const result = await parseHeaderCert(props);
......@@ -87,7 +88,7 @@ export async function parseHeaderCert({
}
// root user
async function parseRootKey(rootKey?: string) {
if (!rootKey || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
if (!rootKey || rootKey !== serviceEnv.ROOT_KEY) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
}
......
packages/service/support/permission/auth/file.ts
......@@ -9,7 +9,7 @@ import jwt from 'jsonwebtoken';
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import { getS3DatasetSource } from '../../../common/s3/sources/dataset';
import { isS3ObjectKey } from '../../../common/s3/utils';
import { env } from '../../../env';
import { serviceEnv } from '../../../env';
export const authCollectionFile = async ({
fileId,
......@@ -44,7 +44,7 @@ export const authFileToken = (token?: string) =>
if (!token) {
return reject(ERROR_ENUM.unAuthFile);
}
jwt.verify(token, env.FILE_TOKEN_KEY, (err, decoded: any) => {
jwt.verify(token, serviceEnv.FILE_TOKEN_KEY, (err, decoded: any) => {
if (err || !decoded.bucketName || !decoded?.teamId || !decoded?.fileId) {
reject(ERROR_ENUM.unAuthFile);
return;
......
packages/service/support/permission/auth/pluginAccessToken.ts
......@@ -2,12 +2,10 @@ import jwt from 'jsonwebtoken';
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import z from 'zod';
import type { NextApiRequest } from 'next';
import { serviceEnv } from '../../../env';
const PLUGIN_ACCESS_TOKEN_SECRET =
process.env.PLUGIN_ACCESS_TOKEN_SECRET || 'plugin_access_token_secret';
const PLUGIN_ACCESS_TOKEN_EXPIRES_IN: number = process.env.PLUGIN_ACCESS_TOKEN_EXPIRES_IN
? parseInt(process.env.PLUGIN_ACCESS_TOKEN_EXPIRES_IN)
: 3600; // Default 1 hour (3600 seconds)
const PLUGIN_ACCESS_TOKEN_SECRET = serviceEnv.PLUGIN_ACCESS_TOKEN_SECRET;
const PLUGIN_ACCESS_TOKEN_EXPIRES_IN = serviceEnv.PLUGIN_ACCESS_TOKEN_EXPIRES_IN;
export const PluginAccessTokenPayloadSchema = z.object({
tmbId: z.string(),
......
packages/service/support/permission/teamLimit.ts
......@@ -8,7 +8,7 @@ import { AppTypeEnum, ToolTypeList, AppFolderTypeList } from '@fastgpt/global/co
import { MongoTeamMember } from '../user/team/teamMemberSchema';
import { TeamMemberStatusEnum } from '@fastgpt/global/support/user/team/constant';
import { getVectorCountByTeamId } from '../../common/vectorDB/controller';
import { env } from '../../env';
import { serviceEnv } from '../../env';
export const checkTeamAIPoints = async (teamId: string) => {
if (!global.subPlans?.standard) return;
......@@ -52,7 +52,7 @@ export const checkTeamDatasetFolderLimit = async ({
teamId,
type: DatasetTypeEnum.folder
});
if (folderCount + amount > env.DATASET_FOLDER_MAX_AMOUNT) {
if (folderCount + amount > serviceEnv.DATASET_FOLDER_MAX_AMOUNT) {
return Promise.reject(TeamErrEnum.datasetFolderAmountNotEnough);
}
};
......@@ -110,7 +110,7 @@ export const checkTeamAppTypeLimit = async ({
$in: AppFolderTypeList
}
});
const maxAppFolderAmount = env.APP_FOLDER_MAX_AMOUNT;
const maxAppFolderAmount = serviceEnv.APP_FOLDER_MAX_AMOUNT;
if (folderCount + amount > maxAppFolderAmount) {
return Promise.reject(TeamErrEnum.appFolderAmountNotEnough);
}
......
packages/service/support/user/session.ts
......@@ -3,6 +3,7 @@ import { getAllKeysByPrefix, getGlobalRedisConnection } from '../../common/redis
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { getLogger, LogCategories } from '../../common/logger';
import { serviceEnv } from '../../env';
const logger = getLogger(LogCategories.MODULE.USER.ACCOUNT);
......@@ -100,7 +101,7 @@ export const delUserAllSession = async (userId: string, whiteList?: (string | un
// 会根据创建时间，删除超出客户端登录限制的 session
const delRedundantSession = async (userId: string) => {
// 至少为 1，默认为 10
let maxSession = process.env.MAX_LOGIN_SESSION ? Number(process.env.MAX_LOGIN_SESSION) : 10;
let maxSession = serviceEnv.MAX_LOGIN_SESSION;
if (maxSession < 1) {
maxSession = 1;
}
......
packages/service/support/wallet/sub/utils.ts
......@@ -24,6 +24,7 @@ import {
incrValueToCache
} from '../../../common/redis/cache';
import { getLogger, LogCategories } from '../../../common/logger';
import { serviceEnv } from '../../../env';
const logger = getLogger(LogCategories.MODULE.WALLET.SUB);
......@@ -328,7 +329,7 @@ export const teamPoint = {
}
};
export const teamQPM = {
getTeamQPMLimit: async (teamId: string): Promise<number | null> => {
getTeamQPMLimit: async (teamId: string): Promise<number | undefined> => {
// 1. 尝试从缓存中获取
const cacheKey = `${CacheKeyEnum.team_qpm_limit}:${teamId}`;
const cached = await getRedisCache(cacheKey);
......@@ -342,8 +343,7 @@ export const teamQPM = {
const limit = teamPlanStatus[SubTypeEnum.standard]?.requestsPerMinute;
if (!limit) {
if (process.env.CHAT_MAX_QPM) return Number(process.env.CHAT_MAX_QPM);
return null;
return serviceEnv.CHAT_MAX_QPM;
}
// 3. Set cache
......
packages/service/test/common/cache/index.test.ts
import { describe, it, expect, beforeEach, vi } from 'vitest';
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { SystemCacheKeyEnum } from '@fastgpt/service/common/cache/type';
import { getGlobalRedisConnection } from '@fastgpt/service/common/redis';
import { serviceEnv } from '@fastgpt/service/env';
vi.mock('@fastgpt/service/core/app/tool/controller', () => ({
refreshSystemTools: vi.fn().mockResolvedValue([])
......@@ -18,6 +19,8 @@ vi.mock('@fastgpt/service/common/redis', async (importOriginal) => {
import { refreshVersionKey, getVersionKey, getCachedData } from '@fastgpt/service/common/cache';
import { initCache } from '@fastgpt/service/common/cache/init';
const originalDisableCache = serviceEnv.DISABLE_CACHE;
describe('refreshVersionKey', () => {
beforeEach(() => {
delete (global as any).systemCache;
......@@ -135,7 +138,11 @@ describe('getCachedData', () => {
const redis = getGlobalRedisConnection() as any;
redis._storage.clear();
mockRefreshFunc.mockReset();
delete process.env.DISABLE_CACHE;
serviceEnv.DISABLE_CACHE = false;
});
afterEach(() => {
serviceEnv.DISABLE_CACHE = originalDisableCache;
});
it('should init systemCache if not present', async () => {
......@@ -177,7 +184,7 @@ describe('getCachedData', () => {
});
it('should refresh when DISABLE_CACHE is true', async () => {
process.env.DISABLE_CACHE = 'true';
serviceEnv.DISABLE_CACHE = true;
const mockData = [{ id: 'tool1' }];
mockRefreshFunc.mockResolvedValue(mockData);
......
packages/service/test/common/file/image/controller.test.ts
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
vi.hoisted(() => {
vi.stubEnv('NEXT_PUBLIC_BASE_URL', '');
});
import { Types } from '@fastgpt/service/common/mongo';
import {
uploadMongoImg,
......@@ -13,9 +16,19 @@ import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants';
const teamId = new Types.ObjectId().toString();
const loadController = async () => {
vi.resetModules();
return import('@fastgpt/service/common/file/image/controller');
};
describe('uploadMongoImg', () => {
beforeEach(async () => {
await MongoImage.deleteMany({});
vi.stubEnv('NEXT_PUBLIC_BASE_URL', '');
});
afterEach(() => {
vi.stubEnv('NEXT_PUBLIC_BASE_URL', '');
});
it('should upload a valid JPEG base64 image', async () => {
......@@ -129,21 +142,19 @@ describe('uploadMongoImg', () => {
});
it('should include NEXT_PUBLIC_BASE_URL in result when set', async () => {
const originalBaseUrl = process.env.NEXT_PUBLIC_BASE_URL;
process.env.NEXT_PUBLIC_BASE_URL = '/sub';
vi.stubEnv('NEXT_PUBLIC_BASE_URL', '/sub');
const { uploadMongoImg: uploadMongoImgWithBase } = await loadController();
const binary = Buffer.from([0xff, 0xd8, 0xff, 0xe0]);
const base64Data = binary.toString('base64');
const base64Img = `data:image/jpeg;base64,${base64Data}`;
const result = await uploadMongoImg({
const result = await uploadMongoImgWithBase({
base64Img,
teamId
});
expect(result).toContain(imageBaseUrl);
process.env.NEXT_PUBLIC_BASE_URL = originalBaseUrl;
expect(result).toMatch(new RegExp(`^/sub${imageBaseUrl}[a-f0-9]{24}\\.jpeg$`));
});
});
......
packages/service/test/common/file/image/utils.test.ts
......@@ -3,8 +3,7 @@ import {
isValidImageContentType,
detectImageTypeFromBuffer,
guessBase64ImageType,
getImageBase64,
addEndpointToImageUrl
getImageBase64
} from '@fastgpt/service/common/file/image/utils';
const mockAxiosGet = vi.fn();
......@@ -19,6 +18,11 @@ vi.mock('@fastgpt/service/common/api/serverRequest', () => ({
serverRequestBaseUrl: 'http://localhost:3000'
}));
const loadUtilsModule = async () => {
vi.resetModules();
return import('@fastgpt/service/common/file/image/utils');
};
describe('isValidImageContentType', () => {
it('should return true for valid image MIME types', () => {
expect(isValidImageContentType('image/jpeg')).toBe(true);
......@@ -52,8 +56,8 @@ describe('isValidImageContentType', () => {
});
it('should be case-sensitive (requires lowercase input)', () => {
// Note: This function expects lowercase input
// The getContentTypeFromHeader function should normalize it first
// Note: This function expects lowercase input.
// Header normalization is handled by getAxiosContentType before validation.
expect(isValidImageContentType('IMAGE/JPEG')).toBe(false);
expect(isValidImageContentType('Image/Png')).toBe(false);
});
......@@ -336,27 +340,30 @@ describe('getImageBase64', () => {
});
describe('addEndpointToImageUrl', () => {
const originalEnv = { ...process.env };
beforeEach(() => {
delete process.env.FE_DOMAIN;
delete process.env.NEXT_PUBLIC_BASE_URL;
vi.stubEnv('FE_DOMAIN', undefined);
vi.stubEnv('NEXT_PUBLIC_BASE_URL', undefined);
});
afterEach(() => {
process.env.FE_DOMAIN = originalEnv.FE_DOMAIN;
process.env.NEXT_PUBLIC_BASE_URL = originalEnv.NEXT_PUBLIC_BASE_URL;
vi.stubEnv('FE_DOMAIN', undefined);
vi.stubEnv('NEXT_PUBLIC_BASE_URL', undefined);
});
it('should return text unchanged when FE_DOMAIN is not set', () => {
delete process.env.FE_DOMAIN;
const loadAddEndpointToImageUrl = async () => {
const { addEndpointToImageUrl } = await loadUtilsModule();
return addEndpointToImageUrl;
};
it('should return text unchanged when FE_DOMAIN is not set', async () => {
const addEndpointToImageUrl = await loadAddEndpointToImageUrl();
const text = '/api/system/img/abc123.png';
expect(addEndpointToImageUrl(text)).toBe(text);
});
it('should prepend FE_DOMAIN to matching image URLs without subRoute', () => {
process.env.FE_DOMAIN = 'https://example.com';
process.env.NEXT_PUBLIC_BASE_URL = '';
it('should prepend FE_DOMAIN to matching image URLs without subRoute', async () => {
vi.stubEnv('FE_DOMAIN', 'https://example.com');
const addEndpointToImageUrl = await loadAddEndpointToImageUrl();
const text = 'Here is an image: /api/system/img/abc123.png in the text';
const result = addEndpointToImageUrl(text);
......@@ -366,9 +373,10 @@ describe('addEndpointToImageUrl', () => {
);
});
it('should prepend FE_DOMAIN to matching image URLs with subRoute', () => {
process.env.FE_DOMAIN = 'https://example.com';
process.env.NEXT_PUBLIC_BASE_URL = '/fastgpt';
it('should prepend FE_DOMAIN to matching image URLs with subRoute', async () => {
vi.stubEnv('FE_DOMAIN', 'https://example.com');
vi.stubEnv('NEXT_PUBLIC_BASE_URL', '/fastgpt');
const addEndpointToImageUrl = await loadAddEndpointToImageUrl();
const text = 'Image: /fastgpt/api/system/img/abc123.png end';
const result = addEndpointToImageUrl(text);
......@@ -376,9 +384,9 @@ describe('addEndpointToImageUrl', () => {
expect(result).toBe('Image: https://example.com/fastgpt/api/system/img/abc123.png end');
});
it('should not modify URLs that already have a full http(s) prefix', () => {
process.env.FE_DOMAIN = 'https://example.com';
process.env.NEXT_PUBLIC_BASE_URL = '';
it('should not modify URLs that already have a full http(s) prefix', async () => {
vi.stubEnv('FE_DOMAIN', 'https://example.com');
const addEndpointToImageUrl = await loadAddEndpointToImageUrl();
const text = 'Already full: https://cdn.example.com/api/system/img/abc123.png';
const result = addEndpointToImageUrl(text);
......@@ -386,9 +394,9 @@ describe('addEndpointToImageUrl', () => {
expect(result).toBe(text);
});
it('should handle multiple image URLs in the same text', () => {
process.env.FE_DOMAIN = 'https://example.com';
process.env.NEXT_PUBLIC_BASE_URL = '';
it('should handle multiple image URLs in the same text', async () => {
vi.stubEnv('FE_DOMAIN', 'https://example.com');
const addEndpointToImageUrl = await loadAddEndpointToImageUrl();
const text = 'First /api/system/img/img1.png and second /api/system/img/img2.jpg here';
const result = addEndpointToImageUrl(text);
......@@ -398,9 +406,9 @@ describe('addEndpointToImageUrl', () => {
);
});
it('should not modify non-matching paths', () => {
process.env.FE_DOMAIN = 'https://example.com';
process.env.NEXT_PUBLIC_BASE_URL = '';
it('should not modify non-matching paths', async () => {
vi.stubEnv('FE_DOMAIN', 'https://example.com');
const addEndpointToImageUrl = await loadAddEndpointToImageUrl();
const text = 'This is /api/other/endpoint and /some/path.png';
const result = addEndpointToImageUrl(text);
......@@ -408,22 +416,23 @@ describe('addEndpointToImageUrl', () => {
expect(result).toBe(text);
});
it('should return empty string unchanged', () => {
process.env.FE_DOMAIN = 'https://example.com';
it('should return empty string unchanged', async () => {
vi.stubEnv('FE_DOMAIN', 'https://example.com');
const addEndpointToImageUrl = await loadAddEndpointToImageUrl();
expect(addEndpointToImageUrl('')).toBe('');
});
it('should handle text with no image URLs', () => {
process.env.FE_DOMAIN = 'https://example.com';
process.env.NEXT_PUBLIC_BASE_URL = '';
it('should handle text with no image URLs', async () => {
vi.stubEnv('FE_DOMAIN', 'https://example.com');
const addEndpointToImageUrl = await loadAddEndpointToImageUrl();
const text = 'This is plain text with no image references at all.';
expect(addEndpointToImageUrl(text)).toBe(text);
});
it('should not double-prepend FE_DOMAIN to already-prefixed URLs with http', () => {
process.env.FE_DOMAIN = 'https://example.com';
process.env.NEXT_PUBLIC_BASE_URL = '';
it('should not double-prepend FE_DOMAIN to already-prefixed URLs with http', async () => {
vi.stubEnv('FE_DOMAIN', 'https://example.com');
const addEndpointToImageUrl = await loadAddEndpointToImageUrl();
const text = 'http://other.com/api/system/img/abc123.png';
const result = addEndpointToImageUrl(text);
......@@ -431,19 +440,19 @@ describe('addEndpointToImageUrl', () => {
expect(result).toBe(text);
});
it('should handle FE_DOMAIN with trailing slash gracefully', () => {
process.env.FE_DOMAIN = 'https://example.com';
process.env.NEXT_PUBLIC_BASE_URL = '';
it('should handle FE_DOMAIN with trailing slash gracefully', async () => {
vi.stubEnv('FE_DOMAIN', 'https://example.com/');
const { addEndpointToImageUrl: addEndpointToImageUrlWithBase } = await loadUtilsModule();
const text = '/api/system/img/file-name_123.webp';
const result = addEndpointToImageUrl(text);
const result = addEndpointToImageUrlWithBase(text);
expect(result).toBe('https://example.com/api/system/img/file-name_123.webp');
});
it('should handle image URLs with various file extensions', () => {
process.env.FE_DOMAIN = 'https://example.com';
process.env.NEXT_PUBLIC_BASE_URL = '';
it('should handle image URLs with various file extensions', async () => {
vi.stubEnv('FE_DOMAIN', 'https://example.com');
const addEndpointToImageUrl = await loadAddEndpointToImageUrl();
expect(addEndpointToImageUrl('/api/system/img/test.jpg')).toBe(
'https://example.com/api/system/img/test.jpg'
......@@ -458,4 +467,17 @@ describe('addEndpointToImageUrl', () => {
'https://example.com/api/system/img/test.webp'
);
});
it('should escape special characters in subRoute', async () => {
vi.stubEnv('FE_DOMAIN', 'https://example.com');
vi.stubEnv('NEXT_PUBLIC_BASE_URL', '/fast.gpt');
const addEndpointToImageUrl = await loadAddEndpointToImageUrl();
expect(addEndpointToImageUrl('/fast.gpt/api/system/img/test.jpg')).toBe(
'https://example.com/fast.gpt/api/system/img/test.jpg'
);
expect(addEndpointToImageUrl('/fastxgpt/api/system/img/test.jpg')).toBe(
'/fastxgpt/api/system/img/test.jpg'
);
});
});
packages/service/test/common/file/utils.test.ts
......@@ -16,74 +16,12 @@ vi.mock('@fastgpt/global/common/system/constants', async (importOriginal) => {
});
import {
getContentTypeFromHeader,
getFileMaxSize,
removeFilesByPaths,
clearDirFiles,
clearTmpUploadFiles
} from '@fastgpt/service/common/file/utils';
describe('getContentTypeFromHeader', () => {
it('should extract and normalize content type from header', () => {
expect(getContentTypeFromHeader('image/jpeg')).toBe('image/jpeg');
expect(getContentTypeFromHeader('image/png; charset=utf-8')).toBe('image/png');
expect(getContentTypeFromHeader('text/html; charset=UTF-8')).toBe('text/html');
expect(getContentTypeFromHeader('application/json;charset=utf-8')).toBe('application/json');
});
it('should handle uppercase content types and convert to lowercase', () => {
expect(getContentTypeFromHeader('Image/JPEG')).toBe('image/jpeg');
expect(getContentTypeFromHeader('IMAGE/PNG')).toBe('image/png');
expect(getContentTypeFromHeader('Application/JSON')).toBe('application/json');
expect(getContentTypeFromHeader('TEXT/HTML')).toBe('text/html');
});
it('should handle mixed case content types', () => {
expect(getContentTypeFromHeader('Image/Jpeg')).toBe('image/jpeg');
expect(getContentTypeFromHeader('IMAGE/png; charset=UTF-8')).toBe('image/png');
expect(getContentTypeFromHeader('Application/Json')).toBe('application/json');
});
it('should trim whitespace', () => {
expect(getContentTypeFromHeader(' image/jpeg ')).toBe('image/jpeg');
expect(getContentTypeFromHeader(' image/png ; charset=utf-8 ')).toBe('image/png');
expect(getContentTypeFromHeader('text/html ;charset=UTF-8')).toBe('text/html');
});
it('should handle empty or undefined input', () => {
// Empty string after processing results in empty string, not undefined
expect(getContentTypeFromHeader('')).toBe('');
expect(getContentTypeFromHeader(undefined as any)).toBe(undefined);
});
it('should handle content types with multiple parameters', () => {
expect(getContentTypeFromHeader('image/jpeg; charset=utf-8; boundary=something')).toBe(
'image/jpeg'
);
expect(getContentTypeFromHeader('multipart/form-data; boundary=----WebKit')).toBe(
'multipart/form-data'
);
});
it('should handle content types without parameters', () => {
expect(getContentTypeFromHeader('image/webp')).toBe('image/webp');
expect(getContentTypeFromHeader('image/gif')).toBe('image/gif');
expect(getContentTypeFromHeader('image/svg+xml')).toBe('image/svg+xml');
});
it('should handle special image formats', () => {
expect(getContentTypeFromHeader('image/x-icon')).toBe('image/x-icon');
expect(getContentTypeFromHeader('image/vnd.microsoft.icon')).toBe('image/vnd.microsoft.icon');
expect(getContentTypeFromHeader('image/heic')).toBe('image/heic');
expect(getContentTypeFromHeader('image/avif')).toBe('image/avif');
});
it('should handle edge cases with semicolons', () => {
expect(getContentTypeFromHeader('image/jpeg;')).toBe('image/jpeg');
expect(getContentTypeFromHeader('image/png;;')).toBe('image/png');
});
});
describe('getFileMaxSize', () => {
it('should return default max size (1000MB) when uploadFileMaxSize is not set', () => {
const original = global.feConfigs?.uploadFileMaxSize;
......
packages/service/test/common/s3/token.test.ts
......@@ -3,6 +3,12 @@ import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
const strongFileTokenKey = '1234567890abcdef1234567890abcdef';
const getExpiredTime = () => new Date(Date.now() + 5 * 60 * 1000);
const originalEnv = {
FILE_TOKEN_KEY: process.env.FILE_TOKEN_KEY,
FILE_DOMAIN: process.env.FILE_DOMAIN,
FE_DOMAIN: process.env.FE_DOMAIN,
NEXT_PUBLIC_BASE_URL: process.env.NEXT_PUBLIC_BASE_URL
};
const extractTokenFromUrl = (url: string) => {
return url.split('/').pop()?.split('?')[0] || '';
......@@ -21,7 +27,10 @@ describe('s3 token validation', () => {
});
afterEach(() => {
vi.unstubAllEnvs();
vi.stubEnv('FILE_TOKEN_KEY', originalEnv.FILE_TOKEN_KEY);
vi.stubEnv('FILE_DOMAIN', originalEnv.FILE_DOMAIN);
vi.stubEnv('FE_DOMAIN', originalEnv.FE_DOMAIN);
vi.stubEnv('NEXT_PUBLIC_BASE_URL', originalEnv.NEXT_PUBLIC_BASE_URL);
vi.restoreAllMocks();
});
......@@ -63,4 +72,15 @@ describe('s3 token validation', () => {
await expect(jwtVerifyS3ObjectKey(token)).rejects.toBe(ERROR_ENUM.unAuthFile);
});
it('normalizes endpoint slashes when signing file URLs', async () => {
vi.stubEnv('FILE_DOMAIN', 'https://files.example.com/');
vi.stubEnv('FE_DOMAIN', undefined);
vi.stubEnv('NEXT_PUBLIC_BASE_URL', '/fastgpt');
const { jwtSignS3ObjectKey } = await loadTokenModule();
const url = jwtSignS3ObjectKey('chat/appId/userId/chatId/file.txt', getExpiredTime());
expect(url).toMatch(/^https:\/\/files\.example\.com\/fastgpt\/api\/system\/file\/[^/?#]+$/);
});
});
packages/service/test/common/secret/aes256gcm.test.ts
......@@ -83,8 +83,8 @@ describe('decryptSecret', () => {
it('should throw on tampered ciphertext', () => {
const encrypted = encryptSecret('secret');
const parts = encrypted.split(':');
// Flip a byte in the encrypted data
const tampered = parts[0] + ':' + 'ff' + parts[1].slice(2) + ':' + parts[2];
const replacement = parts[1][0] === '0' ? '1' : '0';
const tampered = `${parts[0]}:${replacement}${parts[1].slice(1)}:${parts[2]}`;
expect(() => decryptSecret(tampered)).toThrow();
});
......
packages/service/test/common/security/fileUrlValidator.test.ts
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
const originalEnv = {
STORAGE_S3_ENDPOINT: process.env.STORAGE_S3_ENDPOINT,
STORAGE_EXTERNAL_ENDPOINT: process.env.STORAGE_EXTERNAL_ENDPOINT,
FE_DOMAIN: process.env.FE_DOMAIN,
PRO_URL: process.env.PRO_URL
};
describe('fileUrlValidator', () => {
const originalEnv = { ...process.env };
let originalSystemEnv: any;
beforeEach(() => {
originalSystemEnv = global.systemEnv;
vi.resetModules();
delete process.env.STORAGE_S3_ENDPOINT;
delete process.env.STORAGE_EXTERNAL_ENDPOINT;
// @ts-ignore
delete process.env.FE_DOMAIN;
delete process.env.PRO_URL;
vi.stubEnv('STORAGE_S3_ENDPOINT', undefined);
vi.stubEnv('STORAGE_EXTERNAL_ENDPOINT', undefined);
vi.stubEnv('FE_DOMAIN', undefined);
vi.stubEnv('PRO_URL', undefined);
});
afterEach(() => {
process.env = { ...originalEnv };
vi.stubEnv('STORAGE_S3_ENDPOINT', originalEnv.STORAGE_S3_ENDPOINT);
vi.stubEnv('STORAGE_EXTERNAL_ENDPOINT', originalEnv.STORAGE_EXTERNAL_ENDPOINT);
vi.stubEnv('FE_DOMAIN', originalEnv.FE_DOMAIN);
vi.stubEnv('PRO_URL', originalEnv.PRO_URL);
global.systemEnv = originalSystemEnv;
});
describe('systemWhiteList construction', () => {
it('should include STORAGE_S3_ENDPOINT when set', async () => {
process.env.STORAGE_S3_ENDPOINT = 's3.example.com';
vi.stubEnv('STORAGE_S3_ENDPOINT', 'http://s3.example.com');
global.systemEnv = { fileUrlWhitelist: ['other.com'] } as any;
const { validateFileUrlDomain } = await import(
'@fastgpt/service/common/security/fileUrlValidator'
......@@ -27,7 +38,7 @@ describe('fileUrlValidator', () => {
});
it('should extract hostname from STORAGE_EXTERNAL_ENDPOINT', async () => {
process.env.STORAGE_EXTERNAL_ENDPOINT = 'https://external.example.com/path';
vi.stubEnv('STORAGE_EXTERNAL_ENDPOINT', 'https://external.example.com/path');
global.systemEnv = { fileUrlWhitelist: ['other.com'] } as any;
const { validateFileUrlDomain } = await import(
'@fastgpt/service/common/security/fileUrlValidator'
......@@ -35,18 +46,17 @@ describe('fileUrlValidator', () => {
expect(validateFileUrlDomain('http://external.example.com/file.png')).toBe(true);
});
it('should handle invalid STORAGE_EXTERNAL_ENDPOINT gracefully', async () => {
process.env.STORAGE_EXTERNAL_ENDPOINT = 'not-a-valid-url';
it('should reject invalid STORAGE_EXTERNAL_ENDPOINT at env validation stage', async () => {
vi.stubEnv('STORAGE_EXTERNAL_ENDPOINT', 'not-a-valid-url');
global.systemEnv = { fileUrlWhitelist: ['other.com'] } as any;
const { validateFileUrlDomain } = await import(
'@fastgpt/service/common/security/fileUrlValidator'
await expect(import('@fastgpt/service/common/security/fileUrlValidator')).rejects.toThrow(
'Invalid environment variables. Please check: STORAGE_EXTERNAL_ENDPOINT'
);
expect(validateFileUrlDomain('http://other.com/file.png')).toBe(true);
expect(validateFileUrlDomain('http://not-a-valid-url/file.png')).toBe(false);
});
it('should extract hostname from FE_DOMAIN', async () => {
process.env.FE_DOMAIN = 'https://fe.example.com';
vi.stubEnv('FE_DOMAIN', 'https://fe.example.com');
global.systemEnv = { fileUrlWhitelist: ['other.com'] } as any;
const { validateFileUrlDomain } = await import(
'@fastgpt/service/common/security/fileUrlValidator'
......@@ -54,17 +64,17 @@ describe('fileUrlValidator', () => {
expect(validateFileUrlDomain('http://fe.example.com/page')).toBe(true);
});
it('should handle invalid FE_DOMAIN gracefully', async () => {
process.env.FE_DOMAIN = 'invalid-url';
it('should reject invalid FE_DOMAIN at env validation stage', async () => {
vi.stubEnv('FE_DOMAIN', 'invalid-url');
global.systemEnv = { fileUrlWhitelist: ['other.com'] } as any;
const { validateFileUrlDomain } = await import(
'@fastgpt/service/common/security/fileUrlValidator'
await expect(import('@fastgpt/service/common/security/fileUrlValidator')).rejects.toThrow(
'Invalid environment variables. Please check: FE_DOMAIN'
);
expect(validateFileUrlDomain('http://other.com/file.png')).toBe(true);
});
it('should extract hostname from PRO_URL', async () => {
process.env.PRO_URL = 'https://pro.example.com/api';
vi.stubEnv('PRO_URL', 'https://pro.example.com/api');
global.systemEnv = { fileUrlWhitelist: ['other.com'] } as any;
const { validateFileUrlDomain } = await import(
'@fastgpt/service/common/security/fileUrlValidator'
......@@ -72,20 +82,20 @@ describe('fileUrlValidator', () => {
expect(validateFileUrlDomain('http://pro.example.com/resource')).toBe(true);
});
it('should handle invalid PRO_URL gracefully', async () => {
process.env.PRO_URL = 'bad-url';
it('should reject invalid PRO_URL at env validation stage', async () => {
vi.stubEnv('PRO_URL', 'bad-url');
global.systemEnv = { fileUrlWhitelist: ['other.com'] } as any;
const { validateFileUrlDomain } = await import(
'@fastgpt/service/common/security/fileUrlValidator'
await expect(import('@fastgpt/service/common/security/fileUrlValidator')).rejects.toThrow(
'Invalid environment variables. Please check: PRO_URL'
);
expect(validateFileUrlDomain('http://other.com/file.png')).toBe(true);
});
it('should combine all env vars into systemWhiteList', async () => {
process.env.STORAGE_S3_ENDPOINT = 's3.example.com';
process.env.STORAGE_EXTERNAL_ENDPOINT = 'https://external.example.com';
process.env.FE_DOMAIN = 'https://fe.example.com';
process.env.PRO_URL = 'https://pro.example.com';
vi.stubEnv('STORAGE_S3_ENDPOINT', 'http://s3.example.com');
vi.stubEnv('STORAGE_EXTERNAL_ENDPOINT', 'https://external.example.com');
vi.stubEnv('FE_DOMAIN', 'https://fe.example.com');
vi.stubEnv('PRO_URL', 'https://pro.example.com');
global.systemEnv = { fileUrlWhitelist: ['user.com'] } as any;
const { validateFileUrlDomain } = await import(
'@fastgpt/service/common/security/fileUrlValidator'
......@@ -160,7 +170,7 @@ describe('fileUrlValidator', () => {
});
it('should match against both fileUrlWhitelist and systemWhiteList', async () => {
process.env.STORAGE_S3_ENDPOINT = 's3.system.com';
vi.stubEnv('STORAGE_S3_ENDPOINT', 'http://s3.system.com');
global.systemEnv = { fileUrlWhitelist: ['user.com'] } as any;
const { validateFileUrlDomain } = await import(
'@fastgpt/service/common/security/fileUrlValidator'
......
packages/service/test/common/system/utils.test.ts
import { describe, test, expect, beforeEach, afterEach, vi } from 'vitest';
import { isInternalAddress } from '@fastgpt/service/common/system/utils';
import { serviceEnv } from '@fastgpt/service/env';
import dns from 'dns/promises';
describe('SSRF Protection - isInternalAddress', () => {
const originalEnv = process.env.CHECK_INTERNAL_IP;
const originalCheckInternalIp = serviceEnv.CHECK_INTERNAL_IP;
beforeEach(() => {
process.env.CHECK_INTERNAL_IP = 'true';
serviceEnv.CHECK_INTERNAL_IP = true;
// 重建 DNS spy，避免真实 DNS 解析和用例之间的 mock 实现串味
vi.restoreAllMocks();
vi.spyOn(dns, 'resolve4').mockRejectedValue(new Error('No A records'));
......@@ -14,12 +15,7 @@ describe('SSRF Protection - isInternalAddress', () => {
});
afterEach(() => {
// 恢复原始环境变量
if (originalEnv !== undefined) {
process.env.CHECK_INTERNAL_IP = originalEnv;
} else {
delete process.env.CHECK_INTERNAL_IP;
}
serviceEnv.CHECK_INTERNAL_IP = originalCheckInternalIp;
});
describe('Localhost 检查（始终阻止）', () => {
......@@ -74,7 +70,7 @@ describe('SSRF Protection - isInternalAddress', () => {
describe('CHECK_INTERNAL_IP 未设置时（默认行为 - 安全优先）', () => {
beforeEach(() => {
delete process.env.CHECK_INTERNAL_IP;
serviceEnv.CHECK_INTERNAL_IP = false;
});
test('应该允许公共 IP 地址', async () => {
......@@ -112,7 +108,7 @@ describe('SSRF Protection - isInternalAddress', () => {
describe('CHECK_INTERNAL_IP=false 时（向后兼容模式）', () => {
beforeEach(() => {
process.env.CHECK_INTERNAL_IP = 'false';
serviceEnv.CHECK_INTERNAL_IP = false;
});
test('应该允许公共 IP 地址', async () => {
......@@ -446,7 +442,7 @@ describe('SSRF Protection - isInternalAddress', () => {
// GHSA-jhqw-944x-xh94: 云元数据端点 SSRF 保护绕过
describe('GHSA-jhqw-944x-xh94 元数据端点绕过防护', () => {
beforeEach(() => {
delete process.env.CHECK_INTERNAL_IP;
serviceEnv.CHECK_INTERNAL_IP = false;
});
test('应该阻止显式端口绕过 http://169.254.169.254:80/', async () => {
......
packages/service/test/core/agentSkills/sandboxConfig.test.ts
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const envBackup = { ...process.env };
const originalEnv = {
AGENT_SANDBOX_PROVIDER: process.env.AGENT_SANDBOX_PROVIDER,
AGENT_SANDBOX_SEALOS_BASEURL: process.env.AGENT_SANDBOX_SEALOS_BASEURL,
AGENT_SANDBOX_SEALOS_TOKEN: process.env.AGENT_SANDBOX_SEALOS_TOKEN,
AGENT_SANDBOX_OPENSANDBOX_RUNTIME: process.env.AGENT_SANDBOX_OPENSANDBOX_RUNTIME
};
const loadSandboxConfigModule = async () => {
vi.resetModules();
......@@ -10,18 +15,20 @@ const loadSandboxConfigModule = async () => {
describe('sandboxConfig provider helpers', () => {
beforeEach(() => {
vi.clearAllMocks();
process.env = { ...envBackup };
});
afterEach(() => {
process.env = { ...envBackup };
vi.stubEnv('AGENT_SANDBOX_PROVIDER', originalEnv.AGENT_SANDBOX_PROVIDER);
vi.stubEnv('AGENT_SANDBOX_SEALOS_BASEURL', originalEnv.AGENT_SANDBOX_SEALOS_BASEURL);
vi.stubEnv('AGENT_SANDBOX_SEALOS_TOKEN', originalEnv.AGENT_SANDBOX_SEALOS_TOKEN);
vi.stubEnv('AGENT_SANDBOX_OPENSANDBOX_RUNTIME', originalEnv.AGENT_SANDBOX_OPENSANDBOX_RUNTIME);
});
it('parses sealosdevbox config from env', async () => {
process.env.AGENT_SANDBOX_PROVIDER = 'sealosdevbox';
process.env.AGENT_SANDBOX_SEALOS_BASEURL = 'https://devbox.example.com';
process.env.AGENT_SANDBOX_SEALOS_TOKEN = 'sealos-token';
process.env.AGENT_SANDBOX_RUNTIME = 'docker';
vi.stubEnv('AGENT_SANDBOX_PROVIDER', 'sealosdevbox');
vi.stubEnv('AGENT_SANDBOX_SEALOS_BASEURL', 'https://devbox.example.com');
vi.stubEnv('AGENT_SANDBOX_SEALOS_TOKEN', 'sealos-token');
vi.stubEnv('AGENT_SANDBOX_OPENSANDBOX_RUNTIME', 'docker');
const { getSandboxProviderConfig } = await loadSandboxConfigModule();
......
packages/service/test/core/ai/config.test.ts 0 → 100644
import { afterEach, describe, expect, it, vi } from 'vitest';
const originalEnv = {
AIPROXY_API_ENDPOINT: process.env.AIPROXY_API_ENDPOINT,
AIPROXY_API_TOKEN: process.env.AIPROXY_API_TOKEN,
OPENAI_BASE_URL: process.env.OPENAI_BASE_URL,
CHAT_API_KEY: process.env.CHAT_API_KEY
};
const importConfig = async () => {
vi.resetModules();
return import('@fastgpt/service/core/ai/config');
};
describe('AI config defaults', () => {
afterEach(() => {
vi.stubEnv('AIPROXY_API_ENDPOINT', originalEnv.AIPROXY_API_ENDPOINT);
vi.stubEnv('AIPROXY_API_TOKEN', originalEnv.AIPROXY_API_TOKEN);
vi.stubEnv('OPENAI_BASE_URL', originalEnv.OPENAI_BASE_URL);
vi.stubEnv('CHAT_API_KEY', originalEnv.CHAT_API_KEY);
});
it('falls back to OpenAI config when AI Proxy is not explicitly configured', async () => {
vi.stubEnv('AIPROXY_API_ENDPOINT', '');
vi.stubEnv('AIPROXY_API_TOKEN', '');
vi.stubEnv('OPENAI_BASE_URL', 'https://example.com/v1');
vi.stubEnv('CHAT_API_KEY', 'sk-chat');
const { openaiBaseUrl, openaiBaseKey } = await importConfig();
expect(openaiBaseUrl).toBe('https://example.com/v1');
expect(openaiBaseKey).toBe('sk-chat');
});
it('uses AI Proxy only when the endpoint is explicitly configured', async () => {
vi.stubEnv('AIPROXY_API_ENDPOINT', 'http://aiproxy:3000');
vi.stubEnv('AIPROXY_API_TOKEN', 'aiproxy-token');
vi.stubEnv('OPENAI_BASE_URL', 'https://example.com/v1');
vi.stubEnv('CHAT_API_KEY', 'sk-chat');
const { openaiBaseUrl, openaiBaseKey } = await importConfig();
expect(openaiBaseUrl).toBe('http://aiproxy:3000/v1');
expect(openaiBaseKey).toBe('aiproxy-token');
});
it('normalizes trailing slashes from AI Proxy endpoint', async () => {
vi.stubEnv('AIPROXY_API_ENDPOINT', 'http://aiproxy:3000///');
vi.stubEnv('AIPROXY_API_TOKEN', 'aiproxy-token');
vi.stubEnv('OPENAI_BASE_URL', 'https://example.com/v1');
vi.stubEnv('CHAT_API_KEY', 'sk-chat');
const { openaiBaseUrl, openaiBaseKey } = await importConfig();
expect(openaiBaseUrl).toBe('http://aiproxy:3000/v1');
expect(openaiBaseKey).toBe('aiproxy-token');
});
it('falls back to chat API key when AI Proxy endpoint has no token', async () => {
vi.stubEnv('AIPROXY_API_ENDPOINT', 'http://aiproxy:3000');
vi.stubEnv('AIPROXY_API_TOKEN', undefined);
vi.stubEnv('OPENAI_BASE_URL', 'https://example.com/v1');
vi.stubEnv('CHAT_API_KEY', 'sk-chat');
const { openaiBaseUrl, openaiBaseKey } = await importConfig();
expect(openaiBaseUrl).toBe('http://aiproxy:3000/v1');
expect(openaiBaseKey).toBe('sk-chat');
});
it('ignores AI Proxy token when endpoint is not configured', async () => {
vi.stubEnv('AIPROXY_API_ENDPOINT', '');
vi.stubEnv('AIPROXY_API_TOKEN', 'aiproxy-token');
vi.stubEnv('OPENAI_BASE_URL', 'https://example.com/v1');
vi.stubEnv('CHAT_API_KEY', 'sk-chat');
const { openaiBaseUrl, openaiBaseKey } = await importConfig();
expect(openaiBaseUrl).toBe('https://example.com/v1');
expect(openaiBaseKey).toBe('sk-chat');
});
});
packages/service/test/core/ai/llm/utils.test.ts
......@@ -34,6 +34,7 @@ vi.mock('axios', () => ({
import { countGptMessagesTokens } from '@fastgpt/service/common/string/tiktoken/index';
import { getImageBase64 } from '@fastgpt/service/common/file/image/utils';
import { serviceEnv } from '@fastgpt/service/env';
// @ts-ignore
import axios from 'axios';
......@@ -566,8 +567,8 @@ describe('loadRequestMessages function tests', () => {
});
it('should handle invalid remote images gracefully', async () => {
const originalEnv = process.env.MULTIPLE_DATA_TO_BASE64;
process.env.MULTIPLE_DATA_TO_BASE64 = 'false'; // Disable base64 conversion
const originalMultipleDataToBase64 = serviceEnv.MULTIPLE_DATA_TO_BASE64;
serviceEnv.MULTIPLE_DATA_TO_BASE64 = false;
const messages: ChatCompletionMessageParam[] = [
{
......@@ -581,19 +582,13 @@ describe('loadRequestMessages function tests', () => {
mockAxiosHead.mockRejectedValue(new Error('Network error'));
const result = await loadRequestMessages({ messages, useVision: true });
try {
const result = await loadRequestMessages({ messages, useVision: true });
expect(result).toHaveLength(1);
// When image is filtered out and only one text item remains, it becomes string
expect(typeof result[0].content).toBe('string');
expect(result[0].content).toBe('Text');
// Restore original environment
if (originalEnv !== undefined) {
process.env.MULTIPLE_DATA_TO_BASE64 = originalEnv;
} else {
// @ts-ignore
delete process.env.MULTIPLE_DATA_TO_BASE64;
expect(result).toHaveLength(1);
expect(result[0].content).toBe('Text');
} finally {
serviceEnv.MULTIPLE_DATA_TO_BASE64 = originalMultipleDataToBase64;
}
});
......@@ -868,9 +863,6 @@ describe('loadRequestMessages function tests', () => {
});
it('should handle environment variable MULTIPLE_DATA_TO_BASE64', async () => {
const originalEnv = process.env.MULTIPLE_DATA_TO_BASE64;
process.env.MULTIPLE_DATA_TO_BASE64 = 'true';
const messages: ChatCompletionMessageParam[] = [
{
role: ChatCompletionRequestMessageRoleEnum.User,
......@@ -890,13 +882,6 @@ describe('loadRequestMessages function tests', () => {
expect(result).toHaveLength(1);
const content = result[0].content as any[];
expect(content[0].image_url.url).toBe('data:image/png;base64,converted');
// Restore original environment
if (originalEnv !== undefined) {
process.env.MULTIPLE_DATA_TO_BASE64 = originalEnv;
} else {
process.env.MULTIPLE_DATA_TO_BASE64 = '';
}
});
});
});
packages/service/test/core/ai/sandbox/controller.test.ts
......@@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, beforeAll, vi } from 'vitest';
// Mock the env module BEFORE any imports that use it
vi.mock('@fastgpt/service/env', () => ({
env: {
serviceEnv: {
AGENT_SANDBOX_PROVIDER: 'sealosdevbox',
AGENT_SANDBOX_SEALOS_BASEURL: 'http://mock-sandbox.local',
AGENT_SANDBOX_SEALOS_TOKEN: 'mock-token-12345'
......
packages/service/test/core/ai/sandbox/sandbox.integration.test.ts
......@@ -14,7 +14,7 @@ const { Types } = connectionMongo;
const hasSandboxEnv = !!process.env.AGENT_SANDBOX_PROVIDER;
vi.mock('@fastgpt/service/env', () => ({
env: {
serviceEnv: {
AGENT_SANDBOX_PROVIDER: process.env.AGENT_SANDBOX_PROVIDER,
AGENT_SANDBOX_SEALOS_BASEURL: process.env.AGENT_SANDBOX_SEALOS_BASEURL,
AGENT_SANDBOX_SEALOS_TOKEN: process.env.AGENT_SANDBOX_SEALOS_TOKEN,
......
packages/service/test/core/app/http.test.ts
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { runHTTPTool } from '@fastgpt/service/core/app/http';
import { PRIVATE_URL_TEXT } from '@fastgpt/service/common/system/utils';
import { serviceEnv } from '@fastgpt/service/env';
describe('SSRF Vulnerability Fix Tests', () => {
const originalEnv = process.env.CHECK_INTERNAL_IP;
const originalCheckInternalIp = serviceEnv.CHECK_INTERNAL_IP;
beforeEach(() => {
// 确保测试环境启用内部 IP 检查
process.env.CHECK_INTERNAL_IP = 'true';
serviceEnv.CHECK_INTERNAL_IP = true;
});
afterEach(() => {
// 恢复原始环境变量
if (originalEnv !== undefined) {
process.env.CHECK_INTERNAL_IP = originalEnv;
} else {
delete process.env.CHECK_INTERNAL_IP;
}
serviceEnv.CHECK_INTERNAL_IP = originalCheckInternalIp;
});
describe('AWS Metadata Endpoint Protection', () => {
......@@ -251,7 +247,7 @@ describe('SSRF Vulnerability Fix Tests', () => {
describe('Environment Variable Control', () => {
it('should always block cloud metadata endpoints even when CHECK_INTERNAL_IP=false', async () => {
process.env.CHECK_INTERNAL_IP = 'false';
serviceEnv.CHECK_INTERNAL_IP = false;
// 云服务商元数据端点应该始终被阻止，这是安全的关键
const result = await runHTTPTool({
......@@ -265,7 +261,7 @@ describe('SSRF Vulnerability Fix Tests', () => {
});
it('should always block localhost even when CHECK_INTERNAL_IP=false', async () => {
process.env.CHECK_INTERNAL_IP = 'false';
serviceEnv.CHECK_INTERNAL_IP = false;
// localhost 应该始终被阻止
const result = await runHTTPTool({
......@@ -279,7 +275,7 @@ describe('SSRF Vulnerability Fix Tests', () => {
});
it('should block internal addresses by default (no env var)', async () => {
delete process.env.CHECK_INTERNAL_IP;
serviceEnv.CHECK_INTERNAL_IP = false;
const result = await runHTTPTool({
baseUrl: 'http://localhost',
......@@ -292,7 +288,7 @@ describe('SSRF Vulnerability Fix Tests', () => {
});
it('should block internal addresses when CHECK_INTERNAL_IP=true', async () => {
process.env.CHECK_INTERNAL_IP = 'true';
serviceEnv.CHECK_INTERNAL_IP = true;
const result = await runHTTPTool({
baseUrl: 'http://localhost',
......
packages/service/test/core/workflow/dispatch/loopRun/runLoopRun.test.ts
......@@ -19,7 +19,7 @@ vi.mock('@fastgpt/service/core/workflow/dispatch', () => ({
// Shrink max iterations so overflow tests run fast.
vi.mock('@fastgpt/service/env', () => ({
env: { WORKFLOW_MAX_LOOP_TIMES: 5 }
serviceEnv: { WORKFLOW_MAX_LOOP_TIMES: 5 }
}));
// Import after mocks so runLoopRun pulls the mocked modules.
......
packages/service/test/worker/function.test.ts 0 → 100644
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { WorkerNameEnum } from '@fastgpt/service/worker/utils';
// hoisted: 这些 mock 必须在 vi.mock 工厂里可见
const { mockRun, mockGetWorkerController, mockRunWorker, mockEnv } = vi.hoisted(() => {
const mockRun = vi.fn();
return {
mockRun,
mockGetWorkerController: vi.fn(() => ({ run: mockRun })),
mockRunWorker: vi.fn(),
mockEnv: {
PARSE_FILE_WORKERS: 10,
HTML_TO_MARKDOWN_WORKERS: 10,
TEXT_TO_CHUNKS_WORKERS: 10,
PARSE_FILE_TIMEOUT_SECONDS: 300
} as {
PARSE_FILE_WORKERS: number;
HTML_TO_MARKDOWN_WORKERS: number;
TEXT_TO_CHUNKS_WORKERS: number;
PARSE_FILE_TIMEOUT_SECONDS: number;
}
};
});
// 拦截 getWorkerController / runWorker，保留 WorkerNameEnum 等枚举
vi.mock('@fastgpt/service/worker/utils', async (importOriginal) => {
const mod = await importOriginal<typeof import('@fastgpt/service/worker/utils')>();
return {
...mod,
getWorkerController: mockGetWorkerController,
runWorker: mockRunWorker
};
});
// 拦截 env，避免每个用例通过修改 process.env 失效（env 在模块加载时已固化）
vi.mock('@fastgpt/service/env', () => ({
serviceEnv: mockEnv
}));
// 必须在 vi.mock 之后再 import 被测模块
const { text2Chunks, readRawContentFromBuffer } = await import('@fastgpt/service/worker/function');
const { htmlToMarkdown } = await import('@fastgpt/service/common/string/utils');
describe('worker/function', () => {
beforeEach(() => {
mockRun.mockReset();
mockGetWorkerController.mockReset();
mockGetWorkerController.mockImplementation(() => ({ run: mockRun }));
mockRunWorker.mockReset();
});
describe('text2Chunks', () => {
it('test 环境下短路调用本地 splitText2Chunks，不创建 worker', async () => {
const result = await text2Chunks({
text: 'hello world this is a test',
chunkSize: 10,
maxSize: 50
});
expect(result).toBeDefined();
expect(Array.isArray(result.chunks)).toBe(true);
expect(result.chunks.length).toBeGreaterThan(0);
expect(result.chunks.join('')).toContain('hello world');
// 关键：测试环境必须走短路，绝不能调起 worker
expect(mockRunWorker).not.toHaveBeenCalled();
expect(mockGetWorkerController).not.toHaveBeenCalled();
});
it('空文本返回空 chunks 列表', async () => {
const result = await text2Chunks({ text: '', chunkSize: 100, maxSize: 200 });
expect(result.chunks).toEqual([]);
});
});
describe('readRawContentFromBuffer', () => {
afterEach(() => {
// 防止 env 跨用例污染
mockEnv.PARSE_FILE_WORKERS = 10;
mockEnv.PARSE_FILE_TIMEOUT_SECONDS = 300;
});
it('使用 SharedArrayBuffer 包装 Buffer 并通过 pool.run 派发', async () => {
const original = Buffer.from('hello world', 'utf-8');
const expected = { rawText: 'parsed-content' };
mockRun.mockResolvedValueOnce(expected);
const result = await readRawContentFromBuffer({
extension: 'txt',
encoding: 'utf-8',
buffer: original
});
expect(result).toEqual(expected);
// pool 配置
expect(mockGetWorkerController).toHaveBeenCalledTimes(1);
const poolCfg = mockGetWorkerController.mock.calls[0][0];
expect(poolCfg.name).toBe(WorkerNameEnum.readFile);
expect(poolCfg.maxReservedThreads).toBe(10); // 默认值
expect(poolCfg.taskTimeoutMs).toBe(5 * 60 * 1000);
expect(poolCfg.maxTasksPerWorker).toBe(100);
// run 入参
expect(mockRun).toHaveBeenCalledTimes(1);
const runArg = mockRun.mock.calls[0][0];
expect(runArg.extension).toBe('txt');
expect(runArg.encoding).toBe('utf-8');
expect(runArg.bufferSize).toBe(original.length);
expect(runArg.sharedBuffer).toBeInstanceOf(SharedArrayBuffer);
// SharedArrayBuffer 内容必须完整复刻原 Buffer
expect(runArg.sharedBuffer.byteLength).toBe(original.length);
const sharedView = new Uint8Array(runArg.sharedBuffer);
expect(Array.from(sharedView)).toEqual(Array.from(original));
});
it('空 Buffer 也能正常构造（byteLength 为 0）', async () => {
mockRun.mockResolvedValueOnce({ rawText: '' });
const result = await readRawContentFromBuffer({
extension: 'txt',
encoding: 'utf-8',
buffer: Buffer.alloc(0)
});
expect(result).toEqual({ rawText: '' });
const runArg = mockRun.mock.calls[0][0];
expect(runArg.bufferSize).toBe(0);
expect(runArg.sharedBuffer.byteLength).toBe(0);
});
it('二进制 Buffer 不应在拷贝过程中失真', async () => {
const bytes = new Uint8Array([0x00, 0x01, 0xff, 0x80, 0x7f, 0xab, 0xcd]);
const original = Buffer.from(bytes);
mockRun.mockResolvedValueOnce({ rawText: '' });
await readRawContentFromBuffer({
extension: 'pdf',
encoding: 'utf-8',
buffer: original
});
const runArg = mockRun.mock.calls[0][0];
const sharedView = new Uint8Array(runArg.sharedBuffer);
expect(Array.from(sharedView)).toEqual(Array.from(bytes));
});
it('PARSE_FILE_WORKERS 自定义值生效', async () => {
mockEnv.PARSE_FILE_WORKERS = 8;
mockRun.mockResolvedValueOnce({ rawText: '' });
await readRawContentFromBuffer({
extension: 'txt',
encoding: 'utf-8',
buffer: Buffer.from('x')
});
const poolCfg = mockGetWorkerController.mock.calls[0][0];
expect(poolCfg.maxReservedThreads).toBe(8);
});
it('PARSE_FILE_TIMEOUT_SECONDS 自定义值生效（秒 -> 毫秒）', async () => {
mockEnv.PARSE_FILE_TIMEOUT_SECONDS = 120;
mockRun.mockResolvedValueOnce({ rawText: '' });
await readRawContentFromBuffer({
extension: 'txt',
encoding: 'utf-8',
buffer: Buffer.from('x')
});
const poolCfg = mockGetWorkerController.mock.calls[0][0];
expect(poolCfg.taskTimeoutMs).toBe(120 * 1000);
});
it('pool.run 的错误必须原样抛出', async () => {
mockRun.mockRejectedValueOnce(new Error('parse failed'));
await expect(
readRawContentFromBuffer({
extension: 'pdf',
encoding: 'utf-8',
buffer: Buffer.from('garbage')
})
).rejects.toThrow('parse failed');
});
it('多次调用每次都通过 getWorkerController 获取池（不在本层缓存）', async () => {
mockRun.mockResolvedValue({ rawText: 'ok' });
await readRawContentFromBuffer({
extension: 'txt',
encoding: 'utf-8',
buffer: Buffer.from('a')
});
await readRawContentFromBuffer({
extension: 'txt',
encoding: 'utf-8',
buffer: Buffer.from('b')
});
await readRawContentFromBuffer({
extension: 'txt',
encoding: 'utf-8',
buffer: Buffer.from('c')
});
// 单例由 utils.getWorkerController 内部维护，function.ts 不应自行缓存
expect(mockGetWorkerController).toHaveBeenCalledTimes(3);
expect(mockRun).toHaveBeenCalledTimes(3);
});
it('每次调用都生成新的 SharedArrayBuffer（避免跨任务串扰）', async () => {
mockRun.mockResolvedValue({ rawText: 'ok' });
await readRawContentFromBuffer({
extension: 'txt',
encoding: 'utf-8',
buffer: Buffer.from('aaa')
});
await readRawContentFromBuffer({
extension: 'txt',
encoding: 'utf-8',
buffer: Buffer.from('bbb')
});
const sab1 = mockRun.mock.calls[0][0].sharedBuffer;
const sab2 = mockRun.mock.calls[1][0].sharedBuffer;
expect(sab1).not.toBe(sab2);
expect(new Uint8Array(sab1)[0]).toBe('a'.charCodeAt(0));
expect(new Uint8Array(sab2)[0]).toBe('b'.charCodeAt(0));
});
});
describe('htmlToMarkdown', () => {
afterEach(() => {
mockEnv.HTML_TO_MARKDOWN_WORKERS = 10;
mockEnv.PARSE_FILE_TIMEOUT_SECONDS = 300;
});
it('通过 htmlStr2Md worker pool 派发并返回 rawText', async () => {
mockRun.mockResolvedValueOnce({ rawText: '# Title', imageList: [] });
const result = await htmlToMarkdown('<h1>Title</h1>');
expect(result).toBe('# Title');
expect(mockRunWorker).not.toHaveBeenCalled();
expect(mockGetWorkerController).toHaveBeenCalledTimes(1);
const poolCfg = mockGetWorkerController.mock.calls[0][0];
expect(poolCfg.name).toBe(WorkerNameEnum.htmlStr2Md);
expect(poolCfg.maxReservedThreads).toBe(10);
expect(poolCfg.taskTimeoutMs).toBe(5 * 60 * 1000);
expect(poolCfg.maxTasksPerWorker).toBe(100);
expect(mockRun).toHaveBeenCalledWith({ html: '<h1>Title</h1>' });
});
it('空 html 统一传空字符串', async () => {
mockRun.mockResolvedValueOnce({ rawText: '', imageList: [] });
const result = await htmlToMarkdown(null);
expect(result).toBe('');
expect(mockRun).toHaveBeenCalledWith({ html: '' });
});
it('HTML_TO_MARKDOWN_WORKERS 自定义值生效', async () => {
mockEnv.HTML_TO_MARKDOWN_WORKERS = 6;
mockRun.mockResolvedValueOnce({ rawText: 'ok', imageList: [] });
await htmlToMarkdown('<p>ok</p>');
const poolCfg = mockGetWorkerController.mock.calls[0][0];
expect(poolCfg.maxReservedThreads).toBe(6);
});
});
});
packages/service/test/worker/readFile/integration.test.ts 0 → 100644
import { describe, it, expect, beforeAll, afterEach, afterAll, vi } from 'vitest';
import path from 'path';
import { existsSync } from 'fs';
/*
* 真实 spawn 测试：使用 projects/app/worker/readFile.js 构建产物，
* 通过 WorkerPool 实际启动 Node Worker 线程并解析。
*
* WorkerPool 内部会通过 process.cwd()/worker/readFile.js 解析构建产物。
* 测试运行在 packages/service 下，因此本文件临时把 process.cwd() 指向
* projects/app，让测试路径和真实运行时保持一致。
*
* 默认跳过；设置 RUN_READ_FILE_WORKER_INTEGRATION=true 且构建产物存在时才运行。
*/
const APP_PROJECT_DIR = path.resolve(__dirname, '../../../../../projects/app');
const REAL_WORKER_PATH = path.join(APP_PROJECT_DIR, 'worker/readFile.js');
const shouldRunIntegration =
process.env.RUN_READ_FILE_WORKER_INTEGRATION === 'true' && existsSync(REAL_WORKER_PATH);
const { WorkerNameEnum } = await import('@fastgpt/service/worker/utils');
const { readRawContentFromBuffer } = await import('@fastgpt/service/worker/function');
const describeIfEnabled = shouldRunIntegration ? describe : describe.skip;
const getReadFilePool = () => {
const pool = (global as any).workerPoll?.[WorkerNameEnum.readFile];
expect(pool).toBeDefined();
return pool;
};
const getIdleWorker = () => {
const pool = getReadFilePool();
const idleWorker = pool.workerQueue.find((worker: any) => worker.status === 'idle');
expect(idleWorker).toBeDefined();
return idleWorker;
};
const parseText = (text: string) =>
readRawContentFromBuffer({
extension: 'txt',
encoding: 'utf-8',
buffer: Buffer.from(text, 'utf-8')
});
const destroyReadFilePool = async () => {
const workerPoll = (global as any).workerPoll;
const pool = workerPoll?.[WorkerNameEnum.readFile];
if (!pool?.workerQueue) return;
await Promise.all(
pool.workerQueue.map(async (item: any) => {
item.worker.removeAllListeners();
await item.worker.terminate();
})
);
pool.workerQueue = [];
pool.waitQueue = [];
delete workerPoll[WorkerNameEnum.readFile];
};
describeIfEnabled('readFile worker (real spawn integration)', () => {
let cwdSpy: ReturnType<typeof vi.spyOn>;
if (process.env.RUN_READ_FILE_WORKER_INTEGRATION === 'true' && !existsSync(REAL_WORKER_PATH)) {
// eslint-disable-next-line no-console
console.warn(
`[skipped] readFile worker integration requires RUN_READ_FILE_WORKER_INTEGRATION=true and worker bundle at ${REAL_WORKER_PATH}.`
);
}
beforeAll(() => {
cwdSpy = vi.spyOn(process, 'cwd').mockReturnValue(APP_PROJECT_DIR);
});
afterEach(async () => {
await destroyReadFilePool();
});
afterAll(() => {
cwdSpy.mockRestore();
});
it('解析 txt 文本（真实 worker）', async () => {
const text = '这是一个测试 hello world\n第二行';
const result = await parseText(text);
expect(result.rawText).toBe(text);
});
it('解析 md 文本', async () => {
const md = '# Title\n\nbody paragraph.\n\n- item 1\n- item 2';
const result = await readRawContentFromBuffer({
extension: 'md',
encoding: 'utf-8',
buffer: Buffer.from(md, 'utf-8')
});
expect(result.rawText).toContain('# Title');
expect(result.rawText).toContain('item 1');
});
it('解析 csv', async () => {
const csv = 'name,age,city\nAlice,30,Beijing\nBob,25,Shanghai';
const result = await readRawContentFromBuffer({
extension: 'csv',
encoding: 'utf-8',
buffer: Buffer.from(csv, 'utf-8')
});
expect(result.rawText).toContain('Alice');
expect(result.rawText).toContain('30');
expect(result.rawText).toContain('Shanghai');
});
it('未知扩展名应被 reject', async () => {
await expect(
readRawContentFromBuffer({
extension: 'unknown_xyz',
encoding: 'utf-8',
buffer: Buffer.from('x')
})
).rejects.toBeTruthy();
// worker 在 reject 后应仍存活、可继续接任务
const ok = await parseText('still alive');
expect(ok.rawText).toBe('still alive');
});
it('worker 复用：顺序多次调用累积在同一 worker 上', async () => {
await parseText('warmup');
const pool = getReadFilePool();
const targetWorker = getIdleWorker();
const initialTasks = targetWorker.tasksCompleted;
const initialQueueLen = pool.workerQueue.length;
for (let i = 0; i < 5; i++) {
await parseText(`line-${i}`);
}
// 池容量没变（顺序调用不需要新建）
expect(pool.workerQueue.length).toBe(initialQueueLen);
// 同一 worker 任务计数 +5
const sameWorker = pool.workerQueue.find((w: any) => w.id === targetWorker.id);
expect(sameWorker?.tasksCompleted).toBe(initialTasks + 5);
});
it('并发场景：池按上限扩容，所有任务都成功返回', async () => {
const concurrency = 4;
const results = await Promise.all(
Array.from({ length: concurrency }, (_, i) =>
readRawContentFromBuffer({
extension: 'txt',
encoding: 'utf-8',
buffer: Buffer.from(`payload-${i}`, 'utf-8')
})
)
);
expect(results).toHaveLength(concurrency);
results.forEach((r, i) => expect(r.rawText).toBe(`payload-${i}`));
const pool = getReadFilePool();
// 池子大小不应超过 maxReservedThreads
expect(pool.workerQueue.length).toBeLessThanOrEqual(pool.maxReservedThreads);
expect(pool.workerQueue.length).toBeGreaterThan(1);
});
it('maxTasksPerWorker 触发回收：任务数达到阈值后 worker 被销毁', async () => {
await parseText('warmup');
const pool = getReadFilePool();
const idle = getIdleWorker();
const originalMax = pool.maxTasksPerWorker;
const targetId = idle.id;
try {
pool.maxTasksPerWorker = idle.tasksCompleted + 1; // 下一次任务即触发回收
await parseText('recycle me');
} finally {
pool.maxTasksPerWorker = originalMax;
}
// 那个被回收的 worker 应该已经从队列里摘除
expect(pool.workerQueue.find((w: any) => w.id === targetId)).toBeUndefined();
});
it('二进制保真：含 0x00 / 0xFF 的字节透传 worker 不丢字节', async () => {
// 用 csv 这条相对纯文本的路径，但塞入控制字符
const bytes = new Uint8Array([
'a'.charCodeAt(0),
0x00,
'b'.charCodeAt(0),
0xff,
'c'.charCodeAt(0)
]);
const result = await readRawContentFromBuffer({
extension: 'txt',
encoding: 'utf-8',
buffer: Buffer.from(bytes)
});
// 至少 a/b/c 被保留（中间的非法字节由 utf-8 decoder 处理，不应使整个解析失败）
expect(result.rawText).toContain('a');
expect(result.rawText).toContain('b');
expect(result.rawText).toContain('c');
});
});
pro @ f269963d
This diff is collapsed. Click to expand it.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment