Commit 14373770 by Archer Committed by GitHub

fix: volumn manager (#6971)

parent 4a9533a6
......@@ -5,9 +5,10 @@ npm-debug.log
README.md
.next
.git
projects/volume-manager/dist
.yalc/
yalc.lock
testApi/
*.local.*
*.local
\ No newline at end of file
*.local
......@@ -173,8 +173,9 @@ jobs:
id: build
uses: docker/build-push-action@v6
with:
context: projects/volume-manager
context: .
file: projects/volume-manager/Dockerfile
target: runner
platforms: linux/${{ matrix.arch }}
labels: |
org.opencontainers.image.source=https://github.com/${{ github.repository }}
......
......@@ -3,9 +3,21 @@ title: 'V4.15.0-beta2(进行中)'
description: 'FastGPT V4.15.0-beta2 更新说明'
---
## 更新指南
## 升级指南
1. AGENT_SANDBOX_VOLUME_MANAGER_MOUNT_PATH 不再生效,opensandbox 固定挂载持久化数据到 `/workspace`,旧的沙盒持久化会受到影响。
### 镜像变更
- 更新 fastgpt-app(fastgpt 主服务) 镜像 tag: v4.15.0-beta2
- 更新 fastgpt-pro(fastgpt 商业版) 镜像 tag: v4.15.0-beta2
如果使用 Opensandbox,更新下面镜像
- 更新 fastgpt-agent-sandbox 镜像 tag: v0.2.0
- 更新 fastgpt-agent-volume-manager 镜像 tag: v0.2.0
### 环境变量更新
1. 如使用 opensandbox,则 AGENT_SANDBOX_VOLUME_MANAGER_MOUNT_PATH 不再生效,可移除。opensandbox 固定挂载持久化数据到 `/workspace`,旧的沙盒持久化会受到影响。
## 🚀 新增内容
......@@ -45,3 +57,4 @@ description: 'FastGPT V4.15.0-beta2 更新说明'
1. 拆分 AI request、工作流运行详情代码。
2. 用户自定义密钥计费逻辑。
3. 流恢复相关模块补充设计文档与单元测试(stop 状态、stale 清理、历史标题、dataId 校验、表单回填等)。
4. volumn manager 将 bun 改成 Node.js 运行。
......@@ -157,8 +157,8 @@
"content/openapi/intro.mdx": "2026-04-26T21:08:47+08:00",
"content/openapi/share.en.mdx": "2026-04-26T21:08:47+08:00",
"content/openapi/share.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/config/env.en.mdx": "2026-05-23T11:22:44+08:00",
"content/self-host/config/env.mdx": "2026-05-23T11:22:44+08:00",
"content/self-host/config/env.en.mdx": "2026-05-23T22:47:02+08:00",
"content/self-host/config/env.mdx": "2026-05-23T22:47:02+08:00",
"content/self-host/config/json.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/config/json.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/config/model/intro.en.mdx": "2026-05-07T15:06:40+08:00",
......@@ -234,7 +234,7 @@
"content/self-host/upgrading/4-14/4141.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/4141.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/41410.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/41410.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/41410.mdx": "2026-05-23T22:47:02+08:00",
"content/self-host/upgrading/4-14/41411.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/41411.mdx": "2026-04-26T21:28:27+08:00",
"content/self-host/upgrading/4-14/41412.en.mdx": "2026-04-26T21:08:47+08:00",
......@@ -255,10 +255,10 @@
"content/self-host/upgrading/4-14/4142.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/41420.en.mdx": "2026-05-21T15:39:08+08:00",
"content/self-host/upgrading/4-14/41420.mdx": "2026-05-19T14:44:21+08:00",
"content/self-host/upgrading/4-14/41421.en.mdx": "2026-05-22T18:20:28+08:00",
"content/self-host/upgrading/4-14/41421.mdx": "2026-05-22T18:20:28+08:00",
"content/self-host/upgrading/4-14/41422.en.mdx": "2026-05-22T18:20:28+08:00",
"content/self-host/upgrading/4-14/41422.mdx": "2026-05-22T18:20:28+08:00",
"content/self-host/upgrading/4-14/41421.en.mdx": "2026-05-23T22:47:02+08:00",
"content/self-host/upgrading/4-14/41421.mdx": "2026-05-23T22:47:02+08:00",
"content/self-host/upgrading/4-14/41422.en.mdx": "2026-05-23T22:47:02+08:00",
"content/self-host/upgrading/4-14/41422.mdx": "2026-05-23T22:47:02+08:00",
"content/self-host/upgrading/4-14/4143.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/4143.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/4144.en.mdx": "2026-04-26T21:08:47+08:00",
......@@ -278,7 +278,7 @@
"content/self-host/upgrading/4-14/4149.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/4149.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-15/4150.mdx": "2026-05-20T17:52:26+08:00",
"content/self-host/upgrading/4-15/41502.mdx": "2026-05-22T21:07:00+08:00",
"content/self-host/upgrading/4-15/41502.mdx": "2026-05-23T22:47:02+08:00",
"content/self-host/upgrading/outdated/40.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/40.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/41.en.mdx": "2026-04-26T21:08:47+08:00",
......@@ -419,6 +419,6 @@
"content/self-host/upgrading/outdated/499.mdx": "2026-05-07T15:06:40+08:00",
"content/self-host/upgrading/upgrade-intruction.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/upgrade-intruction.mdx": "2026-04-26T21:08:47+08:00",
"content/toc.en.mdx": "2026-05-22T18:20:28+08:00",
"content/toc.mdx": "2026-05-22T18:20:28+08:00"
"content/toc.en.mdx": "2026-05-23T22:47:02+08:00",
"content/toc.mdx": "2026-05-23T22:47:02+08:00"
}
\ No newline at end of file
......@@ -1460,8 +1460,8 @@ importers:
specifier: ^2.30.1
version: 2.30.1
qs:
specifier: ^6.13.1
version: 6.15.1
specifier: ^6.15.2
version: 6.15.2
tiktoken:
specifier: 1.0.17
version: 1.0.17
......@@ -1616,6 +1616,37 @@ importers:
specifier: ^5.0.1
version: 5.0.1
projects/volume-manager:
dependencies:
'@hono/node-server':
specifier: ^1.13.7
version: 1.19.9(hono@4.12.18)
hono:
specifier: ^4.12.18
version: 4.12.18
undici:
specifier: ^7.25.0
version: 7.25.0
zod:
specifier: 'catalog:'
version: 4.1.12
devDependencies:
'@types/node':
specifier: 'catalog:'
version: 20.17.24
tsdown:
specifier: 'catalog:'
version: 0.21.10(typescript@5.9.3)
tsx:
specifier: 'catalog:'
version: 4.20.6
typescript:
specifier: 'catalog:'
version: 5.9.3
vitest:
specifier: 'catalog:'
version: 4.1.5(@opentelemetry/api@1.9.0)(@types/node@20.17.24)(@vitest/coverage-v8@4.1.5)(jsdom@26.1.0(bufferutil@4.1.0)(canvas@3.2.3)(utf-8-validate@5.0.10))(vite@6.2.2(@types/node@20.17.24)(jiti@2.7.0)(lightningcss@1.32.0)(sass@1.85.1)(terser@5.39.0)(tsx@4.20.6)(yaml@2.8.4))
scripts/icon:
dependencies:
express:
......@@ -12449,6 +12480,10 @@ packages:
resolution: {integrity: sha512-6YHEFRL9mfgcAvql/XhwTvf5jKcOiiupt2FiJxHkiX1z4j7WL8J/jRHYLluORvc1XxB5rV20KoeK00gVJamspg==}
engines: {node: '>=0.6'}
qs@6.15.2:
resolution: {integrity: sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==}
engines: {node: '>=0.6'}
qs@6.5.3:
resolution: {integrity: sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==}
engines: {node: '>=0.6'}
......@@ -21970,7 +22005,7 @@ snapshots:
mime: 2.6.0
platform: 1.3.6
pump: 3.0.2
qs: 6.15.1
qs: 6.15.2
sdk-base: 2.0.1
stream-http: 2.8.2
stream-wormhole: 1.1.0
......@@ -22380,7 +22415,7 @@ snapshots:
http-errors: 2.0.1
iconv-lite: 0.7.2
on-finished: 2.4.1
qs: 6.15.1
qs: 6.15.2
raw-body: 3.0.2
type-is: 2.0.1
transitivePeerDependencies:
......@@ -24336,7 +24371,7 @@ snapshots:
once: 1.4.0
parseurl: 1.3.3
proxy-addr: 2.0.7
qs: 6.15.1
qs: 6.15.2
range-parser: 1.2.1
router: 2.2.0
send: 1.2.1
......@@ -28484,6 +28519,10 @@ snapshots:
dependencies:
side-channel: 1.1.0
qs@6.15.2:
dependencies:
side-channel: 1.1.0
qs@6.5.3: {}
quansync@1.0.0: {}
......@@ -30825,7 +30864,7 @@ snapshots:
humanize-ms: 1.2.1
iconv-lite: 0.6.3
pump: 3.0.2
qs: 6.15.1
qs: 6.15.2
statuses: 1.5.0
utility: 1.18.0
optionalDependencies:
......@@ -30836,7 +30875,7 @@ snapshots:
form-data: 4.0.5
formstream: 1.5.2
mime-types: 2.1.35
qs: 6.15.1
qs: 6.15.2
type-fest: 4.41.0
undici: 7.25.0
ylru: 2.0.0
......@@ -5,6 +5,7 @@ packages:
- projects/code-sandbox
- projects/marketplace
- projects/mcp_server
- projects/volume-manager
- pro/admin
- pro/sso
......
Subproject commit d0eeadc2c023da2478caa0448faa8e5f92e59b3e
Subproject commit 128dc15d02ab52ce55a791c05cf6041cddd98271
......@@ -31,7 +31,7 @@
"hono": "^4.12.18",
"lodash": "catalog:",
"moment": "^2.30.1",
"qs": "^6.13.1",
"qs": "^6.15.2",
"tiktoken": "1.0.17",
"uuid": "^9.0.1",
"zod": "catalog:"
......
# Build:
# docker build -t fastgpt-volume-manager:latest .
# Build from repository root:
# docker build -f projects/volume-manager/Dockerfile -t fastgpt-volume-manager:latest .
FROM oven/bun:1.3-alpine
FROM node:24-alpine AS builder
WORKDIR /app
COPY package.json ./
RUN bun install --frozen-lockfile
ARG proxy
COPY . .
RUN npm install -g pnpm@10.33.2
COPY pnpm-lock.yaml pnpm-workspace.yaml package.json tsconfig.json ./
COPY projects/volume-manager/ ./projects/volume-manager/
RUN if [ -z "$proxy" ]; then \
pnpm install --frozen-lockfile --ignore-scripts --filter @fastgpt/volume-manager...; \
else \
pnpm install --frozen-lockfile --ignore-scripts --filter @fastgpt/volume-manager... --registry=https://registry.npmmirror.com; \
fi
RUN cd /app/projects/volume-manager && pnpm build
FROM node:24-alpine AS runner
WORKDIR /app
ENV NODE_ENV=production
COPY --from=builder /app/projects/volume-manager/dist ./dist
EXPOSE 3000
CMD ["bun", "src/index.ts"]
CMD ["node", "dist/index.js"]
#!/bin/bash
set -e
echo "Building volume-manager..."
# 清理旧的构建产物,避免历史文件被复制进镜像。
rm -rf dist
# 与 code-sandbox 一致:依赖整体打进单入口 bundle,runner 阶段只复制 dist。
echo "Building entries..."
pnpm exec tsdown
# package.json 已声明 type:module,直接改后缀为 .js。
mv dist/index.mjs dist/index.js
echo ""
echo "Build complete!"
echo " - index.js: $(du -h dist/index.js | cut -f1)"
......@@ -4,17 +4,26 @@
"private": true,
"type": "module",
"scripts": {
"dev": "bun --watch src/index.ts",
"build": "bun build src/index.ts --outdir dist --target bun",
"start": "bun dist/index.js",
"test": "vitest --run"
"dev": "tsx watch src/index.ts",
"build": "sh build.sh",
"start": "node dist/index.js",
"test": "vitest run"
},
"engines": {
"node": ">=20.19.0",
"pnpm": "10.x"
},
"dependencies": {
"hono": "^4.6.0",
"@hono/node-server": "^1.13.7",
"hono": "^4.12.18",
"undici": "^7.25.0",
"zod": "catalog:"
},
"devDependencies": {
"@types/bun": "latest",
"@types/node": "catalog:",
"tsdown": "catalog:",
"tsx": "catalog:",
"typescript": "catalog:",
"vitest": "catalog:"
}
}
import { Agent } from 'undici';
import type { IVolumeDriver, EnsureResult } from './IVolumeDriver';
import { toVolumeName } from '../utils/naming';
import { env } from '../env';
......@@ -6,19 +7,20 @@ import { logDebug } from '../utils/logger';
export class DockerVolumeDriver implements IVolumeDriver {
private readonly socketPath: string;
private readonly prefix: string;
private readonly dispatcher: Agent;
constructor(socketPath = env.VM_DOCKER_SOCKET, prefix = env.VM_VOLUME_NAME_PREFIX) {
this.socketPath = socketPath;
this.prefix = prefix;
this.dispatcher = new Agent({ connect: { socketPath } });
}
private dockerFetch(path: string, init?: RequestInit): Promise<Response> {
// Bun supports unix socket via the `unix` fetch option
// Node fetch 通过 Undici dispatcher 访问 Docker Unix socket。
return fetch(`http://localhost/${env.VM_DOCKER_API_VERSION}${path}`, {
...init,
// @ts-ignore - Bun-specific option
unix: this.socketPath
});
dispatcher: this.dispatcher
} as RequestInit & { dispatcher: Agent });
}
async ensure(sessionId: string): Promise<EnsureResult> {
......
import { readFileSync } from 'fs';
import { Agent } from 'undici';
import type { IVolumeDriver, EnsureResult } from './IVolumeDriver';
import { toVolumeName } from '../utils/naming';
import { env } from '../env';
......@@ -12,10 +13,6 @@ function readToken(): string {
return readFileSync(TOKEN_PATH, 'utf-8').trim();
}
function fetchOpts(extra: RequestInit = {}): RequestInit {
return { ...extra, tls: { ca: readFileSync(CA_PATH, 'utf-8') } } as RequestInit;
}
function pvcBody(name: string, sessionId: string): object {
return {
apiVersion: 'v1',
......@@ -36,12 +33,19 @@ function pvcBody(name: string, sessionId: string): object {
export class K8sVolumeDriver implements IVolumeDriver {
private readonly namespace: string;
private readonly prefix: string;
private dispatcher?: Agent;
constructor(namespace = env.VM_K8S_NAMESPACE, prefix = env.VM_VOLUME_NAME_PREFIX) {
this.namespace = namespace;
this.prefix = prefix;
}
private fetchOpts(extra: RequestInit = {}): RequestInit & { dispatcher: Agent } {
// Kubernetes in-cluster CA 是服务账号挂载文件,按实例懒加载,避免模块初始化依赖运行环境。
this.dispatcher ??= new Agent({ connect: { ca: readFileSync(CA_PATH, 'utf-8') } });
return { ...extra, dispatcher: this.dispatcher } as RequestInit & { dispatcher: Agent };
}
private headers(): Record<string, string> {
return {
Authorization: `Bearer ${readToken()}`,
......@@ -60,7 +64,7 @@ export class K8sVolumeDriver implements IVolumeDriver {
const getUrl = this.pvcUrl(name);
logDebug(`K8s GET PVC url=${getUrl}`);
const getRes = await fetch(getUrl, fetchOpts({ headers: this.headers() }));
const getRes = await fetch(getUrl, this.fetchOpts({ headers: this.headers() }));
logDebug(`K8s GET PVC status=${getRes.status}`);
if (getRes.ok) {
......@@ -76,7 +80,7 @@ export class K8sVolumeDriver implements IVolumeDriver {
logDebug(`K8s POST PVC url=${postUrl} name=${name}`);
const createRes = await fetch(
postUrl,
fetchOpts({
this.fetchOpts({
method: 'POST',
headers: this.headers(),
body: JSON.stringify(pvcBody(name, sessionId))
......@@ -99,7 +103,7 @@ export class K8sVolumeDriver implements IVolumeDriver {
logDebug(`K8s DELETE PVC url=${delUrl}`);
const res = await fetch(
delUrl,
fetchOpts({
this.fetchOpts({
method: 'DELETE',
headers: this.headers()
})
......
import { serve } from '@hono/node-server';
import { Hono } from 'hono';
import { env } from './env';
import { DockerVolumeDriver } from './drivers/DockerVolumeDriver';
......@@ -26,9 +27,9 @@ app.use('/v1/*', async (c, next) => {
// Volume routes
app.route('/v1/volumes', volumeRoutes(service));
const server = Bun.serve({
serve({
port: env.PORT,
fetch: app.fetch
});
console.log(`[volume-manager] Listening on port ${server.port} (runtime: ${env.VM_RUNTIME})`);
console.log(`[volume-manager] Listening on port ${env.PORT} (runtime: ${env.VM_RUNTIME})`);
......@@ -29,6 +29,8 @@ describe('DockerVolumeDriver', () => {
const driver = new DockerVolumeDriver();
const result = await driver.ensure(VALID_ID);
expect(result).toEqual({ claimName: VOLUME_NAME, created: false });
const [, opts] = fetchMock.mock.calls[0];
expect((opts as any).dispatcher).toBeTruthy();
});
it('ensure creates volume on 404', async () => {
......
......@@ -57,13 +57,18 @@ describe('K8sVolumeDriver', () => {
await expect(driver.ensure(VALID_ID)).rejects.toThrow('403');
});
it('fetch calls include tls.ca from ca.crt', async () => {
it('fetch calls use an Undici dispatcher with ca.crt loaded', async () => {
fetchMock.mockResolvedValueOnce({ ok: true, status: 200 });
const { K8sVolumeDriver } = await import('../../src/drivers/K8sVolumeDriver');
const { readFileSync } = await import('fs');
const driver = new K8sVolumeDriver();
await driver.ensure(VALID_ID);
const [, opts] = fetchMock.mock.calls[0];
expect((opts as any).tls?.ca).toBe('mock-ca-cert');
expect((opts as any).dispatcher).toBeTruthy();
expect(readFileSync).toHaveBeenCalledWith(
'/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
'utf-8'
);
});
it('remove treats 404 as success', async () => {
......
......@@ -5,7 +5,7 @@
"moduleResolution": "bundler",
"strict": true,
"skipLibCheck": true,
"types": ["bun-types"]
"types": ["node"]
},
"include": ["src", "test"]
}
import { defineConfig } from 'tsdown';
export default defineConfig({
entry: 'src/index.ts',
format: 'esm',
platform: 'node',
target: 'node20',
minify: true,
outDir: 'dist',
deps: {
alwaysBundle: [/.*/],
onlyBundle: false
}
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment