Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 64f70a41 by Archer Committed by GitHub
Options

feat: vector integrationTest;feat: ob quantization (#6366) 

* feat(vectordb): add OceanBase HNSW quantization (HNSW_SQ/HNSW_BQ) (#6348)

Support OceanBase vector index quantization via VECTOR_VQ_LEVEL:
- 32 (default): hnsw + inner_product
- 8: hnsw_sq + inner_product (2-3x memory savings)
- 1: hnsw_bq + cosine (~15x memory savings)

HNSW_BQ requires cosine distance per OceanBase docs.
Tested on OceanBase 4.3.5.5 (BP5).

Closes #6202

* feat: add test inclusion for vectorDB tests in vitest configuration (#6358)

* feat: add test inclusion for vectorDB tests in vitest configuration

* refactor: update vectorDB README and setup for environment configuration

- Enhanced README to clarify the use of factory pattern for vectorDB integration tests.
- Updated instructions for setting up environment variables from a local file.
- Removed obsolete PG integration test file and adjusted test execution instructions.
- Improved structure explanation for shared test data and factory functions.

* perf: integrationTest

* feat: vector integration

---------

Co-authored-by: ZHANG Yixin <hi.yixinz@gmail.com>
Co-authored-by: Jingchao <alswlx@gmail.com>
parent 358109f5
Showing with 761 additions and 163 deletions
deploy/docker/cn/docker-compose.oceanbase.yml
...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config ...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config
# 向量库相关配置 # 向量库相关配置
x-vec-config: &x-vec-config x-vec-config: &x-vec-config
OCEANBASE_URL: mysql://root%40tenantname:tenantpassword@ob:2881/test OCEANBASE_URL: mysql://root%40tenantname:tenantpassword@ob:2881/mysql
version: '3.3' version: '3.3'
...@@ -52,8 +52,8 @@ services: ...@@ -52,8 +52,8 @@ services:
- OB_SERVER_IP=127.0.0.1 - OB_SERVER_IP=127.0.0.1
# 更多环境变量配置见oceanbase官方文档： https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000002013494 # 更多环境变量配置见oceanbase官方文档： https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000002013494
volumes: volumes:
- ../ob/data:/root/ob - ./ob/data:/root/ob
- ../ob/config:/root/.obd/cluster - ./ob/config:/root/.obd/cluster
configs: configs:
- source: init_sql - source: init_sql
target: /root/boot/init.d/init.sql target: /root/boot/init.d/init.sql
......
deploy/docker/cn/docker-compose.seekdb.yml
...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config ...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config
# 向量库相关配置 # 向量库相关配置
x-vec-config: &x-vec-config x-vec-config: &x-vec-config
SEEKDB_URL: mysql://root%40tenantname:tenantpassword@seekdb:3306/fastgpt SEEKDB_URL: mysql://root:seekdbpassword@seekdb:2881/mysql
version: '3.3' version: '3.3'
...@@ -37,23 +37,20 @@ services: ...@@ -37,23 +37,20 @@ services:
container_name: seekdb container_name: seekdb
restart: always restart: always
# ports: # 生产环境建议不要暴露 # ports: # 生产环境建议不要暴露
# - 3306:3306 # - 2881:2881
# - 2886:2886
networks: networks:
- fastgpt - fastgpt
environment: environment:
# SeekDB 连接配置（兼容 MySQL 协议） # SeekDB 连接配置（兼容 MySQL 协议）
- MYSQL_ROOT_PASSWORD=seekdbpassword - ROOT_PASSWORD=seekdbpassword
# SeekDB 租户配置（与 OceanBase 兼容）
- OB_TENANT_NAME=tenantname
- OB_TENANT_PASSWORD=tenantpassword
# MODE分为MINI和NORMAL， 后者会最大程度使用主机资源 # MODE分为MINI和NORMAL， 后者会最大程度使用主机资源
- MODE=MINI - MODE=MINI
- OB_SERVER_IP=127.0.0.1
volumes: volumes:
- ../seekdb/data:/var/lib/mysql - ./seekdb/data:/var/lib/mysql
- ../seekdb/config:/etc/mysql/conf.d - ./seekdb/config:/etc/mysql/conf.d
healthcheck: healthcheck:
test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] test: ['CMD', 'mysqladmin', 'ping', '-h', '127.0.0.1', '-P2881', '-uroot', '-pseekdbpassword']
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 1000 retries: 1000
......
deploy/docker/global/docker-compose.oceanbase.yml
...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config ...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config
# 向量库相关配置 # 向量库相关配置
x-vec-config: &x-vec-config x-vec-config: &x-vec-config
OCEANBASE_URL: mysql://root%40tenantname:tenantpassword@ob:2881/test OCEANBASE_URL: mysql://root%40tenantname:tenantpassword@ob:2881/mysql
version: '3.3' version: '3.3'
...@@ -52,8 +52,8 @@ services: ...@@ -52,8 +52,8 @@ services:
- OB_SERVER_IP=127.0.0.1 - OB_SERVER_IP=127.0.0.1
# 更多环境变量配置见oceanbase官方文档： https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000002013494 # 更多环境变量配置见oceanbase官方文档： https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000002013494
volumes: volumes:
- ../ob/data:/root/ob - ./ob/data:/root/ob
- ../ob/config:/root/.obd/cluster - ./ob/config:/root/.obd/cluster
configs: configs:
- source: init_sql - source: init_sql
target: /root/boot/init.d/init.sql target: /root/boot/init.d/init.sql
......
deploy/docker/global/docker-compose.seekdb.yml
...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config ...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config
# 向量库相关配置 # 向量库相关配置
x-vec-config: &x-vec-config x-vec-config: &x-vec-config
SEEKDB_URL: mysql://root%40tenantname:tenantpassword@seekdb:3306/fastgpt SEEKDB_URL: mysql://root:seekdbpassword@seekdb:2881/mysql
version: '3.3' version: '3.3'
...@@ -37,23 +37,20 @@ services: ...@@ -37,23 +37,20 @@ services:
container_name: seekdb container_name: seekdb
restart: always restart: always
# ports: # 生产环境建议不要暴露 # ports: # 生产环境建议不要暴露
# - 3306:3306 # - 2881:2881
# - 2886:2886
networks: networks:
- fastgpt - fastgpt
environment: environment:
# SeekDB 连接配置（兼容 MySQL 协议） # SeekDB 连接配置（兼容 MySQL 协议）
- MYSQL_ROOT_PASSWORD=seekdbpassword - ROOT_PASSWORD=seekdbpassword
# SeekDB 租户配置（与 OceanBase 兼容）
- OB_TENANT_NAME=tenantname
- OB_TENANT_PASSWORD=tenantpassword
# MODE分为MINI和NORMAL， 后者会最大程度使用主机资源 # MODE分为MINI和NORMAL， 后者会最大程度使用主机资源
- MODE=MINI - MODE=MINI
- OB_SERVER_IP=127.0.0.1
volumes: volumes:
- ../seekdb/data:/var/lib/mysql - ./seekdb/data:/var/lib/mysql
- ../seekdb/config:/etc/mysql/conf.d - ./seekdb/config:/etc/mysql/conf.d
healthcheck: healthcheck:
test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] test: ['CMD', 'mysqladmin', 'ping', '-h', '127.0.0.1', '-P2881', '-uroot', '-pseekdbpassword']
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 1000 retries: 1000
......
deploy/init.mjs
...@@ -76,7 +76,7 @@ const vector = { ...@@ -76,7 +76,7 @@ const vector = {
ob: { ob: {
db: '', db: '',
config: `\ config: `\
OCEANBASE_URL: mysql://root%40tenantname:tenantpassword@ob:2881/test OCEANBASE_URL: mysql://root%40tenantname:tenantpassword@ob:2881/mysql
`, `,
extra: `\ extra: `\
configs: configs:
...@@ -89,7 +89,7 @@ configs: ...@@ -89,7 +89,7 @@ configs:
seekdb: { seekdb: {
db: '', db: '',
config: `\ config: `\
SEEKDB_URL: mysql://root%40tenantname:tenantpassword@seekdb:3306/fastgpt SEEKDB_URL: mysql://root:seekdbpassword@seekdb:2881/mysql
`, `,
extra: `` extra: ``
}, },
......
deploy/templates/vector/ob.txt
...@@ -18,8 +18,8 @@ ...@@ -18,8 +18,8 @@
- OB_SERVER_IP=127.0.0.1 - OB_SERVER_IP=127.0.0.1
# 更多环境变量配置见oceanbase官方文档： https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000002013494 # 更多环境变量配置见oceanbase官方文档： https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000002013494
volumes: volumes:
- ../ob/data:/root/ob - ./ob/data:/root/ob
- ../ob/config:/root/.obd/cluster - ./ob/config:/root/.obd/cluster
configs: configs:
- source: init_sql - source: init_sql
target: /root/boot/init.d/init.sql target: /root/boot/init.d/init.sql
......
deploy/templates/vector/seekdb.txt
...@@ -3,23 +3,20 @@ ...@@ -3,23 +3,20 @@
container_name: seekdb container_name: seekdb
restart: always restart: always
# ports: # 生产环境建议不要暴露 # ports: # 生产环境建议不要暴露
# - 3306:3306 # - 2881:2881
# - 2886:2886
networks: networks:
- fastgpt - fastgpt
environment: environment:
# SeekDB 连接配置（兼容 MySQL 协议） # SeekDB 连接配置（兼容 MySQL 协议）
- MYSQL_ROOT_PASSWORD=seekdbpassword - ROOT_PASSWORD=seekdbpassword
# SeekDB 租户配置（与 OceanBase 兼容）
- OB_TENANT_NAME=tenantname
- OB_TENANT_PASSWORD=tenantpassword
# MODE分为MINI和NORMAL， 后者会最大程度使用主机资源 # MODE分为MINI和NORMAL， 后者会最大程度使用主机资源
- MODE=MINI - MODE=MINI
- OB_SERVER_IP=127.0.0.1
volumes: volumes:
- ../seekdb/data:/var/lib/mysql - ./seekdb/data:/var/lib/mysql
- ../seekdb/config:/etc/mysql/conf.d - ./seekdb/config:/etc/mysql/conf.d
healthcheck: healthcheck:
test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] test: ['CMD', 'mysqladmin', 'ping', '-h', '127.0.0.1', '-P2881', '-uroot', '-pseekdbpassword']
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 1000 retries: 1000
......
document/content/docs/toc.mdx
...@@ -121,6 +121,7 @@ description: FastGPT 文档目录 ...@@ -121,6 +121,7 @@ description: FastGPT 文档目录
- [/docs/upgrading/4-14/4145](/docs/upgrading/4-14/4145) - [/docs/upgrading/4-14/4145](/docs/upgrading/4-14/4145)
- [/docs/upgrading/4-14/41451](/docs/upgrading/4-14/41451) - [/docs/upgrading/4-14/41451](/docs/upgrading/4-14/41451)
- [/docs/upgrading/4-14/4146](/docs/upgrading/4-14/4146) - [/docs/upgrading/4-14/4146](/docs/upgrading/4-14/4146)
- [/docs/upgrading/4-14/4147](/docs/upgrading/4-14/4147)
- [/docs/upgrading/4-8/40](/docs/upgrading/4-8/40) - [/docs/upgrading/4-8/40](/docs/upgrading/4-8/40)
- [/docs/upgrading/4-8/41](/docs/upgrading/4-8/41) - [/docs/upgrading/4-8/41](/docs/upgrading/4-8/41)
- [/docs/upgrading/4-8/42](/docs/upgrading/4-8/42) - [/docs/upgrading/4-8/42](/docs/upgrading/4-8/42)
......
document/content/docs/upgrading/4-14/4147.mdx 0 → 100644
---
title: 'V4.14.7(进行中)'
description: 'FastGPT V4.14.7 更新说明'
---
## 🚀 新增内容
## ⚙️ 优化
## 🐛 修复
## 插件
## 代码质量
1. 增加向量数据库集成测试。
\ No newline at end of file
document/data/doc-last-modified.json
...@@ -124,7 +124,7 @@ ...@@ -124,7 +124,7 @@
"document/content/docs/upgrading/4-14/4144.mdx": "2025-12-16T14:56:04+08:00", "document/content/docs/upgrading/4-14/4144.mdx": "2025-12-16T14:56:04+08:00",
"document/content/docs/upgrading/4-14/4145.mdx": "2026-01-18T23:59:15+08:00", "document/content/docs/upgrading/4-14/4145.mdx": "2026-01-18T23:59:15+08:00",
"document/content/docs/upgrading/4-14/41451.mdx": "2026-01-20T11:53:27+08:00", "document/content/docs/upgrading/4-14/41451.mdx": "2026-01-20T11:53:27+08:00",
"document/content/docs/upgrading/4-14/4146.mdx": "2026-01-30T18:47:43+08:00", "document/content/docs/upgrading/4-14/4146.mdx": "2026-01-30T22:34:42+08:00",
"document/content/docs/upgrading/4-8/40.mdx": "2025-08-02T19:38:37+08:00", "document/content/docs/upgrading/4-8/40.mdx": "2025-08-02T19:38:37+08:00",
"document/content/docs/upgrading/4-8/41.mdx": "2025-08-02T19:38:37+08:00", "document/content/docs/upgrading/4-8/41.mdx": "2025-08-02T19:38:37+08:00",
"document/content/docs/upgrading/4-8/42.mdx": "2025-08-02T19:38:37+08:00", "document/content/docs/upgrading/4-8/42.mdx": "2025-08-02T19:38:37+08:00",
......
document/public/deploy/docker/cn/docker-compose.oceanbase.yml
...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config ...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config
# 向量库相关配置 # 向量库相关配置
x-vec-config: &x-vec-config x-vec-config: &x-vec-config
OCEANBASE_URL: mysql://root%40tenantname:tenantpassword@ob:2881/test OCEANBASE_URL: mysql://root%40tenantname:tenantpassword@ob:2881/mysql
version: '3.3' version: '3.3'
...@@ -52,8 +52,8 @@ services: ...@@ -52,8 +52,8 @@ services:
- OB_SERVER_IP=127.0.0.1 - OB_SERVER_IP=127.0.0.1
# 更多环境变量配置见oceanbase官方文档： https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000002013494 # 更多环境变量配置见oceanbase官方文档： https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000002013494
volumes: volumes:
- ../ob/data:/root/ob - ./ob/data:/root/ob
- ../ob/config:/root/.obd/cluster - ./ob/config:/root/.obd/cluster
configs: configs:
- source: init_sql - source: init_sql
target: /root/boot/init.d/init.sql target: /root/boot/init.d/init.sql
......
document/public/deploy/docker/cn/docker-compose.seekdb.yml
...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config ...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config
# 向量库相关配置 # 向量库相关配置
x-vec-config: &x-vec-config x-vec-config: &x-vec-config
SEEKDB_URL: mysql://root%40tenantname:tenantpassword@seekdb:3306/fastgpt SEEKDB_URL: mysql://root:seekdbpassword@seekdb:2881/mysql
version: '3.3' version: '3.3'
...@@ -37,23 +37,20 @@ services: ...@@ -37,23 +37,20 @@ services:
container_name: seekdb container_name: seekdb
restart: always restart: always
# ports: # 生产环境建议不要暴露 # ports: # 生产环境建议不要暴露
# - 3306:3306 # - 2881:2881
# - 2886:2886
networks: networks:
- fastgpt - fastgpt
environment: environment:
# SeekDB 连接配置（兼容 MySQL 协议） # SeekDB 连接配置（兼容 MySQL 协议）
- MYSQL_ROOT_PASSWORD=seekdbpassword - ROOT_PASSWORD=seekdbpassword
# SeekDB 租户配置（与 OceanBase 兼容）
- OB_TENANT_NAME=tenantname
- OB_TENANT_PASSWORD=tenantpassword
# MODE分为MINI和NORMAL， 后者会最大程度使用主机资源 # MODE分为MINI和NORMAL， 后者会最大程度使用主机资源
- MODE=MINI - MODE=MINI
- OB_SERVER_IP=127.0.0.1
volumes: volumes:
- ../seekdb/data:/var/lib/mysql - ./seekdb/data:/var/lib/mysql
- ../seekdb/config:/etc/mysql/conf.d - ./seekdb/config:/etc/mysql/conf.d
healthcheck: healthcheck:
test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] test: ['CMD', 'mysqladmin', 'ping', '-h', '127.0.0.1', '-P2881', '-uroot', '-pseekdbpassword']
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 1000 retries: 1000
......
document/public/deploy/docker/global/docker-compose.oceanbase.yml
...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config ...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config
# 向量库相关配置 # 向量库相关配置
x-vec-config: &x-vec-config x-vec-config: &x-vec-config
OCEANBASE_URL: mysql://root%40tenantname:tenantpassword@ob:2881/test OCEANBASE_URL: mysql://root%40tenantname:tenantpassword@ob:2881/mysql
version: '3.3' version: '3.3'
...@@ -52,8 +52,8 @@ services: ...@@ -52,8 +52,8 @@ services:
- OB_SERVER_IP=127.0.0.1 - OB_SERVER_IP=127.0.0.1
# 更多环境变量配置见oceanbase官方文档： https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000002013494 # 更多环境变量配置见oceanbase官方文档： https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000002013494
volumes: volumes:
- ../ob/data:/root/ob - ./ob/data:/root/ob
- ../ob/config:/root/.obd/cluster - ./ob/config:/root/.obd/cluster
configs: configs:
- source: init_sql - source: init_sql
target: /root/boot/init.d/init.sql target: /root/boot/init.d/init.sql
......
document/public/deploy/docker/global/docker-compose.seekdb.yml
...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config ...@@ -26,7 +26,7 @@ x-share-db-config: &x-share-db-config
# 向量库相关配置 # 向量库相关配置
x-vec-config: &x-vec-config x-vec-config: &x-vec-config
SEEKDB_URL: mysql://root%40tenantname:tenantpassword@seekdb:3306/fastgpt SEEKDB_URL: mysql://root:seekdbpassword@seekdb:2881/mysql
version: '3.3' version: '3.3'
...@@ -37,23 +37,20 @@ services: ...@@ -37,23 +37,20 @@ services:
container_name: seekdb container_name: seekdb
restart: always restart: always
# ports: # 生产环境建议不要暴露 # ports: # 生产环境建议不要暴露
# - 3306:3306 # - 2881:2881
# - 2886:2886
networks: networks:
- fastgpt - fastgpt
environment: environment:
# SeekDB 连接配置（兼容 MySQL 协议） # SeekDB 连接配置（兼容 MySQL 协议）
- MYSQL_ROOT_PASSWORD=seekdbpassword - ROOT_PASSWORD=seekdbpassword
# SeekDB 租户配置（与 OceanBase 兼容）
- OB_TENANT_NAME=tenantname
- OB_TENANT_PASSWORD=tenantpassword
# MODE分为MINI和NORMAL， 后者会最大程度使用主机资源 # MODE分为MINI和NORMAL， 后者会最大程度使用主机资源
- MODE=MINI - MODE=MINI
- OB_SERVER_IP=127.0.0.1
volumes: volumes:
- ../seekdb/data:/var/lib/mysql - ./seekdb/data:/var/lib/mysql
- ../seekdb/config:/etc/mysql/conf.d - ./seekdb/config:/etc/mysql/conf.d
healthcheck: healthcheck:
test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] test: ['CMD', 'mysqladmin', 'ping', '-h', '127.0.0.1', '-P2881', '-uroot', '-pseekdbpassword']
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 1000 retries: 1000
......
package.json
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
"create:i18n": "node ./scripts/i18n/index.js", "create:i18n": "node ./scripts/i18n/index.js",
"lint": "eslint \"**/*.{ts,tsx}\" --fix --ignore-path .eslintignore", "lint": "eslint \"**/*.{ts,tsx}\" --fix --ignore-path .eslintignore",
"test": "vitest run", "test": "vitest run",
"test:workflow": "vitest run workflow" "test:vector": "vitest run --config test/integrationTest/vectorDB/vitest.config.mts"
}, },
"devDependencies": { "devDependencies": {
"@chakra-ui/cli": "^2.4.1", "@chakra-ui/cli": "^2.4.1",
......
packages/service/common/vectorDB/constants.ts
...@@ -25,3 +25,58 @@ export const VectorVQ = (() => { ...@@ -25,3 +25,58 @@ export const VectorVQ = (() => {
} }
return 32; return 32;
})(); })();
/**
* OceanBase HNSW Index Configuration
*
* VECTOR_VQ_LEVEL mapping:
* - 32 (default): hnsw + inner_product
* - 8: hnsw_sq + inner_product
* - 1: hnsw_bq + cosine
*
* See https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000004920602
* for the recommended way of choosing parameters (`m`, `ef_construction`, `ef_search`). It varies for data volume.
*
* HNSW_BQ requires cosine or l2 distance. inner_product is not supported up until V4.3.5 BP5 (current lts version until Jan 2026).
* See https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000004920603
* `HNSW_BQ distance 参数支持 l2 和 cosine。cosine 从 V4.3.5 BP4 版本开始支持。` and section `距离函数使用规则`.
*
* Tested on OceanBase 4.3.5-lts:
* ```sql
* -- HNSW_BQ + cosine: VECTOR INDEX SCAN ✓
* CREATE VECTOR INDEX idx ON t(vec) WITH (distance=cosine, type=hnsw_bq, m=16, ef_construction=200);
* EXPLAIN SELECT id, cosine_distance(vec, '[...]') AS score FROM t ORDER BY score ASC APPROXIMATE LIMIT 10;
* -- |1 |└─VECTOR INDEX SCAN|t(idx)|
* ```
*/
export const OceanBaseIndexConfig = (() => {
const level = process.env.VECTOR_VQ_LEVEL;
if (level === '1') {
return {
type: 'hnsw_bq' as const,
distance: 'cosine' as const,
distanceFunc: 'cosine_distance',
orderDirection: 'ASC' as const,
scoreTransform: (score: number) => 1 - score / 2
};
}
if (level === '8') {
return {
type: 'hnsw_sq' as const,
distance: 'inner_product' as const,
distanceFunc: 'inner_product',
orderDirection: 'DESC' as const,
scoreTransform: (score: number) => score
};
}
return {
type: 'hnsw' as const,
distance: 'inner_product' as const,
distanceFunc: 'inner_product',
orderDirection: 'DESC' as const,
scoreTransform: (score: number) => score
};
})();
packages/service/common/vectorDB/controller.ts
...@@ -19,8 +19,8 @@ import { throttle } from 'lodash'; ...@@ -19,8 +19,8 @@ import { throttle } from 'lodash';
import { retryFn } from '@fastgpt/global/common/system/utils'; import { retryFn } from '@fastgpt/global/common/system/utils';
const getVectorObj = (): VectorControllerType => { const getVectorObj = (): VectorControllerType => {
if (SEEKDB_ADDRESS) return new SeekVectorCtrl(); if (SEEKDB_ADDRESS) return new SeekVectorCtrl({ type: 'seekdb' });
if (OCEANBASE_ADDRESS) return new ObVectorCtrl(); if (OCEANBASE_ADDRESS) return new ObVectorCtrl({ type: 'oceanbase' });
if (PG_ADDRESS) return new PgVectorCtrl(); if (PG_ADDRESS) return new PgVectorCtrl();
if (MILVUS_ADDRESS) return new MilvusCtrl(); if (MILVUS_ADDRESS) return new MilvusCtrl();
......
packages/service/common/vectorDB/milvus/index.ts
...@@ -220,24 +220,27 @@ export class MilvusCtrl implements VectorControllerType { ...@@ -220,24 +220,27 @@ export class MilvusCtrl implements VectorControllerType {
.filter((id) => !forbidCollectionIdList.includes(id)); .filter((id) => !forbidCollectionIdList.includes(id));
})(); })();
const collectionIdQuery = formatFilterCollectionId const collectionIdQuery = formatFilterCollectionId
? `and (collectionId in [${formatFilterCollectionId.map((id) => `"${id}"`)}])` ? `and (collectionId in [${formatFilterCollectionId.map((id) => `"${id}"`).join(',')}])`
: ``; : ``;
// Empty data // Empty data
if (formatFilterCollectionId && formatFilterCollectionId.length === 0) { if (formatFilterCollectionId && formatFilterCollectionId.length === 0) {
return { results: [] }; return { results: [] };
} }
const { results } = await retryFn(() => const filterStr =
`(teamId == "${teamId}") and (datasetId in [${datasetIds.map((id) => `"${id}"`).join(',')}]) ${collectionIdQuery} ${forbidColQuery}`.trim();
const searchResult = await retryFn(() =>
client.search({ client.search({
collection_name: DatasetVectorTableName, collection_name: DatasetVectorTableName,
data: vector, vector: vector,
limit, limit,
filter: `(teamId == "${teamId}") and (datasetId in [${datasetIds.map((id) => `"${id}"`).join(',')}]) ${collectionIdQuery} ${forbidColQuery}`, expr: filterStr,
output_fields: ['collectionId'] output_fields: ['collectionId']
}) })
); );
const rows = results as { const rows = (searchResult.results || []) as {
score: number; score: number;
id: string; id: string;
collectionId: string; collectionId: string;
...@@ -280,9 +283,9 @@ export class MilvusCtrl implements VectorControllerType { ...@@ -280,9 +283,9 @@ export class MilvusCtrl implements VectorControllerType {
filter: filter || undefined filter: filter || undefined
}); });
const total = result.data?.[0]?.['count(*)'] as number; const total = result.data?.[0]?.['count(*)'];
return total; return Number(total);
}; };
getVectorDataByTime: VectorControllerType['getVectorDataByTime'] = async (start, end) => { getVectorDataByTime: VectorControllerType['getVectorDataByTime'] = async (start, end) => {
......
packages/service/common/vectorDB/oceanbase/controller.ts
...@@ -5,47 +5,9 @@ import mysql, { ...@@ -5,47 +5,9 @@ import mysql, {
type ResultSetHeader type ResultSetHeader
} from 'mysql2/promise'; } from 'mysql2/promise';
import { addLog } from '../../system/log'; import { addLog } from '../../system/log';
import { OCEANBASE_ADDRESS } from '../constants'; import { OCEANBASE_ADDRESS, SEEKDB_ADDRESS } from '../constants';
import { delay } from '@fastgpt/global/common/system/utils'; import { delay } from '@fastgpt/global/common/system/utils';
export const getClient = async (): Promise<Pool> => {
if (!OCEANBASE_ADDRESS) {
return Promise.reject('OCEANBASE_ADDRESS is not set');
}
if (global.obClient) {
return global.obClient;
}
global.obClient = mysql.createPool({
uri: OCEANBASE_ADDRESS,
waitForConnections: true,
connectionLimit: Number(process.env.DB_MAX_LINK || 20),
connectTimeout: 20000,
idleTimeout: 60000,
queueLimit: 0,
enableKeepAlive: true,
keepAliveInitialDelay: 0
});
try {
// Test the connection with a simple query instead of calling connect()
await global.obClient.query('SELECT 1');
addLog.info(`[OceanBase] connect`);
return global.obClient;
} catch (error) {
addLog.error(`[OceanBase] connect error`, error);
global.obClient?.end();
global.obClient = null;
await delay(1000);
addLog.info(`[OceanBase] retry connect`);
return getClient();
}
};
type WhereProps = (string | [string, string | number])[]; type WhereProps = (string | [string, string | number])[];
type GetProps = { type GetProps = {
fields?: string[]; fields?: string[];
...@@ -68,7 +30,49 @@ type InsertProps = { ...@@ -68,7 +30,49 @@ type InsertProps = {
values: ValuesProps[]; values: ValuesProps[];
}; };
class ObClass { export class ObClass {
controllerType: 'oceanbase' | 'seekdb';
constructor({ type }: { type: 'oceanbase' | 'seekdb' }) {
this.controllerType = type;
}
private async getClient(): Promise<Pool> {
const address = this.controllerType === 'oceanbase' ? OCEANBASE_ADDRESS : SEEKDB_ADDRESS;
if (!address) {
return Promise.reject('OCEANBASE_ADDRESS || SEEKDB_ADDRESS is not set');
}
if (global.obClient) {
return global.obClient;
}
global.obClient = mysql.createPool({
uri: address,
waitForConnections: true,
connectionLimit: Number(process.env.DB_MAX_LINK || 20),
connectTimeout: 20000,
idleTimeout: 60000,
queueLimit: 0,
enableKeepAlive: true,
keepAliveInitialDelay: 0
});
try {
// Test the connection with a simple query instead of calling connect()
await global.obClient.query('SELECT 1');
addLog.info(`[${this.controllerType}] connect`);
return global.obClient;
} catch (error) {
addLog.error(`[${this.controllerType}] connect error`, error);
global.obClient?.end();
global.obClient = null;
await delay(1000);
addLog.info(`[${this.controllerType}] retry connect`);
return this.getClient();
}
}
private getWhereStr(where?: WhereProps) { private getWhereStr(where?: WhereProps) {
return where return where
? `WHERE ${where ? `WHERE ${where
...@@ -122,7 +126,7 @@ class ObClass { ...@@ -122,7 +126,7 @@ class ObClass {
LIMIT ${props.limit || 10} OFFSET ${props.offset || 0} LIMIT ${props.limit || 10} OFFSET ${props.offset || 0}
`; `;
const client = await getClient(); const client = await this.getClient();
return client.query<T>(sql); return client.query<T>(sql);
} }
async count(table: string, props: GetProps) { async count(table: string, props: GetProps) {
...@@ -131,14 +135,14 @@ class ObClass { ...@@ -131,14 +135,14 @@ class ObClass {
${this.getWhereStr(props.where)} ${this.getWhereStr(props.where)}
`; `;
const client = await getClient(); const client = await this.getClient();
return client.query<({ count: number } & RowDataPacket)[]>(sql).then(([res]) => { return client.query<({ count: number } & RowDataPacket)[]>(sql).then(([res]) => {
return res[0]?.['COUNT(*)'] || 0; return res[0]?.['COUNT(*)'] || 0;
}); });
} }
async delete(table: string, props: DeleteProps) { async delete(table: string, props: DeleteProps) {
const sql = `DELETE FROM ${table} ${this.getWhereStr(props.where)}`; const sql = `DELETE FROM ${table} ${this.getWhereStr(props.where)}`;
const client = await getClient(); const client = await this.getClient();
return client.query(sql); return client.query(sql);
} }
async update(table: string, props: UpdateProps) { async update(table: string, props: UpdateProps) {
...@@ -151,7 +155,7 @@ class ObClass { ...@@ -151,7 +155,7 @@ class ObClass {
const sql = `UPDATE ${table} SET ${this.getUpdateValStr(props.values)} ${this.getWhereStr( const sql = `UPDATE ${table} SET ${this.getUpdateValStr(props.values)} ${this.getWhereStr(
props.where props.where
)}`; )}`;
const client = await getClient(); const client = await this.getClient();
return client.query(sql); return client.query(sql);
} }
/** /**
...@@ -175,7 +179,7 @@ class ObClass { ...@@ -175,7 +179,7 @@ class ObClass {
const sql = `INSERT INTO ${table} (${fields}) VALUES ${this.getInsertValStr(props.values)}`; const sql = `INSERT INTO ${table} (${fields}) VALUES ${this.getInsertValStr(props.values)}`;
// 获取专用连接而不是从连接池获取 // 获取专用连接而不是从连接池获取
const connection = await (await getClient()).getConnection(); const connection = await (await this.getClient()).getConnection();
try { try {
const result = await connection.query<ResultSetHeader>(sql); const result = await connection.query<ResultSetHeader>(sql);
...@@ -216,26 +220,23 @@ class ObClass { ...@@ -216,26 +220,23 @@ class ObClass {
insertIds: [] insertIds: []
}; };
} catch (error) { } catch (error) {
addLog.error('[OceanBase] batch insert error', error); addLog.error('[${this.controllerType}] batch insert error', error);
throw error; throw error;
} finally { } finally {
connection.release(); // 释放连接回连接池 connection.release(); // 释放连接回连接池
} }
} }
async query<T extends QueryResult = any>(sql: string) { async query<T extends QueryResult = any>(sql: string) {
const client = await getClient(); const client = await this.getClient();
const start = Date.now(); const start = Date.now();
return client.query<T>(sql).then((res) => { return client.query<T>(sql).then((res) => {
const time = Date.now() - start; const time = Date.now() - start;
if (time > 300) { if (time > 300) {
addLog.warn(`[OceanBase] query time: ${time}ms, sql: ${sql}`); addLog.warn(`[${this.controllerType}] query time: ${time}ms, sql: ${sql}`);
} }
return res; return res;
}); });
} }
} }
export const ObClient = new ObClass();
export const Oceanbase = global.obClient;
packages/service/common/vectorDB/oceanbase/index.ts
/* oceanbase vector crud */ /* oceanbase vector crud */
import { DatasetVectorTableName } from '../constants'; import { DatasetVectorTableName, OceanBaseIndexConfig } from '../constants';
import { ObClient } from './controller'; import { ObClass } from './controller';
import { type RowDataPacket } from 'mysql2/promise'; import { type RowDataPacket } from 'mysql2/promise';
import type { VectorControllerType } from '../type'; import type { VectorControllerType } from '../type';
import dayjs from 'dayjs'; import dayjs from 'dayjs';
import { addLog } from '../../system/log'; import { addLog } from '../../system/log';
export class ObVectorCtrl implements VectorControllerType { export class ObVectorCtrl implements VectorControllerType {
constructor() {} private obClient: ObClass;
private controllerType: 'oceanbase' | 'seekdb';
constructor({ type }: { type: 'oceanbase' | 'seekdb' }) {
this.obClient = new ObClass({ type });
this.controllerType = type;
}
init: VectorControllerType['init'] = async () => { init: VectorControllerType['init'] = async () => {
try { try {
await ObClient.query(` await this.obClient.query(`
CREATE TABLE IF NOT EXISTS ${DatasetVectorTableName} ( CREATE TABLE IF NOT EXISTS ${DatasetVectorTableName} (
id BIGINT AUTO_INCREMENT PRIMARY KEY, id BIGINT AUTO_INCREMENT PRIMARY KEY,
vector VECTOR(1536) NOT NULL, vector VECTOR(1536) NOT NULL,
...@@ -20,19 +25,19 @@ export class ObVectorCtrl implements VectorControllerType { ...@@ -20,19 +25,19 @@ export class ObVectorCtrl implements VectorControllerType {
createtime TIMESTAMP DEFAULT CURRENT_TIMESTAMP createtime TIMESTAMP DEFAULT CURRENT_TIMESTAMP
); );
`); `);
await ObClient.query( await this.obClient.query(
`CREATE VECTOR INDEX IF NOT EXISTS vector_index ON ${DatasetVectorTableName}(vector) WITH (distance=inner_product, type=hnsw, m=32, ef_construction=128);` `CREATE VECTOR INDEX IF NOT EXISTS vector_index ON ${DatasetVectorTableName}(vector) WITH (distance=${OceanBaseIndexConfig.distance}, type=${OceanBaseIndexConfig.type}, m=16, ef_construction=200);`
); );
await ObClient.query( await this.obClient.query(
`CREATE INDEX IF NOT EXISTS team_dataset_collection_index ON ${DatasetVectorTableName}(team_id, dataset_id, collection_id);` `CREATE INDEX IF NOT EXISTS team_dataset_collection_index ON ${DatasetVectorTableName}(team_id, dataset_id, collection_id);`
); );
await ObClient.query( await this.obClient.query(
`CREATE INDEX IF NOT EXISTS create_time_index ON ${DatasetVectorTableName}(createtime);` `CREATE INDEX IF NOT EXISTS create_time_index ON ${DatasetVectorTableName}(createtime);`
); );
addLog.info('init oceanbase successful'); addLog.info(`[${this.controllerType}] init successful`);
} catch (error) { } catch (error) {
addLog.error('init oceanbase error', error); addLog.error(`[${this.controllerType}] init error`, error);
} }
}; };
...@@ -46,7 +51,7 @@ export class ObVectorCtrl implements VectorControllerType { ...@@ -46,7 +51,7 @@ export class ObVectorCtrl implements VectorControllerType {
{ key: 'collection_id', value: String(collectionId) } { key: 'collection_id', value: String(collectionId) }
]); ]);
const { rowCount, insertIds } = await ObClient.insert(DatasetVectorTableName, { const { rowCount, insertIds } = await this.obClient.insert(DatasetVectorTableName, {
values values
}); });
...@@ -89,7 +94,7 @@ export class ObVectorCtrl implements VectorControllerType { ...@@ -89,7 +94,7 @@ export class ObVectorCtrl implements VectorControllerType {
if (!where) return; if (!where) return;
await ObClient.delete(DatasetVectorTableName, { await this.obClient.delete(DatasetVectorTableName, {
where: [where] where: [where]
}); });
}; };
...@@ -126,48 +131,52 @@ export class ObVectorCtrl implements VectorControllerType { ...@@ -126,48 +131,52 @@ export class ObVectorCtrl implements VectorControllerType {
return { results: [] }; return { results: [] };
} }
const rows = await ObClient.query< const rows = await this.obClient
({ .query<
id: string; ({
collection_id: string; id: string;
score: number; collection_id: string;
} & RowDataPacket)[][] score: number;
>( } & RowDataPacket)[][]
`BEGIN; >(
`BEGIN;
SET ob_hnsw_ef_search = ${global.systemEnv?.hnswEfSearch || 100}; SET ob_hnsw_ef_search = ${global.systemEnv?.hnswEfSearch || 100};
SELECT id, collection_id, inner_product(vector, [${vector}]) AS score SELECT id, collection_id, ${OceanBaseIndexConfig.distanceFunc}(vector, [${vector}]) AS score
FROM ${DatasetVectorTableName} FROM ${DatasetVectorTableName}
WHERE team_id='${teamId}' WHERE team_id='${teamId}'
AND dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')}) AND dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})
${filterCollectionIdSql} ${filterCollectionIdSql}
${forbidCollectionSql} ${forbidCollectionSql}
ORDER BY score desc APPROXIMATE LIMIT ${limit}; ORDER BY score ${OceanBaseIndexConfig.orderDirection} APPROXIMATE LIMIT ${limit};
COMMIT;` COMMIT;`
).then(([rows]) => rows[2]); )
.then(([rows]) => rows[2]);
return { return {
results: rows.map((item) => ({ results: rows.map((item) => ({
id: String(item.id), id: String(item.id),
collectionId: item.collection_id, collectionId: item.collection_id,
score: item.score score: OceanBaseIndexConfig.scoreTransform(item.score)
})) }))
}; };
}; };
getVectorDataByTime: VectorControllerType['getVectorDataByTime'] = async (start, end) => { getVectorDataByTime: VectorControllerType['getVectorDataByTime'] = async (start, end) => {
const rows = await ObClient.query< const rows = await this.obClient
({ .query<
id: string; ({
team_id: string; id: string;
dataset_id: string; team_id: string;
} & RowDataPacket)[] dataset_id: string;
>( } & RowDataPacket)[]
`SELECT id, team_id, dataset_id >(
`SELECT id, team_id, dataset_id
FROM ${DatasetVectorTableName} FROM ${DatasetVectorTableName}
WHERE createtime BETWEEN '${dayjs(start).format('YYYY-MM-DD HH:mm:ss')}' AND '${dayjs( WHERE createtime BETWEEN '${dayjs(start).format('YYYY-MM-DD HH:mm:ss')}' AND '${dayjs(
end end
).format('YYYY-MM-DD HH:mm:ss')}'; ).format('YYYY-MM-DD HH:mm:ss')}';
` `
).then(([rows]) => rows); )
.then(([rows]) => rows);
return rows.map((item) => ({ return rows.map((item) => ({
id: String(item.id), id: String(item.id),
...@@ -197,7 +206,7 @@ export class ObVectorCtrl implements VectorControllerType { ...@@ -197,7 +206,7 @@ export class ObVectorCtrl implements VectorControllerType {
} }
// If no conditions provided, count all // If no conditions provided, count all
const total = await ObClient.count(DatasetVectorTableName, { const total = await this.obClient.count(DatasetVectorTableName, {
where: whereConditions.length > 0 ? whereConditions : undefined where: whereConditions.length > 0 ? whereConditions : undefined
}); });
......
packages/service/common/vectorDB/seekdb/index.ts
...@@ -6,7 +6,6 @@ ...@@ -6,7 +6,6 @@
*/ */
// 导出 OceanBase 控制器（复用） // 导出 OceanBase 控制器（复用）
export { ObClient as SeekClient } from '../oceanbase/controller';
export { ObVectorCtrl as SeekVectorCtrl } from '../oceanbase'; export { ObVectorCtrl as SeekVectorCtrl } from '../oceanbase';
// 导出类型 // 导出类型
......
projects/app/.env.template
...@@ -55,8 +55,9 @@ MONGODB_URI="mongodb://myusername:mypassword@localhost:27017/fastgpt?authSource= ...@@ -55,8 +55,9 @@ MONGODB_URI="mongodb://myusername:mypassword@localhost:27017/fastgpt?authSource=
# 日志库 # 日志库
MONGODB_LOG_URI="mongodb://myusername:mypassword@localhost:27017/fastgpt?authSource=admin&directConnection=true" MONGODB_LOG_URI="mongodb://myusername:mypassword@localhost:27017/fastgpt?authSource=admin&directConnection=true"
# 向量库优先级: pg > oceanbase > milvus # 向量库优先级: pg > oceanbase > milvus
# 向量量化等级: PG支持32/16, OceanBase支持32/8/1
VECTOR_VQ_LEVEL=32
# PG 向量库连接参数 # PG 向量库连接参数
VECTOR_VQ_LEVEL=16 # 向量量化等级（目前支持 PG：32,16， 其他数据库未支持）
PG_URL=postgresql://username:password@localhost:5432/postgres PG_URL=postgresql://username:password@localhost:5432/postgres
# OceanBase 向量库连接参数 # OceanBase 向量库连接参数
# OCEANBASE_URL= # OCEANBASE_URL=
......
test/integrationTest/READMD.md 0 → 100644
# FastGPT 集成测试
## 目录
- vectorDB: 向量数据库
\ No newline at end of file
test/integrationTest/vectorDB/.env.test.tempalte 0 → 100644
VECTOR_VQ_LEVEL=32
# PG
PG_URL=postgresql://username:password@localhost:6001/postgres
# OceanBase 可以用云服务来测
# OCEANBASE_URL=mysql://root%40tenantname:tenantpassword@localhost:6005/mysql
# SeekDB vector database connection
SEEKDB_URL=mysql://root:seekdbpassword@127.0.0.1:6003/mysql
# Milvus vector database connection
MILVUS_ADDRESS=http://localhost:6002
MILVUS_TOKEN=
\ No newline at end of file
test/integrationTest/vectorDB/README.md 0 → 100644
# 向量数据库集成测试
对 FastGPT 各向量库控制器（PGVector、后续 Oceanbase/Milvus）做真实环境下的集成测试，保证向量相关操作兼容和稳定。采用**工厂模式**：同一套数据集（fixtures）和同一套用例（factory）驱动 n 个向量库测试。
## 环境变量
测试环境变量由 **test/.env.test.local** 提供（不提交到 git）。请复制模板并填写：
```bash
cp test/.env.test.template test/.env.test.local
# 编辑 test/.env.test.local，填入 PG_URL 等
```
`setup.ts` 会在测试启动时读取 `test/.env.test.local` 并注入到 `process.env`。
| 变量 | 说明 | 适用驱动 |
|------|------|----------|
| `PG_URL` | PostgreSQL + pgvector 连接串 | PgVectorCtrl |
| `OCEANBASE_URL` | Oceanbase 连接串（后续） | ObVectorCtrl |
| `MILVUS_ADDRESS` | Milvus 地址（后续） | MilvusCtrl |
未设置对应环境变量时，该驱动的集成测试会**整体跳过**，不会报错。
## 运行方式
在项目根目录执行：
```bash
# 仅运行单元测试（未配置 .env.test.local 或未设 PG_URL 时，vectorDB 集成测试会跳过）
pnpm test
# 运行所有向量库测试（包含 vectorDB 集成测试与相关单元测试）
pnpm test:vector
```
## 结构说明
- **fixtures.ts**：统一测试数据（`TEST_TEAM_ID`、`TEST_DATASET_ID`、`TEST_COLLECTION_ID`、1536 维 `TEST_VECTORS`），所有向量库共用。
- **factory.ts**：工厂函数 `runVectorDBTests(driver)`，同一套用例（init、insert、getVectorCount、embRecall、getVectorDataByTime、delete）供各驱动复用。
- **integration.test.ts**：注册各驱动（PG、后续 Oceanbase/Milvus），按 `driver.envKey` 决定是否跳过；每个驱动执行同一套 `runVectorDBTests(driver)`。
新增向量库时：在 `integration.test.ts` 的 `drivers` 数组中增加一项（`name`、`envKey`、`createCtrl`），无需改 fixtures 或 factory。
test/integrationTest/vectorDB/globalSetup.ts 0 → 100644
// Load vector database environment variables before tests run
export default async function setup() {
console.log('Vector DB integration tests - environment loaded');
console.log('PG_URL configured:', Boolean(process.env.PG_URL));
console.log('OCEANBASE_URL configured:', Boolean(process.env.OCEANBASE_URL));
console.log('MILVUS_ADDRESS configured:', Boolean(process.env.MILVUS_ADDRESS));
console.log('SEEKDB_URL configured:', Boolean(process.env.SEEKDB_URL));
return async () => {
// Cleanup if needed
};
}
test/integrationTest/vectorDB/milvus/index.integration.test.ts 0 → 100644
import { describe } from 'vitest';
import { MilvusCtrl } from '@fastgpt/service/common/vectorDB/milvus';
import { createVectorDBTestSuite } from '../testSuites';
const isEnabled = Boolean(process.env.MILVUS_ADDRESS);
const describePg = isEnabled ? describe : describe.skip;
describePg('Milvus Vector Integration', () => {
const vectorCtrl = new MilvusCtrl();
createVectorDBTestSuite(vectorCtrl);
});
test/integrationTest/vectorDB/oceanbase/index.integration.test.ts 0 → 100644
import { describe } from 'vitest';
import { ObVectorCtrl } from '@fastgpt/service/common/vectorDB/oceanbase';
import { createVectorDBTestSuite } from '../testSuites';
const isEnabled = Boolean(process.env.OCEANBASE_URL);
const describePg = isEnabled ? describe : describe.skip;
describePg('Oceanbase Vector Integration', () => {
const vectorCtrl = new ObVectorCtrl({ type: 'oceanbase' });
createVectorDBTestSuite(vectorCtrl);
});
test/integrationTest/vectorDB/pg/index.integration.test.ts 0 → 100644
import { describe } from 'vitest';
import { PgVectorCtrl } from '@fastgpt/service/common/vectorDB/pg';
import { createVectorDBTestSuite } from '../testSuites';
const isEnabled = Boolean(process.env.PG_URL);
const describePg = isEnabled ? describe : describe.skip;
describePg('PG Vector Integration', () => {
const vectorCtrl = new PgVectorCtrl();
createVectorDBTestSuite(vectorCtrl);
});
test/integrationTest/vectorDB/seekdb/index.integration.test.ts 0 → 100644
import { describe } from 'vitest';
import { SeekVectorCtrl } from '@fastgpt/service/common/vectorDB/seekdb';
import { createVectorDBTestSuite } from '../testSuites';
const isEnabled = Boolean(process.env.SEEKDB_URL);
const describePg = isEnabled ? describe : describe.skip;
describePg('Seekdb Vector Integration', () => {
const vectorCtrl = new SeekVectorCtrl({ type: 'seekdb' });
createVectorDBTestSuite(vectorCtrl);
});
test/integrationTest/vectorDB/setup.ts 0 → 100644
import { loadVectorDBEnv } from './utils';
// Load env before any modules that read process.env
loadVectorDBEnv({ envFileNames: ['.env.test.local'] });
test/integrationTest/vectorDB/testData.ts 0 → 100644
export const VECTOR_DIM = 1536;
const buildBaseVector = () =>
Array.from({ length: VECTOR_DIM }, (_, index) => ((index % 10) + 1) / 100);
const baseVector = buildBaseVector();
export const TEST_VECTORS = [
baseVector,
baseVector.map((value) => value * 0.7),
baseVector.map((value) => value * 0.3)
];
export const QUERY_VECTOR = baseVector;
export const TEST_COLLECTION_IDS = ['col_1', 'col_2', 'col_3'];
export const createTestIds = () => {
const suffix = `${Date.now()}_${Math.random().toString(36).slice(2, 10)}`;
return {
teamId: `test_team`,
datasetId: `test_dataset_${suffix}`
};
};
test/integrationTest/vectorDB/testSuites.ts 0 → 100644
import { beforeAll, describe, expect, test } from 'vitest';
import type { VectorControllerType } from '@fastgpt/service/common/vectorDB/type';
import { createTestIds, QUERY_VECTOR, TEST_COLLECTION_IDS, TEST_VECTORS } from './testData';
const insertTestVectors = async (
vectorCtrl: VectorControllerType,
teamId: string,
datasetId: string
) => {
const insertIds: string[] = [];
await Promise.all(
TEST_VECTORS.map(async (vector, index) => {
const { insertIds: ids } = await vectorCtrl.insert({
teamId,
datasetId,
collectionId: TEST_COLLECTION_IDS[index],
vectors: [vector]
});
insertIds.push(ids[0]);
})
);
await new Promise((resolve) => setTimeout(resolve, 500));
return insertIds;
};
const cleanupTestVectors = async (
vectorCtrl: VectorControllerType,
teamId: string,
datasetId: string
) => {
try {
await vectorCtrl.delete({
teamId,
datasetIds: [datasetId]
});
} catch (error) {
// Ignore cleanup errors
}
};
export const createVectorDBTestSuite = (vectorCtrl: VectorControllerType) => {
describe.sequential('vectorDB integration', () => {
beforeAll(async () => {
await vectorCtrl.init();
});
test('insert and count', async () => {
const { teamId, datasetId } = createTestIds();
const insertIds = await insertTestVectors(vectorCtrl, teamId, datasetId);
expect(insertIds).toHaveLength(TEST_VECTORS.length);
const count = await vectorCtrl.getVectorCount({ teamId, datasetId });
expect(count).toBe(TEST_VECTORS.length);
const collectionCount = await vectorCtrl.getVectorCount({
teamId,
datasetId,
collectionId: TEST_COLLECTION_IDS[0]
});
expect(collectionCount).toBe(1);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('embRecall returns results', async () => {
const { teamId, datasetId } = createTestIds();
await insertTestVectors(vectorCtrl, teamId, datasetId);
const { results } = await vectorCtrl.embRecall({
teamId,
datasetIds: [datasetId],
vector: QUERY_VECTOR,
limit: 3,
forbidCollectionIdList: []
});
expect(results.length).toBeGreaterThan(0);
expect(results.every((item) => TEST_COLLECTION_IDS.includes(item.collectionId))).toBe(true);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('embRecall respects forbidCollectionIdList', async () => {
const { teamId, datasetId } = createTestIds();
await insertTestVectors(vectorCtrl, teamId, datasetId);
const { results } = await vectorCtrl.embRecall({
teamId,
datasetIds: [datasetId],
vector: QUERY_VECTOR,
limit: 10,
forbidCollectionIdList: [TEST_COLLECTION_IDS[0]]
});
expect(results.length).toBeGreaterThan(0);
expect(results.every((item) => item.collectionId !== TEST_COLLECTION_IDS[0])).toBe(true);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('embRecall respects filterCollectionIdList', async () => {
const { teamId, datasetId } = createTestIds();
await insertTestVectors(vectorCtrl, teamId, datasetId);
const { results } = await vectorCtrl.embRecall({
teamId,
datasetIds: [datasetId],
vector: QUERY_VECTOR,
limit: 10,
forbidCollectionIdList: [],
filterCollectionIdList: [TEST_COLLECTION_IDS[1]]
});
expect(results.length).toBeGreaterThan(0);
expect(results.every((item) => item.collectionId === TEST_COLLECTION_IDS[1])).toBe(true);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('getVectorDataByTime returns data', async () => {
const { teamId, datasetId } = createTestIds();
const insertIds = await insertTestVectors(vectorCtrl, teamId, datasetId);
await new Promise((resolve) => setTimeout(resolve, 500));
const start = new Date(0);
const end = new Date(Date.now() + 600_000);
const data = await vectorCtrl.getVectorDataByTime(start, end);
const matchedIds = data
.filter((item) => item.teamId === teamId && item.datasetId === datasetId)
.map((item) => item.id);
expect(matchedIds.length).toBeGreaterThan(0);
expect(matchedIds).toEqual(expect.arrayContaining(insertIds));
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
test('delete by idList removes vectors', async () => {
const { teamId, datasetId } = createTestIds();
const insertIds = await insertTestVectors(vectorCtrl, teamId, datasetId);
await vectorCtrl.delete({
teamId,
idList: insertIds.slice(0, 2)
});
const count = await vectorCtrl.getVectorCount({ teamId, datasetId });
expect(count).toBe(TEST_VECTORS.length - 2);
await cleanupTestVectors(vectorCtrl, teamId, datasetId);
});
});
};
test/integrationTest/vectorDB/utils.ts 0 → 100644
import { existsSync, readFileSync } from 'fs';
import { resolve } from 'path';
type LoadVectorEnvOptions = {
envFileNames?: string[];
};
const parseEnvFile = (filePath: string) => {
const content = readFileSync(filePath, 'utf-8');
const lines = content.split('\n');
for (const rawLine of lines) {
const line = rawLine.trim();
if (!line || line.startsWith('#')) continue;
const separatorIndex = line.indexOf('=');
if (separatorIndex === -1) continue;
const key = line.slice(0, separatorIndex).trim();
const value = line.slice(separatorIndex + 1).trim();
if (!key || process.env[key]) continue;
process.env[key] = value;
}
};
export const loadVectorDBEnv = (options: LoadVectorEnvOptions = {}) => {
const envFileNames = options.envFileNames ?? ['.env.test.local'];
const baseDir = resolve(__dirname);
for (const envFileName of envFileNames) {
const filePath = resolve(baseDir, envFileName);
if (existsSync(filePath)) {
parseEnvFile(filePath);
}
}
};
test/integrationTest/vectorDB/vitest.config.mts 0 → 100644
import { resolve } from 'path';
import { defineConfig } from 'vitest/config';
export default defineConfig({
resolve: {
alias: {
'@': resolve(__dirname, '../../../projects/app/src'),
'@fastgpt': resolve(__dirname, '../../../packages'),
'@test': resolve(__dirname, '../..')
}
},
test: {
name: 'vectorDB',
root: resolve(__dirname),
setupFiles: './setup.ts',
include: ['**/*.test.ts'],
exclude: ['node_modules', 'dist'],
testTimeout: 60000,
hookTimeout: 60000,
fileParallelism: false,
reporters: ['verbose']
}
});
test/integrationTest/vectorDB/yml/docker-compose.yml 0 → 100644
# 用于部署的 docker-compose 文件:
# - FastGPT 端口映射为 3000:3000
# - FastGPT-mcp-server 端口映射 3005:3000
# - 建议修改账密后再运行
version: '3.3'
services:
# pg DB
pgTest:
image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:0.8.0-pg15
container_name: pgTest
restart: always
ports:
- 6001:5432
networks:
- test
environment:
# 这里的配置只有首次运行生效。修改后，重启镜像是不会生效的。需要把持久化数据删除再重启，才有效果
- POSTGRES_USER=username
- POSTGRES_PASSWORD=password
- POSTGRES_DB=postgres
volumes:
- ./local/pg/data:/var/lib/postgresql/data
healthcheck:
test: ['CMD', 'pg_isready', '-U', 'username', '-d', 'postgres']
interval: 5s
timeout: 5s
retries: 10
# Vector DB
milvus-test-minio:
container_name: milvus-test-minio
image: minio/minio:RELEASE.2023-03-20T20-16-18Z
environment:
MINIO_ACCESS_KEY: minioadmin
MINIO_SECRET_KEY: minioadmin
networks:
- testVector
volumes:
- ./local/milvus-minio:/minio_data
command: minio server /minio_data --console-address ":9001"
healthcheck:
test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live']
interval: 30s
timeout: 20s
retries: 3
# milvus
milvus-test-etcd:
container_name: milvus-test-etcd
image: quay.io/coreos/etcd:v3.5.5
environment:
- ETCD_AUTO_COMPACTION_MODE=revision
- ETCD_AUTO_COMPACTION_RETENTION=1000
- ETCD_QUOTA_BACKEND_BYTES=4294967296
- ETCD_SNAPSHOT_COUNT=50000
networks:
- testVector
volumes:
- ./local/milvus/etcd:/etcd
command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
healthcheck:
test: ['CMD', 'etcdctl', 'endpoint', 'health']
interval: 30s
timeout: 20s
retries: 3
milvus-test:
container_name: milvus-test-standalone
image: milvusdb/milvus:v2.4.3
command: ['milvus', 'run', 'standalone']
ports:
- 6002:19530
security_opt:
- seccomp:unconfined
environment:
ETCD_ENDPOINTS: milvus-test-etcd:2379
MINIO_ADDRESS: milvus-test-minio:9000
networks:
- test
- testVector
volumes:
- ./local/milvus/data:/var/lib/milvus
healthcheck:
test: ['CMD', 'curl', '-f', 'http://localhost:9091/healthz']
interval: 30s
start_period: 90s
timeout: 20s
retries: 3
depends_on:
- 'milvus-test-etcd'
- 'milvus-test-minio'
# Ob
ob-test:
image: oceanbase/oceanbase-ce:4.3.5-lts
container_name: ob-test
restart: always
ports: # 生产环境建议不要暴露
- 6005:2881
networks:
- test
environment:
# 这里的配置只有首次运行生效。修改后，重启镜像是不会生效的。需要把持久化数据删除再重启，才有效果
- OB_SYS_PASSWORD=obsyspassword
# 不同于传统数据库，OceanBase 数据库的账号包含更多字段，包括用户名、租户名和集群名。经典格式为"用户名@租户名#集群名"
# 比如用mysql客户端连接时，根据本文件的默认配置，应该指定 "-uroot@tenantname"
- OB_TENANT_NAME=tenantname
- OB_TENANT_PASSWORD=tenantpassword
# MODE分为MINI和NORMAL， 后者会最大程度使用主机资源
- MODE=MINI
- OB_SERVER_IP=127.0.0.1
# 更多环境变量配置见oceanbase官方文档： https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000002013494
volumes:
- ./local/ob/data:/root/ob
- ./local/ob/config:/root/.obd/cluster
configs:
- source: init_sql
target: /root/boot/init.d/init.sql
healthcheck:
# Use sys tenant for health check as tenantname may take long to initialize
# obclient -h127.0.0.1 -P2881 -uroot@sys -pobsyspassword -e "SELECT 1;"
test:
[
'CMD-SHELL',
'obclient -h$${OB_SERVER_IP} -P2881 -uroot@$${OB_TENANT_NAME} -p$${OB_TENANT_PASSWORD} -e "SELECT 1;"'
]
interval: 30s
timeout: 10s
retries: 1000
start_period: 60s
# Seekdb
seekdb-test:
image: oceanbase/seekdb:1.0.1.0-100000392025122619
container_name: seekdb-test
restart: always
ports: # 生产环境建议不要暴露
- 6003:2881
- 6004:2886
networks:
- test
environment:
# SeekDB 连接配置（兼容 MySQL 协议）
- ROOT_PASSWORD=seekdbpassword
# MODE分为MINI和NORMAL， 后者会最大程度使用主机资源
- MODE=MINI
volumes:
- ./local/seekdb/data:/var/lib/mysql
- ./local/seekdb/config:/etc/mysql/conf.d
healthcheck:
test: ['CMD', 'mysqladmin', 'ping', '-h', '127.0.0.1', '-P2881', '-uroot', '-pseekdbpassword']
interval: 30s
timeout: 10s
retries: 1000
start_period: 10s
networks:
test:
testVector:
configs:
init_sql:
name: init_sql
content: |
ALTER SYSTEM SET ob_vector_memory_limit_percentage = 30;
test/setup.ts
import './mocks'; import './mocks';
import { existsSync, readFileSync } from 'fs'; import { existsSync, readFileSync } from 'fs';
import { connectMongo } from '@fastgpt/service/common/mongo/init'; import { connectMongo } from '@fastgpt/service/common/mongo/init';
import { initGlobalVariables } from '@/service/common/system'; import { initGlobalVariables } from '@/service/common/system';
import { afterAll, beforeAll, beforeEach, inject, onTestFinished, vi } from 'vitest'; import { afterAll, beforeAll, beforeEach, inject, onTestFinished, vi } from 'vitest';
......
vitest.config.mts
...@@ -32,6 +32,7 @@ export default defineConfig({ ...@@ -32,6 +32,7 @@ export default defineConfig({
'projects/sandbox/test/**/*.test.ts', 'projects/sandbox/test/**/*.test.ts',
'projects/marketplace/test/**/*.test.ts' 'projects/marketplace/test/**/*.test.ts'
], ],
exclude: ['test/vectorDB/**'],
testTimeout: 20000, testTimeout: 20000,
hookTimeout: 30000, hookTimeout: 30000,
reporters: ['github-actions', 'default'] reporters: ['github-actions', 'default']
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment