Commit ac04d444 by Archer Committed by GitHub

Add Zod check for api (#6741)

* feat: llm request zod

* feat: apidataset zod

* feat: training zod

* permission data

* feat: dataset data zod

* add log categories

* update skill

* fix: test

* fix: training billId field

* fix: review

* fix: review

* feat: collection zod

* feat: dataset colletion schema

* fix: review

* review

* fix: ts

* feat: update team

* fix: type
parent 6253f224
...@@ -106,9 +106,7 @@ FastGPT 是一个 AI Agent 构建平台,通过 Flow 提供开箱即用的数据 ...@@ -106,9 +106,7 @@ FastGPT 是一个 AI Agent 构建平台,通过 Flow 提供开箱即用的数据
## 代码规范 ## 代码规范
- 采用 DDD 架构 + 模块划分的代码风格,按业务进行划分,然后再按 controller + service + entity 划分。 [FastGPT 代码规范](./skills/system-pr_review/style/syntax.md)
- 使用 type 进行类型声明。
- function props 数量不能超过 2 个,多参数采用对象传递。
## 运行要求 ## 运行要求
......
...@@ -57,14 +57,14 @@ export const CreateDatasetBodySchema = z.object({ ...@@ -57,14 +57,14 @@ export const CreateDatasetBodySchema = z.object({
description: '知识库名称' description: '知识库名称'
}) })
}); });
export type CreateDatasetBody = z.infer<typeof CreateDatasetBodySchema>; export type CreateDatasetBodyType = z.infer<typeof CreateDatasetBodySchema>;
// 出参 Schema // 出参 Schema
export const CreateDatasetResponseSchema = ObjectIdSchema.meta({ export const CreateDatasetResponseSchema = ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05', example: '68ad85a7463006c963799a05',
description: '新创建的知识库 ID' description: '新创建的知识库 ID'
}); });
export type CreateDatasetResponse = z.infer<typeof CreateDatasetResponseSchema>; export type CreateDatasetResponseType = z.infer<typeof CreateDatasetResponseSchema>;
``` ```
### 步骤 2: 实现 API 路由 ### 步骤 2: 实现 API 路由
...@@ -77,13 +77,13 @@ import type { ApiRequestProps } from '@fastgpt/service/type/next'; ...@@ -77,13 +77,13 @@ import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { import {
CreateDatasetBodySchema, CreateDatasetBodySchema,
CreateDatasetResponseSchema, CreateDatasetResponseSchema,
type CreateDatasetResponse type CreateDatasetResponseType
} from '@fastgpt/global/openapi/core/dataset/api'; } from '@fastgpt/global/openapi/core/dataset/api';
// ❌ 不要在路由文件中重导出类型别名 // ❌ 不要在路由文件中重导出类型别名
// export type DatasetCreateBody = CreateDatasetBody; // export type DatasetCreateBodyType = CreateDatasetBodyType;
async function handler(req: ApiRequestProps): Promise<CreateDatasetResponse> { async function handler(req: ApiRequestProps): Promise<CreateDatasetResponseType> {
// 1. 入参验证 // 1. 入参验证
const { parentId, name } = CreateDatasetBodySchema.parse(req.body); const { parentId, name } = CreateDatasetBodySchema.parse(req.body);
...@@ -148,10 +148,10 @@ export const DatasetPath: OpenAPIPath = { ...@@ -148,10 +148,10 @@ export const DatasetPath: OpenAPIPath = {
```typescript ```typescript
// ✅ 正确: 从 openapi 导入 // ✅ 正确: 从 openapi 导入
import type { CreateDatasetBody } from '@fastgpt/global/openapi/core/dataset/api'; import type { CreateDatasetBodyType } from '@fastgpt/global/openapi/core/dataset/api';
// ❌ 错误: 从路由文件导入 // ❌ 错误: 从路由文件导入
import type { DatasetCreateBody } from '@/pages/api/core/dataset/create'; import type { DatasetCreateBodyType } from '@/pages/api/core/dataset/create';
// ❌ 错误: 从旧的 global 文件导入 // ❌ 错误: 从旧的 global 文件导入
import type { CreateDatasetParams } from '@/global/core/dataset/api'; import type { CreateDatasetParams } from '@/global/core/dataset/api';
...@@ -164,12 +164,12 @@ import type { CreateDatasetParams } from '@/global/core/dataset/api'; ...@@ -164,12 +164,12 @@ import type { CreateDatasetParams } from '@/global/core/dataset/api';
```typescript ```typescript
import createHandler from '@/pages/api/core/dataset/create'; import createHandler from '@/pages/api/core/dataset/create';
import type { import type {
CreateDatasetBody, CreateDatasetBodyType,
CreateDatasetResponse CreateDatasetResponseType
} from '@fastgpt/global/openapi/core/dataset/api'; } from '@fastgpt/global/openapi/core/dataset/api';
import { Call } from '@test/utils/request'; import { Call } from '@test/utils/request';
const res = await Call<CreateDatasetBody, {}, CreateDatasetResponse>(createHandler, { const res = await Call<CreateDatasetBodyType, {}, CreateDatasetResponseType>(createHandler, {
auth: users.members[0], auth: users.members[0],
body: { name: 'test', intro: 'intro', avatar: 'avatar', type: DatasetTypeEnum.dataset } body: { name: 'test', intro: 'intro', avatar: 'avatar', type: DatasetTypeEnum.dataset }
}); });
...@@ -304,8 +304,8 @@ grep -r "CreateDatasetParams" projects/app/src/ ...@@ -304,8 +304,8 @@ grep -r "CreateDatasetParams" projects/app/src/
```typescript ```typescript
// ❌ 删除这些 // ❌ 删除这些
export type DatasetCreateQuery = {}; export type DatasetCreateQuery = {};
export type DatasetCreateBody = CreateDatasetBody; export type DatasetCreateBodyType = CreateDatasetBodyType;
export type DatasetCreateResponse = CreateDatasetResponse; export type DatasetCreateResponse = CreateDatasetResponseType;
``` ```
## 审查检查清单 ## 审查检查清单
......
...@@ -11,21 +11,41 @@ description: 当用户传入一个 review 的 pr 链接时候,触发该 skill ...@@ -11,21 +11,41 @@ description: 当用户传入一个 review 的 pr 链接时候,触发该 skill
## 步骤 0:拉取代码 ## 步骤 0:拉取代码
```bash 使用以下命令**无需切换分支**,直接使用 PR 编号即可:
# 检出 PR 分支到本地
gh pr checkout <number>
```bash
# 获取 PR 基本信息 # 获取 PR 基本信息
gh pr view --json number,title,body,author,state,headRefName,baseRefName,additions,deletions,files gh pr view <number> --json number,title,body,author,state,headRefName,baseRefName,additions,deletions,files
# 获取完整 diff # 获取完整 diff
gh pr diff gh pr diff <number>
# 查看 commit 历史 # 查看 commit 历史
gh pr view --json commits --jq '.commits[].messageHeadline' gh pr view <number> --json commits --jq '.commits[].messageHeadline'
# 检查 CI 状态 # 检查 CI 状态
gh pr checks gh pr checks <number>
```
如需在本地运行 **tsc / 单元测试**,使用 `git worktree` 创建独立目录,**不影响当前分支**:
```bash
# 1. 拉取 PR 代码到临时分支
git fetch upstream pull/<number>/head:pr/<number>
# 2. 在独立目录检出(与当前工作区完全隔离)
git worktree add ~/pr-worktrees/pr-<number> pr/<number>
# 3. 进入该目录安装依赖、运行测试
cd ~/pr-worktrees/pr-<number>
pnpm install
pnpm tsc --noEmit # 类型检查
pnpm test # 单元测试
# 4. 审查完毕后清理
cd -
git worktree remove ~/pr-worktrees/pr-<number>
git branch -D pr/<number>
``` ```
--- ---
...@@ -91,6 +111,7 @@ gh pr checks ...@@ -91,6 +111,7 @@ gh pr checks
- [] [包结构规范](./style/package.md) - [] [包结构规范](./style/package.md)
- [] [日志规范](./style/logger.md) - [] [日志规范](./style/logger.md)
- [] [Service 解耦规范](./style/service-decoupling.md) - [] [Service 解耦规范](./style/service-decoupling.md)
- [] [语法风格规范](./style/syntax.md)
--- ---
......
# 代码规范
## 基础代码组织模式
采用 DDD 架构,按业务域 → 子功能 → 固定文件三层划分。
### 目录结构
```
packages/
├── global/core/ # 类型、常量(前后端共享)
│ ├── app/
│ │ ├── type.ts # 顶层聚合类型
│ │ ├── constants.ts
│ │ ├── workflow/
│ │ │ ├── type.ts
│ │ │ └── constants.ts
│ │ ├── version/
│ │ │ └── type.ts
│ │ └── evaluation/
│ │ └── type.ts
│ ├── chat/
│ ├── dataset/
│ └── plugin/
│
└── service/core/ # 后端业务逻辑(不可在前端引用)
├── app/
│ ├── schema.ts # App 主表 Mongoose Schema
│ ├── entity.ts # findById / create / updateById 等基础操作封装
│ ├── service.ts # 聚合业务逻辑(跨子功能协调),不允许互相引用,只允许单向依赖,跨 service 的协调需由上层通过 props 传入另一个 service 或者衍生方法
│ ├── utils.ts # 纯函数工具,无副作用,可独立单测
│ ├── version/
│ │ ├── schema.ts
│ │ ├── entity.ts
│ │ ├── service.ts
│ │ └── utils.ts
│ ├── evaluation/
│ │ ├── schema.ts # 合并多个 schema 到单文件
│ │ ├── entity.ts
│ │ ├── service.ts
│ │ └── utils.ts
│ ├── logs/
│ └── tool/
│ ├── service.ts
│ └── utils.ts
├── chat/
├── dataset/
└── plugin/
```
### 叶子目录固定文件说明
| 文件 | 职责 |
|------|------|
| `schema.ts` | Mongoose Schema 定义,导出 Model 和 SchemaType |
| `entity.ts` | 数据访问封装:`findById`、`create`、`updateById` 等基础操作 |
| `service.ts` | 业务逻辑:调用 entity,跨模块协调,处理业务规则 |
| `utils.ts` | 纯函数工具,无副作用,可独立单测 |
```typescript
// entity.ts 示例 —— 只做数据访问,不含业务判断
export const findAppById = (id: string) =>
MongoApp.findById(id).lean();
export const createApp = (data: AppCreateParams, session?: ClientSession) =>
MongoApp.create([data], { session });
// service.ts 示例 —— 调用 entity,处理业务规则
export const createAppAndInitVersion = async (data: AppCreateParams, session?: ClientSession) => {
const app = await createApp(data, session);
await createVersion({ appId: app._id, ... }, session);
return app;
};
// service 需协同,通过 props 传入另一个 service 或者衍生方法。
const service1 = xxxx
const service2 = (props: {id:string; service1: typeof service1 }) => {
const data = findAppById(id)
return props.service1(data);
};
```
### 层级约束
- `global/core/` 只放类型和常量,**禁止**引入 mongoose、服务端 SDK
- `service/core/` 只在服务端使用,**禁止**被 `packages/web/` 或前端页面直接引用
- 子功能目录不超过 **3 层**嵌套
- 一个目录内无需拆子功能时,直接放 `schema.ts` + `entity.ts` + `service.ts` + `utils.ts`
- 多个 schema 文件(如 `evalSchema.ts` + `evalItemSchema.ts`)**合并**到单个 `schema.ts`
## 使用 `type` 进行类型声明,不使用 `interface`
```typescript
// ❌ 不好的实践
interface User {
id: string;
name: string;
}
// ✅ 好的实践
type User = {
id: string;
name: string;
}
```
---
## 使用 IIFE 写法来取代 if/else 进行变量条件赋值。
```typescript
// ❌ 不好的实践
if (condition) {
value = true;
} else {
value = false;
}
// ✅ 好的实践
const value = (() => {
if (condition) {
return true;
}
return false;
})();
```
---
## 类型推导:Zod schema 同时承担校验和类型
用 `z.infer` 从 schema 推导类型,不重复手写相同结构的 type。
```typescript
// ❌ 不好的实践
type MessageParam = { role: 'user' | 'assistant'; content: string };
const MessageParamSchema = z.object({ role: z.enum(['user', 'assistant']), content: z.string() });
// ✅ 好的实践
export const MessageParamSchema = z.discriminatedUnion('role', [...]);
export type MessageParam = z.infer<typeof MessageParamSchema>;
```
---
## 可选链调用回调
用 `?.()` 调用可选回调,取代 `if (fn) fn()` 的冗余写法。
```typescript
// ❌ 不好的实践
if (onProgress) {
onProgress({ phase: 'creatingContainer' });
}
// ✅ 好的实践
onProgress?.({ phase: 'creatingContainer' });
```
---
## 空值合并取默认值
用 `??` 取代 `||` 处理默认值,避免 `0`、`false`、`''` 被错误覆盖。
```typescript
// ❌ 不好的实践
const version = lastVersion?.version || 0; // version 为 0 时被误覆盖
const text = item?.value || '';
// ✅ 好的实践
const version = (lastVersion?.version ?? -1) + 1;
const text = item?.value ?? '';
```
---
## 解构重命名
同名变量来自多个来源时,解构时重命名,避免命名冲突。
```typescript
// ❌ 不好的实践
const r1 = await getSkillGuidance(...);
const r2 = await createLLMResponse(...);
const inputTokens = r1.usage.inputTokens + r2.usage.inputTokens;
// ✅ 好的实践
const { usage: guidanceUsage } = await getSkillGuidance(...);
const { usage: generateUsage } = await createLLMResponse(...);
const inputTokens = guidanceUsage.inputTokens + generateUsage.inputTokens;
```
---
## 类型守卫
用 `is` 关键字收窄 `unknown` / `any` 类型,替代强制断言。
```typescript
// ❌ 不好的实践
function process(value: unknown) {
const n = value as number; // 不安全
}
// ✅ 好的实践
const isValidNumber = (value: unknown): value is number =>
typeof value === 'number' && Number.isFinite(value);
if (isValidNumber(value)) {
// 此处 value 安全收窄为 number
}
```
---
## 非关键清理用 `.catch()` 链
次要的清理操作(不影响主流程)用 `.catch()` 吞掉错误,不污染主 try/catch。
```typescript
// ❌ 不好的实践
try {
await client.delete();
} catch {
// 清理失败,主流程中断
}
// ✅ 好的实践
await client.delete().catch(() => {});
```
---
## 函数参数不超过 2 个,多参数用对象传递
独立参数不超过 2 个,超过时改为对象参数,便于扩展且无需关心顺序。
```typescript
// ❌ 不好的实践
function createVersion(skillId: string, teamId: string, tmbId: string, version: number) {}
// ✅ 好的实践
function createVersion(data: { skillId: string; teamId: string; tmbId: string; version: number }) {}
```
---
## 数据写操作函数支持可选 session 参数
涉及数据库写操作的函数统一支持可选的 `session` 参数,便于上层组合事务。事务统一通过 `mongoSessionRun` 发起,内部自动处理 startTransaction / commit / abort / retry。
```typescript
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { type ClientSession } from '@fastgpt/service/common/mongo';
// entity.ts —— 基础操作透传 session
export const createVersion = (data: CreateVersionData, session?: ClientSession) =>
MongoAppVersion.create([data], { session });
// service.ts —— 需要事务时用 mongoSessionRun 包裹,外部已有 session 时直接传入
export const createAppAndInitVersion = async (
data: AppCreateParams,
session?: ClientSession
) => {
const create = async (session: ClientSession) => {
const app = await createApp(data, session);
await createVersion({ appId: app._id, version: 0 }, session);
return app;
};
if (session) {
return create(session);
} else {
return mongoSessionRun(create);
}
};
```
...@@ -257,7 +257,7 @@ curl --location --request DELETE 'http://localhost:3000/api/core/dataset/delete? ...@@ -257,7 +257,7 @@ curl --location --request DELETE 'http://localhost:3000/api/core/dataset/delete?
| 参数 | 说明 | 必填 | | 参数 | 说明 | 必填 |
| ---------------- | ----------------------------------------------------------------------------------------------------------- | ---- | | ---------------- | ----------------------------------------------------------------------------------------------------------- | ---- |
| datasetId | 知识库ID | ✅ | | datasetId | 知识库ID | ✅ |
| parentId: | 父级ID,不填则默认为根目录 | | | parentId | 父级ID,不填则默认为根目录 | |
| trainingType | 数据处理方式。chunk: 按文本长度进行分割;qa: 问答对提取 | ✅ | | trainingType | 数据处理方式。chunk: 按文本长度进行分割;qa: 问答对提取 | ✅ |
| indexPrefixTitle | 是否自动生成标题索引 | | | indexPrefixTitle | 是否自动生成标题索引 | |
| customPdfParse | 是否开启PDF增强解析, 默认 false: 关闭;true: 开启; | | | customPdfParse | 是否开启PDF增强解析, 默认 false: 关闭;true: 开启; | |
...@@ -378,10 +378,7 @@ data 为集合的 ID。 ...@@ -378,10 +378,7 @@ data 为集合的 ID。
"data": { "data": {
"collectionId": "65abcfab9d1448617cba5f0d", "collectionId": "65abcfab9d1448617cba5f0d",
"results": { "results": {
"insertLen": 5, // 分割成多少段 "insertLen": 5
"overToken": [],
"repeat": [],
"error": []
} }
} }
} }
...@@ -440,10 +437,7 @@ data 为集合的 ID。 ...@@ -440,10 +437,7 @@ data 为集合的 ID。
"data": { "data": {
"collectionId": "65abd0ad9d1448617cba6031", "collectionId": "65abd0ad9d1448617cba6031",
"results": { "results": {
"insertLen": 1, "insertLen": 1
"overToken": [],
"repeat": [],
"error": []
} }
} }
} }
...@@ -492,10 +486,7 @@ data 为集合的 ID。 ...@@ -492,10 +486,7 @@ data 为集合的 ID。
"data": { "data": {
"collectionId": "65abc044e4704bac793fbd81", "collectionId": "65abc044e4704bac793fbd81",
"results": { "results": {
"insertLen": 1, "insertLen": 1
"overToken": [],
"repeat": [],
"error": []
} }
} }
} }
...@@ -561,10 +552,7 @@ data 为集合的 ID。 ...@@ -561,10 +552,7 @@ data 为集合的 ID。
"data": { "data": {
"collectionId": "65abc044e4704bac793fbd81", "collectionId": "65abc044e4704bac793fbd81",
"results": { "results": {
"insertLen": 1, "insertLen": 1
"overToken": [],
"repeat": [],
"error": []
} }
} }
} }
...@@ -624,10 +612,7 @@ data 为集合的 ID。 ...@@ -624,10 +612,7 @@ data 为集合的 ID。
"data": { "data": {
"collectionId": "6646fcedfabd823cdc6de746", "collectionId": "6646fcedfabd823cdc6de746",
"results": { "results": {
"insertLen": 1, "insertLen": 1
"overToken": [],
"repeat": [],
"error": []
} }
} }
} }
...@@ -992,10 +977,7 @@ curl --location --request POST 'http://localhost:3000/api/core/dataset/data/push ...@@ -992,10 +977,7 @@ curl --location --request POST 'http://localhost:3000/api/core/dataset/data/push
"code": 200, "code": 200,
"statusText": "", "statusText": "",
"data": { "data": {
"insertLen": 1, // 最终插入成功的数量 "insertLen": 1 // 最终插入成功的数量
"overToken": [], // 超出 token 的
"repeat": [], // 重复的数量
"error": [] // 其他错误
} }
} }
``` ```
......
...@@ -9,9 +9,10 @@ description: 'FastGPT V4.14.11 更新说明' ...@@ -9,9 +9,10 @@ description: 'FastGPT V4.14.11 更新说明'
## ⚙️ 优化 ## ⚙️ 优化
1. 对大量接口增加了 zod 参数校验,减少攻击和错误参数类型风险。
## 🐛 修复 ## 🐛 修复
1. 对话 Agent 模式,模型存在刷新后被重置问题。 1. 对话 Agent 模式,模型存在刷新后被重置问题。
2. 部分接口未正确进行权限校验。 2. 部分接口未正确进行权限校验。
3. 修复部分接口 nosql 注入分析。 3. API 推送接口,计费异常。
\ No newline at end of file \ No newline at end of file
...@@ -143,8 +143,8 @@ ...@@ -143,8 +143,8 @@
"document/content/docs/openapi/index.mdx": "2026-02-12T18:45:30+08:00", "document/content/docs/openapi/index.mdx": "2026-02-12T18:45:30+08:00",
"document/content/docs/openapi/intro.en.mdx": "2026-02-26T22:14:30+08:00", "document/content/docs/openapi/intro.en.mdx": "2026-02-26T22:14:30+08:00",
"document/content/docs/openapi/intro.mdx": "2026-02-12T18:45:30+08:00", "document/content/docs/openapi/intro.mdx": "2026-02-12T18:45:30+08:00",
"document/content/docs/openapi/share.en.mdx": "2026-02-26T22:14:30+08:00", "document/content/docs/openapi/share.en.mdx": "2026-04-10T22:55:44+08:00",
"document/content/docs/openapi/share.mdx": "2026-02-12T18:45:30+08:00", "document/content/docs/openapi/share.mdx": "2026-04-10T22:55:44+08:00",
"document/content/docs/self-host/config/json.en.mdx": "2026-03-03T17:39:47+08:00", "document/content/docs/self-host/config/json.en.mdx": "2026-03-03T17:39:47+08:00",
"document/content/docs/self-host/config/json.mdx": "2026-03-03T17:39:47+08:00", "document/content/docs/self-host/config/json.mdx": "2026-03-03T17:39:47+08:00",
"document/content/docs/self-host/config/model/intro.en.mdx": "2026-03-30T10:05:42+08:00", "document/content/docs/self-host/config/model/intro.en.mdx": "2026-03-30T10:05:42+08:00",
...@@ -222,7 +222,7 @@ ...@@ -222,7 +222,7 @@
"document/content/docs/self-host/upgrading/4-14/4141.mdx": "2026-03-03T17:39:47+08:00", "document/content/docs/self-host/upgrading/4-14/4141.mdx": "2026-03-03T17:39:47+08:00",
"document/content/docs/self-host/upgrading/4-14/41410.en.mdx": "2026-03-31T23:15:29+08:00", "document/content/docs/self-host/upgrading/4-14/41410.en.mdx": "2026-03-31T23:15:29+08:00",
"document/content/docs/self-host/upgrading/4-14/41410.mdx": "2026-04-08T16:15:25+08:00", "document/content/docs/self-host/upgrading/4-14/41410.mdx": "2026-04-08T16:15:25+08:00",
"document/content/docs/self-host/upgrading/4-14/41411.mdx": "2026-04-10T13:58:10+08:00", "document/content/docs/self-host/upgrading/4-14/41411.mdx": "2026-04-12T10:39:41+08:00",
"document/content/docs/self-host/upgrading/4-14/4142.en.mdx": "2026-03-03T17:39:47+08:00", "document/content/docs/self-host/upgrading/4-14/4142.en.mdx": "2026-03-03T17:39:47+08:00",
"document/content/docs/self-host/upgrading/4-14/4142.mdx": "2026-03-03T17:39:47+08:00", "document/content/docs/self-host/upgrading/4-14/4142.mdx": "2026-03-03T17:39:47+08:00",
"document/content/docs/self-host/upgrading/4-14/4143.en.mdx": "2026-03-03T17:39:47+08:00", "document/content/docs/self-host/upgrading/4-14/4143.en.mdx": "2026-03-03T17:39:47+08:00",
......
import z from 'zod';
export const CreatePostPresignedUrlResponseSchema = z.object({
url: z.string().nonempty(),
key: z.string().nonempty(),
headers: z.record(z.string(), z.string()),
maxSize: z.number().positive().optional() // bytes
});
export type CreatePostPresignedUrlResponseType = z.infer<
typeof CreatePostPresignedUrlResponseSchema
>;
import type { BucketNameEnum } from './constants'; import { BucketNameEnum } from './constants';
import { ObjectIdSchema } from '../type/mongo';
import z from 'zod';
export type FileTokenQuery = { const FileTokenQuerySchema = z.object({
bucketName: `${BucketNameEnum}`; bucketName: z.enum(BucketNameEnum),
teamId: string; teamId: ObjectIdSchema,
uid: string; // tmbId/ share uid/ teamChat uid uid: z.string().nonempty(),
fileId: string; fileId: z.string().nonempty(),
customExpireMinutes?: number; customExpireMinutes: z.number().optional()
}; });
export type FileTokenQuery = z.infer<typeof FileTokenQuerySchema>;
...@@ -3,25 +3,33 @@ import z from 'zod'; ...@@ -3,25 +3,33 @@ import z from 'zod';
export const ParentIdSchema = z.string().nullish(); export const ParentIdSchema = z.string().nullish();
export type ParentIdType = string | null | undefined; export type ParentIdType = string | null | undefined;
export type GetPathProps = { export const GetPathPropsSchema = z.object({
sourceId?: ParentIdType; sourceId: ParentIdSchema.optional(),
type: 'current' | 'parent'; type: z.enum(['current', 'parent']).optional()
}; });
export type GetPathProps = z.infer<typeof GetPathPropsSchema>;
export type ParentTreePathItemType = { export const ParentTreePathItemSchema = z.object({
parentId: ParentIdType; parentId: ParentIdSchema,
parentName: string; parentName: z.string()
}; });
export type ParentTreePathItemType = z.infer<typeof ParentTreePathItemSchema>;
export type GetResourceFolderListProps = { export const GetResourceFolderListPropsSchema = z.object({
parentId: ParentIdType; parentId: ParentIdSchema
}; });
export type GetResourceFolderListItemResponse = { export type GetResourceFolderListProps = z.infer<typeof GetResourceFolderListPropsSchema>;
name: string;
id: string;
};
export type GetResourceListItemResponse = GetResourceFolderListItemResponse & { export const GetResourceFolderListItemResponseSchema = z.object({
avatar: string; name: z.string(),
isFolder: boolean; id: z.string()
}; });
export type GetResourceFolderListItemResponse = z.infer<
typeof GetResourceFolderListItemResponseSchema
>;
export const GetResourceListItemResponseSchema = GetResourceFolderListItemResponseSchema.extend({
avatar: z.string(),
isFolder: z.boolean()
});
export type GetResourceListItemResponse = z.infer<typeof GetResourceListItemResponseSchema>;
import { i18nT } from '../../../web/i18n/utils'; import { i18nT } from '../../../web/i18n/utils';
import type { CompletionUsage } from './type'; import type { CompletionUsage } from './llm/type';
import type { LLMModelItemType, EmbeddingModelItemType, STTModelType } from './model.schema'; import type { LLMModelItemType, EmbeddingModelItemType, STTModelType } from './model.schema';
export const getLLMDefaultUsage = (): CompletionUsage => { export const getLLMDefaultUsage = (): CompletionUsage => {
......
import type openai from 'openai';
import type { Stream } from 'openai/streaming';
import z from 'zod';
/* 通用类型 */
export const ChatCompletionContentPartTextSchema = z.object({
type: z.literal('text'),
text: z.string(),
key: z.string().optional()
});
// tool function
export const ChatCompletionMessageToolCallFunctionSchema = z.object({
arguments: z.string().meta({ description: '工具参数' }),
name: z.string().meta({ description: '工具名称' })
});
// Function call message
export const ChatCompletionMessageFunctionCallSchema =
ChatCompletionMessageToolCallFunctionSchema.extend({
id: z.string().optional().meta({ description: '工具调用 ID' }),
toolName: z.string().optional().meta({ description: '工具名称' }),
toolAvatar: z.string().optional().meta({ description: '工具头像' })
});
export type ChatCompletionMessageFunctionCall = z.infer<
typeof ChatCompletionMessageFunctionCallSchema
>;
/**
* System message: 对齐 openai SDK 的 ChatCompletionSystemMessageParam
* content 仅允许字符串或纯文本 part 数组
*/
export const ChatCompletionSystemMessageParamSchema = z.object({
role: z.literal('system'),
content: z.union([z.string(), z.array(ChatCompletionContentPartTextSchema)]),
name: z.string().optional()
});
export type ChatCompletionSystemMessageParam = z.infer<
typeof ChatCompletionSystemMessageParamSchema
>;
/* ---------- User Input message: ChatCompletionContentPart schemas ----------
* openai SDK 不导出 runtime zod schema,这里手写对齐 SDK 的联合类型,
* 并加上 FastGPT 的扩展字段:所有分支可选 `key`,以及自定义 `file_url` 分支。
* 外部再扩展新分支:
* z.discriminatedUnion('type', [
* ...ChatCompletionContentPartSchema.options,
* MyCustomPartSchema
* ])
*/
export const ChatCompletionContentPartImageSchema = z.object({
type: z.literal('image_url'),
image_url: z.object({
url: z.string(),
detail: z.enum(['auto', 'low', 'high']).optional()
}),
key: z.string().optional()
});
export const ChatCompletionContentPartInputAudioSchema = z.object({
type: z.literal('input_audio'),
input_audio: z.object({
data: z.string(),
format: z.enum(['wav', 'mp3'])
}),
key: z.string().optional()
});
// SDK 的 `file` 分支(base64 / file_id 输入)
export const ChatCompletionContentPartFileSchema = z.object({
type: z.literal('file'),
file: z.object({
file_data: z.string().optional(),
file_id: z.string().optional(),
filename: z.string().optional()
}),
key: z.string().optional()
});
// FastGPT 自定义扩展:外链文件
export const ChatCompletionContentPartFileTypeSchema = z.object({
type: z.literal('file_url'),
name: z.string(),
url: z.string(),
key: z.string().optional()
});
export const ChatCompletionContentPartSchema = z.discriminatedUnion('type', [
ChatCompletionContentPartTextSchema,
ChatCompletionContentPartImageSchema,
ChatCompletionContentPartInputAudioSchema,
ChatCompletionContentPartFileSchema,
ChatCompletionContentPartFileTypeSchema
]);
export type ChatCompletionContentPart = z.infer<typeof ChatCompletionContentPartSchema>;
export type ChatCompletionContentPartText = z.infer<typeof ChatCompletionContentPartTextSchema>;
export const ChatCompletionUserMessageParamSchema = z.object({
role: z.literal('user'),
content: z.union([z.string(), z.array(ChatCompletionContentPartSchema)]),
name: z.string().optional()
});
export type ChatCompletionUserMessageParam = z.infer<typeof ChatCompletionUserMessageParamSchema>;
/* ========= User end ===== */
/**
* Tool message: 对齐 openai SDK 的 ChatCompletionToolMessageParam,新增可选 `name`
* SDK content 仅允许纯文本 part
*/
export const ChatCompletionToolMessageParamSchema = z.object({
role: z.literal('tool'),
content: z.union([z.string(), z.array(ChatCompletionContentPartTextSchema)]),
tool_call_id: z.string(),
name: z.string().optional()
});
export type ChatCompletionToolMessageParam = z.infer<typeof ChatCompletionToolMessageParamSchema>;
/**
* Function message: 对齐 openai SDK 的 ChatCompletionFunctionMessageParam
* SDK 已标记 deprecated,保留用于旧接口兼容
*/
export const ChatCompletionFunctionMessageParamSchema = z.object({
role: z.literal('function'),
content: z.string().nullable(),
name: z.string()
});
export type ChatCompletionFunctionMessageParam = z.infer<
typeof ChatCompletionFunctionMessageParamSchema
>;
/**
* Assistant message: 对齐 openai SDK 的 ChatCompletionAssistantMessageParam
* - content: 文本或 text/refusal part 数组,可空
* - tool_calls / function_call(已废弃)/ audio / refusal 全部可选
* - 新增 FastGPT 扩展 `interactive` 字段
*/
// SDK 的 refusal content part(仅出现在 assistant 消息里)
export const ChatCompletionContentPartRefusalSchema = z.object({
type: z.literal('refusal'),
refusal: z.string()
});
export type ChatCompletionContentPartRefusal = z.infer<
typeof ChatCompletionContentPartRefusalSchema
>;
// SDK 的 ChatCompletionMessageToolCall(目前 type 只有 'function' 一种)
export const ChatCompletionMessageToolCallSchema = z.object({
id: z.string(),
type: z.literal('function'),
function: ChatCompletionMessageToolCallFunctionSchema
});
export type ChatCompletionMessageToolCall = z.infer<typeof ChatCompletionMessageToolCallSchema>;
export const ChatCompletionAssistantMessageParamSchema = z.object({
role: z.literal('assistant'),
content: z
.union([
z.string(),
z.array(
z.discriminatedUnion('type', [
ChatCompletionContentPartTextSchema,
ChatCompletionContentPartRefusalSchema
])
)
])
.nullish()
.meta({
description: 'Assistant message content',
example: 'Hello, how are you?'
}),
tool_calls: z.array(ChatCompletionMessageToolCallSchema).optional().meta({
description: '工具调用'
}),
// FastGPT 自定义扩展。为避免与 workflow/interactive 形成循环依赖,此处用 z.any() 占位,
// 真实类型见 packages/global/core/workflow/template/system/interactive/type.ts:WorkflowInteractiveResponseType
interactive: z.any().optional().meta({
description: '交互式响应(FastGPT 自定义扩展)'
}),
// 下面的几个,目前系统没用到
audio: z.object({ id: z.string() }).nullish(),
function_call: ChatCompletionMessageToolCallFunctionSchema.nullish().meta({
description: '函数调用',
deprecated: true
}),
name: z.string().optional(),
refusal: z.string().nullish()
});
export type ChatCompletionAssistantMessageParam = z.infer<
typeof ChatCompletionAssistantMessageParamSchema
>;
/* =====Assistant end ===== */
/**
* Developer message: 对齐 openai SDK 的 ChatCompletionDeveloperMessageParam
* o1+ 模型用 developer 消息代替 system
*/
export const ChatCompletionDeveloperMessageParamSchema = z.object({
role: z.literal('developer'),
content: z.union([z.string(), z.array(ChatCompletionContentPartTextSchema)]),
name: z.string().optional()
});
export type ChatCompletionDeveloperMessageParam = z.infer<
typeof ChatCompletionDeveloperMessageParamSchema
>;
/**
* ChatCompletionMessageParam: 6 个 role 的 discriminated union
* 每个分支附加 FastGPT 全局扩展字段:reasoning_content / dataId / hideInUI
*/
const messageParamExtraFields = {
reasoning_content: z.string().optional(),
dataId: z.string().optional(),
hideInUI: z.boolean().optional()
};
export const ChatCompletionMessageParamSchema = z.discriminatedUnion('role', [
ChatCompletionSystemMessageParamSchema.extend(messageParamExtraFields),
ChatCompletionDeveloperMessageParamSchema.extend(messageParamExtraFields),
ChatCompletionUserMessageParamSchema.extend(messageParamExtraFields),
ChatCompletionAssistantMessageParamSchema.extend(messageParamExtraFields),
ChatCompletionToolMessageParamSchema.extend(messageParamExtraFields),
ChatCompletionFunctionMessageParamSchema.extend(messageParamExtraFields)
]);
export type ChatCompletionMessageParam = z.infer<typeof ChatCompletionMessageParamSchema>;
/* ========= Message end ===== */
/* ===== 一些自定义扩展类型 ===== */
// Stream response
export type StreamResponseType = Stream<
openai.Chat.Completions.ChatCompletionChunk & { error?: any }
>;
export type UnStreamResponseType = openai.Chat.Completions.ChatCompletion & {
error?: any;
};
export const CompletionFinishReasonSchema = z
.union([
z.enum(['error', 'close', 'stop', 'length', 'tool_calls', 'content_filter', 'function_call']),
z.literal(null),
z.undefined()
])
.meta({ description: '模型完成原因' });
export type CompletionFinishReason = z.infer<typeof CompletionFinishReasonSchema>;
// export type { Stream };
export * from 'openai';
export * from 'openai/resources';
export type PromptTemplateItem = {
title: string;
desc: string;
value: Record<string, string>;
};
/* v8 ignore file */ /* v8 ignore file */
import { type PromptTemplateItem } from '../type'; import { type PromptTemplateItem } from '../llm/type';
import { i18nT } from '../../../../web/i18n/utils'; import { i18nT } from '../../../../web/i18n/utils';
import { getPromptByVersion } from './utils'; import { getPromptByVersion } from './utils';
......
import type { I18nStringType } from '../../../common/i18n/type'; import type { I18nStringType } from '../../../common/i18n/type';
import { hashStr } from '../../../common/string/tools'; import { hashStr } from '../../../common/string/tools';
import type { ChatCompletionTool } from '../type'; import type { ChatCompletionTool } from '../llm/type';
import { z } from 'zod'; import { z } from 'zod';
// ---- 沙盒状态 ---- // ---- 沙盒状态 ----
......
import openai from 'openai';
import type {
ChatCompletion as SdkChatCompletion,
ChatCompletionMessageToolCall,
ChatCompletionMessageParam as OpenaiChatCompletionMessageParam,
ChatCompletionContentPart as SdkChatCompletionContentPart,
ChatCompletionUserMessageParam as SdkChatCompletionUserMessageParam,
ChatCompletionToolMessageParam as SdkChatCompletionToolMessageParam,
ChatCompletionAssistantMessageParam as SdkChatCompletionAssistantMessageParam
} from 'openai/resources';
import type { WorkflowInteractiveResponseType } from '../workflow/template/system/interactive/type';
import type { Stream } from 'openai/streaming';
import z from 'zod';
// Extension of ChatCompletionMessageParam, Add file url type
export type ChatCompletionContentPartFile = {
type: 'file_url';
name: string;
url: string;
key?: string;
};
// Rewrite ChatCompletionContentPart, Add file type
export type ChatCompletionContentPart =
| (SdkChatCompletionContentPart & { key?: string })
| ChatCompletionContentPartFile;
type CustomChatCompletionUserMessageParam = Omit<SdkChatCompletionUserMessageParam, 'content'> & {
role: 'user';
content: string | Array<ChatCompletionContentPart>;
};
export type CustomChatCompletionToolMessageParam = SdkChatCompletionToolMessageParam & {
role: 'tool';
name?: string;
};
type CustomChatCompletionAssistantMessageParam = SdkChatCompletionAssistantMessageParam & {
role: 'assistant';
interactive?: WorkflowInteractiveResponseType;
};
export type ChatCompletionMessageParam = (
| Exclude<
SdkChatCompletionMessageParam,
| SdkChatCompletionUserMessageParam
| SdkChatCompletionToolMessageParam
| SdkChatCompletionAssistantMessageParam
>
| CustomChatCompletionUserMessageParam
| CustomChatCompletionToolMessageParam
| CustomChatCompletionAssistantMessageParam
) & {
reasoning_content?: string;
dataId?: string;
hideInUI?: boolean;
};
export type SdkChatCompletionMessageParam = OpenaiChatCompletionMessageParam;
/* ToolChoice and functionCall extension */
export type ChatCompletionAssistantToolParam = {
role: 'assistant';
tool_calls: ChatCompletionMessageToolCall[];
};
export type ChatCompletionMessageFunctionCall =
SdkChatCompletionAssistantMessageParam.FunctionCall & {
id?: string;
toolName?: string;
toolAvatar?: string;
};
// Stream response
export type StreamChatType = Stream<openai.Chat.Completions.ChatCompletionChunk & { error?: any }>;
export type UnStreamChatType = openai.Chat.Completions.ChatCompletion;
// UnStream response
export type ChatCompletion = SdkChatCompletion & {
error?: any;
};
export const CompletionFinishReasonSchema = z
.union([
z.enum(['error', 'close', 'stop', 'length', 'tool_calls', 'content_filter', 'function_call']),
z.literal(null),
z.undefined()
])
.meta({ description: '模型完成原因' });
export type CompletionFinishReason = z.infer<typeof CompletionFinishReasonSchema>;
export type { Stream };
export default openai;
export * from 'openai';
export * from 'openai/resources';
// Other
export type PromptTemplateItem = {
title: string;
desc: string;
value: Record<string, string>;
};
...@@ -17,7 +17,7 @@ import type { ...@@ -17,7 +17,7 @@ import type {
ChatCompletionMessageParam, ChatCompletionMessageParam,
ChatCompletionMessageToolCall, ChatCompletionMessageToolCall,
ChatCompletionToolMessageParam ChatCompletionToolMessageParam
} from '../../core/ai/type'; } from '../ai/llm/type';
import { ChatCompletionRequestMessageRoleEnum } from '../../core/ai/constants'; import { ChatCompletionRequestMessageRoleEnum } from '../../core/ai/constants';
import { getPlanCallResponseText } from './utils'; import { getPlanCallResponseText } from './utils';
......
import { ChatCompletionRequestMessageRoleEnum } from '../../ai/constants'; import { ChatCompletionRequestMessageRoleEnum } from '../../ai/constants';
import type { import type { ChatCompletionContentPart, ChatCompletionMessageParam } from '../../ai/llm/type';
ChatCompletionContentPart,
ChatCompletionMessageParam,
ChatCompletionMessageToolCall,
ChatCompletionToolMessageParam
} from '../../ai/type';
import { ChatFileTypeEnum, ChatRoleEnum } from '../constants'; import { ChatFileTypeEnum, ChatRoleEnum } from '../constants';
import type { HelperBotChatItemType } from './type'; import type { HelperBotChatItemType } from './type';
import { simpleUserContentPart } from '../adapt'; import { simpleUserContentPart } from '../adapt';
......
import type { ChunkSettingsType, DatasetDataIndexItemType } from './type';
import type { DatasetCollectionTypeEnum, DatasetCollectionDataProcessModeEnum } from './constants';
import type { ParentIdType } from '../../common/parentFolder/type';
import type { APIFileItemType } from './apiDataset/type';
/* ================= collection ===================== */
// Input + store params
type DatasetCollectionStoreDataType = ChunkSettingsType & {
parentId?: ParentIdType;
metadata?: Record<string, any>;
customPdfParse?: boolean;
};
// create collection params
export type CreateDatasetCollectionParams = DatasetCollectionStoreDataType & {
datasetId: string;
name: string;
type: DatasetCollectionTypeEnum;
fileId?: string;
rawLink?: string;
externalFileId?: string;
externalFileUrl?: string;
apiFileId?: string;
apiFileParentId?: string; //when file is imported by folder, the parentId is the folderId
rawTextLength?: number;
hashRawText?: string;
tags?: string[];
createTime?: Date;
updateTime?: Date;
};
export type ApiCreateDatasetCollectionParams = DatasetCollectionStoreDataType & {
datasetId: string;
tags?: string[];
};
export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
name: string;
text: string;
};
export type LinkCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
link: string;
};
export type ApiDatasetCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
name: string;
apiFileId: string;
};
export type ApiDatasetCreateDatasetCollectionV2Params = ApiCreateDatasetCollectionParams & {
apiFiles: APIFileItemType[];
};
export type FileIdCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
fileId: string;
};
export type reTrainingDatasetFileCollectionParams = DatasetCollectionStoreDataType & {
datasetId: string;
collectionId: string;
};
export type FileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
fileMetadata?: Record<string, any>;
collectionMetadata?: Record<string, any>;
};
export type CsvTableCreateDatasetCollectionParams = {
datasetId: string;
parentId?: string;
fileId: string;
};
export type ExternalFileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
externalFileId?: string;
externalFileUrl: string;
filename?: string;
};
export type ImageCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
collectionName: string;
};
/* ================= tag ===================== */
export type CreateDatasetCollectionTagParams = {
datasetId: string;
tag: string;
};
export type AddTagsToCollectionsParams = {
originCollectionIds: string[];
collectionIds: string[];
datasetId: string;
tag: string;
};
export type UpdateDatasetCollectionTagParams = {
datasetId: string;
tagId: string;
tag: string;
};
/* ================= data ===================== */
export type PgSearchRawType = {
id: string;
collection_id: string;
score: number;
};
export type PushDatasetDataChunkProps = {
q?: string;
a?: string;
imageId?: string;
chunkIndex?: number;
indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];
};
export type PostDatasetSyncParams = {
datasetId: string;
};
export type PushDatasetDataProps = {
collectionId: string;
data: PushDatasetDataChunkProps[];
trainingType?: DatasetCollectionDataProcessModeEnum;
indexSize?: number;
autoIndexes?: boolean;
imageIndex?: boolean;
prompt?: string;
billId?: string;
// Abandon
trainingMode?: DatasetCollectionDataProcessModeEnum;
};
export type PushDatasetDataResponse = {
insertLen: number;
};
import type { DatasetDataIndexItemType, DatasetDataSchemaType } from './type';
export type CreateDatasetDataProps = {
teamId: string;
tmbId: string;
datasetId: string;
collectionId: string;
chunkIndex?: number;
q: string;
a?: string;
imageId?: string;
indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];
indexPrefix?: string;
};
export type UpdateDatasetDataProps = {
dataId: string;
q: string;
a?: string;
indexes?: (Omit<DatasetDataIndexItemType, 'dataId'> & {
dataId?: string; // pg data id
})[];
imageId?: string;
indexPrefix?: string;
};
export type PatchIndexesProps =
| {
type: 'create';
index: Omit<DatasetDataIndexItemType, 'dataId'> & {
dataId?: string;
};
}
| {
type: 'update';
index: DatasetDataIndexItemType;
}
| {
type: 'delete';
index: DatasetDataIndexItemType;
}
| {
type: 'unChange';
index: DatasetDataIndexItemType;
};
...@@ -65,7 +65,7 @@ export const DatasetSchema = z ...@@ -65,7 +65,7 @@ export const DatasetSchema = z
.object({ .object({
_id: ObjectIdSchema.meta({ description: '数据集 ID' }), _id: ObjectIdSchema.meta({ description: '数据集 ID' }),
parentId: ParentIdSchema.meta({ description: '父级 ID' }), parentId: ParentIdSchema.meta({ description: '父级 ID' }),
userId: ObjectIdSchema.meta({ description: '用户 ID' }), userId: ObjectIdSchema.optional().meta({ description: '用户 ID', deprecated: true }),
teamId: ObjectIdSchema.meta({ description: '团队 ID' }), teamId: ObjectIdSchema.meta({ description: '团队 ID' }),
tmbId: ObjectIdSchema.meta({ description: '团队成员 ID' }), tmbId: ObjectIdSchema.meta({ description: '团队成员 ID' }),
updateTime: z.date().meta({ description: '更新时间' }), updateTime: z.date().meta({ description: '更新时间' }),
...@@ -143,7 +143,10 @@ export const DatasetCollectionSchema = ChunkSettingsSchema.omit({ ...@@ -143,7 +143,10 @@ export const DatasetCollectionSchema = ChunkSettingsSchema.omit({
metadata: z.record(z.string(), z.any()).optional().meta({ description: '其他元数据' }), metadata: z.record(z.string(), z.any()).optional().meta({ description: '其他元数据' }),
customPdfParse: z.boolean().optional().meta({ description: '自定义 PDF 解析' }), customPdfParse: z.boolean().optional().meta({ description: '自定义 PDF 解析' }),
trainingType: z.enum(DatasetCollectionDataProcessModeEnum).meta({ description: '训练类型' }) trainingType: z
.enum(DatasetCollectionDataProcessModeEnum)
.optional()
.meta({ description: '训练类型' })
}); });
export type DatasetCollectionSchemaType = z.infer<typeof DatasetCollectionSchema>; export type DatasetCollectionSchemaType = z.infer<typeof DatasetCollectionSchema>;
...@@ -157,10 +160,20 @@ export type DatasetCollectionTagsSchemaType = z.infer<typeof DatasetCollectionTa ...@@ -157,10 +160,20 @@ export type DatasetCollectionTagsSchemaType = z.infer<typeof DatasetCollectionTa
/* ===== Data ===== */ /* ===== Data ===== */
export const DatasetDataIndexItemSchema = z.object({ export const DatasetDataIndexItemSchema = z.object({
type: z.enum(DatasetDataIndexTypeEnum).meta({ description: '索引类型' }), type: z
.enum(DatasetDataIndexTypeEnum)
.optional()
.default(DatasetDataIndexTypeEnum.custom)
.meta({ description: '索引类型' }),
dataId: z.string().meta({ description: 'vectorDB ID' }), dataId: z.string().meta({ description: 'vectorDB ID' }),
text: z.string().meta({ description: '索引文本' }) text: z.string().meta({ description: '索引文本' })
}); });
const DatasetDataIndexOptionalSchema = DatasetDataIndexItemSchema.omit({ dataId: true }).extend({
dataId: z.string().optional().meta({
example: '68ad85a7463006c963799a05',
description: 'PG 数据 ID(可选)'
})
});
export type DatasetDataIndexItemType = z.infer<typeof DatasetDataIndexItemSchema>; export type DatasetDataIndexItemType = z.infer<typeof DatasetDataIndexItemSchema>;
export const DatasetDataFieldSchema = z.object({ export const DatasetDataFieldSchema = z.object({
...@@ -177,7 +190,7 @@ export type DatasetDataHistoryType = z.infer<typeof DatasetDataHistorySchema>; ...@@ -177,7 +190,7 @@ export type DatasetDataHistoryType = z.infer<typeof DatasetDataHistorySchema>;
export const DatasetDataSchema = DatasetDataFieldSchema.extend({ export const DatasetDataSchema = DatasetDataFieldSchema.extend({
_id: ObjectIdSchema.meta({ description: '数据 ID' }), _id: ObjectIdSchema.meta({ description: '数据 ID' }),
userId: ObjectIdSchema.meta({ description: '用户 ID' }), userId: ObjectIdSchema.optional().meta({ description: '用户 ID', deprecated: true }),
teamId: ObjectIdSchema.meta({ description: '团队 ID' }), teamId: ObjectIdSchema.meta({ description: '团队 ID' }),
tmbId: ObjectIdSchema.meta({ description: '团队成员 ID' }), tmbId: ObjectIdSchema.meta({ description: '团队成员 ID' }),
datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }), datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }),
...@@ -206,7 +219,6 @@ export type DatasetDataTextSchemaType = z.infer<typeof DatasetDataTextSchema>; ...@@ -206,7 +219,6 @@ export type DatasetDataTextSchemaType = z.infer<typeof DatasetDataTextSchema>;
/* ===== Training ===== */ /* ===== Training ===== */
export const DatasetTrainingSchema = z.object({ export const DatasetTrainingSchema = z.object({
_id: ObjectIdSchema.meta({ description: '训练 ID' }), _id: ObjectIdSchema.meta({ description: '训练 ID' }),
userId: ObjectIdSchema.meta({ description: '用户 ID' }),
teamId: ObjectIdSchema.meta({ description: '团队 ID' }), teamId: ObjectIdSchema.meta({ description: '团队 ID' }),
tmbId: ObjectIdSchema.meta({ description: '团队成员 ID' }), tmbId: ObjectIdSchema.meta({ description: '团队成员 ID' }),
datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }), datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }),
...@@ -227,7 +239,9 @@ export const DatasetTrainingSchema = z.object({ ...@@ -227,7 +239,9 @@ export const DatasetTrainingSchema = z.object({
.array(DatasetDataIndexItemSchema.omit({ dataId: true })) .array(DatasetDataIndexItemSchema.omit({ dataId: true }))
.meta({ description: '向量索引' }), .meta({ description: '向量索引' }),
retryCount: z.number().meta({ description: '重试次数' }), retryCount: z.number().meta({ description: '重试次数' }),
errorMsg: z.string().optional().meta({ description: '错误信息' }) errorMsg: z.string().optional().meta({ description: '错误信息' }),
userId: ObjectIdSchema.optional().meta({ description: '用户 ID', deprecated: true })
}); });
export type DatasetTrainingSchemaType = z.infer<typeof DatasetTrainingSchema>; export type DatasetTrainingSchemaType = z.infer<typeof DatasetTrainingSchema>;
...@@ -322,6 +336,50 @@ export const DatasetDataItemSchema = DatasetDataFieldSchema.extend({ ...@@ -322,6 +336,50 @@ export const DatasetDataItemSchema = DatasetDataFieldSchema.extend({
}); });
export type DatasetDataItemType = z.infer<typeof DatasetDataItemSchema>; export type DatasetDataItemType = z.infer<typeof DatasetDataItemSchema>;
// Update dataset data
export const UpdateDatasetDataPropsSchema = z.object({
dataId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '数据 ID'
}),
q: z.string().meta({
example: '什么是 FastGPT?',
description: '问题/主文本'
}),
a: z.string().optional().meta({
example: 'FastGPT 是一个 AI Agent 构建平台',
description: '回答/补充文本'
}),
indexes: z.array(DatasetDataIndexOptionalSchema).optional().meta({
description: '向量索引列表'
}),
imageId: z.string().optional().meta({
description: '图片 ID'
}),
indexPrefix: z.string().optional().meta({
description: '索引前缀标题'
})
});
export type UpdateDatasetDataPropsType = z.infer<typeof UpdateDatasetDataPropsSchema>;
// Create dataset data
export const CreateDatasetDataPropsSchema = z.object({
teamId: ObjectIdSchema.meta({ description: '团队 ID' }),
tmbId: ObjectIdSchema.meta({ description: '团队成员 ID' }),
datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }),
collectionId: ObjectIdSchema.meta({ description: '集合 ID' }),
chunkIndex: z.int().min(0).optional().meta({ description: '块索引' }),
q: z.string().meta({ description: '问题/主文本' }),
a: z.string().optional().meta({ description: '回答/补充文本' }),
imageId: z.string().optional().meta({ description: '图片 ID' }),
indexes: z
.array(DatasetDataIndexItemSchema.omit({ dataId: true }))
.optional()
.meta({ description: '向量索引列表' }),
indexPrefix: z.string().optional().meta({ description: '索引前缀标题' })
});
export type CreateDatasetDataPropsType = z.infer<typeof CreateDatasetDataPropsSchema>;
/* --------------- file ---------------------- */ /* --------------- file ---------------------- */
export const DatasetFileSchema = z.object({ export const DatasetFileSchema = z.object({
_id: ObjectIdSchema.meta({ description: '文件 ID' }), _id: ObjectIdSchema.meta({ description: '文件 ID' }),
......
import { ObjectIdSchema } from '../../../common/type/mongo';
import z from 'zod';
export const PresignDatasetFileGetUrlSchema = z.union([
z.object({
key: z
.string()
.nonempty()
.refine((key) => key.startsWith('dataset/'), {
message: 'Invalid key format: must start with "dataset/"'
})
.transform((k) => decodeURIComponent(k)),
preview: z.boolean().optional()
}),
z.object({
collectionId: ObjectIdSchema
// datasetId: ObjectIdSchema
})
]);
export type PresignDatasetFileGetUrlParams = z.infer<typeof PresignDatasetFileGetUrlSchema>;
export const PresignDatasetFilePostUrlSchema = z.object({
filename: z.string().min(1),
datasetId: ObjectIdSchema
});
export type PresignDatasetFilePostUrlParams = z.infer<typeof PresignDatasetFilePostUrlSchema>;
export const ShortPreviewLinkSchema = z.object({
k: z
.string()
.nonempty()
.transform((k) => `chat:temp_file:${decodeURIComponent(k)}`)
});
export type ShortPreviewLinkParams = z.infer<typeof ShortPreviewLinkSchema>;
import z from 'zod'; import z from 'zod';
import type { ChatCompletionTool } from '../../../ai/type'; import type { ChatCompletionTool } from '../../../ai/llm/type';
export enum SandboxToolIds { export enum SandboxToolIds {
readFile = 'sandbox_read_file', readFile = 'sandbox_read_file',
......
...@@ -18,7 +18,7 @@ import { ReadFileNodeResponseSchema } from '../template/system/readFiles/type'; ...@@ -18,7 +18,7 @@ import { ReadFileNodeResponseSchema } from '../template/system/readFiles/type';
import type { WorkflowResponseType } from '../../../../service/core/workflow/dispatch/type'; import type { WorkflowResponseType } from '../../../../service/core/workflow/dispatch/type';
import type { AiChatQuoteRoleType } from '../template/system/aiChat/type'; import type { AiChatQuoteRoleType } from '../template/system/aiChat/type';
import type { OpenaiAccountType } from '../../../support/user/team/type'; import type { OpenaiAccountType } from '../../../support/user/team/type';
import { CompletionFinishReasonSchema } from '../../ai/type'; import { CompletionFinishReasonSchema } from '../../ai/llm/type';
import type { import type {
InteractiveNodeResponseType, InteractiveNodeResponseType,
WorkflowInteractiveResponseType WorkflowInteractiveResponseType
......
import { NodeOutputItemSchema } from '../../../runtime/type'; import { NodeOutputItemSchema } from '../../../runtime/type';
import { FlowNodeInputTypeEnum } from '../../../../../core/workflow/node/constant'; import { FlowNodeInputTypeEnum } from '../../../../../core/workflow/node/constant';
import { WorkflowIOValueTypeEnum } from '../../../../../core/workflow/constants'; import { WorkflowIOValueTypeEnum } from '../../../../../core/workflow/constants';
import type { ChatCompletionMessageParam } from '../../../../ai/type';
import { AppFileSelectConfigTypeSchema } from '../../../../app/type/config.schema'; import { AppFileSelectConfigTypeSchema } from '../../../../app/type/config.schema';
import { RuntimeEdgeItemTypeSchema } from '../../../type/edge'; import { RuntimeEdgeItemTypeSchema } from '../../../type/edge';
import z from 'zod'; import z from 'zod';
import { ChatCompletionMessageParamSchema } from '../../../../ai/llm/type';
export const InteractiveBasicTypeSchema = z.object({ export const InteractiveBasicTypeSchema = z.object({
entryNodeIds: z.array(z.string()), entryNodeIds: z.array(z.string()),
...@@ -42,21 +42,12 @@ export const ToolCallChildrenInteractiveSchema = z.object({ ...@@ -42,21 +42,12 @@ export const ToolCallChildrenInteractiveSchema = z.object({
params: z.object({ params: z.object({
childrenResponse: z.any(), childrenResponse: z.any(),
toolParams: z.object({ toolParams: z.object({
memoryRequestMessages: z.array(z.any()), // 这轮工具中,产生的新的 messages memoryRequestMessages: z.array(ChatCompletionMessageParamSchema), // 这轮工具中,产生的新的 messages
toolCallId: z.string() // 记录对应 tool 的id,用于后续交互节点可以替换掉 tool 的 response toolCallId: z.string() // 记录对应 tool 的id,用于后续交互节点可以替换掉 tool 的 response
}) })
}) })
}); });
export type ToolCallChildrenInteractive = InteractiveNodeType & { export type ToolCallChildrenInteractive = z.infer<typeof ToolCallChildrenInteractiveSchema>;
type: 'toolChildrenInteractive';
params: {
childrenResponse: WorkflowInteractiveResponseType;
toolParams: {
memoryRequestMessages: ChatCompletionMessageParam[]; // 这轮工具中,产生的新的 messages
toolCallId: string; // 记录对应 tool 的id,用于后续交互节点可以替换掉 tool 的 response
};
};
};
// Loop bode // Loop bode
export const LoopInteractiveSchema = z.object({ export const LoopInteractiveSchema = z.object({
......
...@@ -16,7 +16,12 @@ export type PaginationProps<T = {}> = T & { ...@@ -16,7 +16,12 @@ export type PaginationProps<T = {}> = T & {
pageNum: number | string; pageNum: number | string;
}>; }>;
export const PaginationResponseSchema = <T extends z.ZodTypeAny>(itemSchema: T) => export const PaginationResponseSchema = <T extends z.ZodTypeAny>(
itemSchema: T
): z.ZodObject<{
total: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
list: z.ZodDefault<z.ZodOptional<z.ZodArray<T>>>;
}> =>
z.object({ z.object({
total: z.number().optional().default(0), total: z.number().optional().default(0),
list: z.array(itemSchema).optional().default([]) list: z.array(itemSchema).optional().default([])
......
...@@ -52,7 +52,7 @@ export type SandboxReadBody = z.infer<typeof SandboxReadBodySchema>; ...@@ -52,7 +52,7 @@ export type SandboxReadBody = z.infer<typeof SandboxReadBodySchema>;
export const SandboxReadResponseSchema = z export const SandboxReadResponseSchema = z
.string() .string()
.openapi({ format: 'binary', description: '文件内容流' }); .meta({ format: 'binary', description: '文件内容流' });
/** /**
* 下载文件或目录 - 请求体(响应为文件流或 ZIP) * 下载文件或目录 - 请求体(响应为文件流或 ZIP)
...@@ -64,7 +64,7 @@ export type SandboxDownloadBody = z.input<typeof SandboxDownloadBodySchema>; ...@@ -64,7 +64,7 @@ export type SandboxDownloadBody = z.input<typeof SandboxDownloadBodySchema>;
export const SandboxDownloadResponseSchema = z export const SandboxDownloadResponseSchema = z
.string() .string()
.openapi({ format: 'binary', description: '文件流或 ZIP 包' }); .meta({ format: 'binary', description: '文件流或 ZIP 包' });
/** /**
* 检查沙盒是否存在 * 检查沙盒是否存在
......
import z from 'zod';
import { ObjectIdSchema } from '../../../../common/type/mongo';
import { ChatCompletionMessageParamSchema } from '../../../../core/ai/llm/type';
import { getNanoid } from '../../../../common/string/tools';
import { AppChatConfigTypeSchema, AppSchemaTypeSchema } from '../../../../core/app/type';
import { AuthUserTypeEnum } from '../../../../support/permission/constant';
import { OutLinkChatAuthSchema } from '../../../../support/permission/chat';
import { StoreNodeItemTypeSchema } from '../../../../core/workflow/type/node';
import { StoreEdgeItemTypeSchema } from '../../../../core/workflow/type/edge';
const WebCompletionsSchema = z.object({
chatId: z
.string()
.max(1024)
.optional()
.meta({ description: '聊天ID, 传入的话会自动获取历史记录,不传入则认为是新对话' }),
appId: ObjectIdSchema.optional(),
customUid: z.string().max(1024).optional().meta({ description: '自定义用户ID(分享链接)' }),
metadata: z.record(z.string(), z.any()).optional().meta({ description: '元数据' })
});
// completions 接口实际上并没有用完所有字段,所以这里就取局部即可
const ChatCompletionCreateParamsSchema = z.object({
messages: z.array(ChatCompletionMessageParamSchema).optional().default([]).meta({
description: '消息列表'
}),
stream: z.boolean().optional().default(false).meta({
description: '是否流式返回'
})
});
export const CompletionsPropsSchema = OutLinkChatAuthSchema.extend(WebCompletionsSchema.shape)
.extend(ChatCompletionCreateParamsSchema.shape)
.extend({
variables: z.record(z.string(), z.any()).optional().default({}).meta({
description: '全局变量或插件输入'
}),
responseChatItemId: z
.string()
.optional()
.default(() => getNanoid())
.meta({
description: '自定义响应的 assistant 的消息 ID,如果不传入,则自动生成一个'
}),
detail: z.boolean().optional().default(false).meta({
description: '是否返回详细信息,包括 reasoning_content, tool_calls, usage 等'
}),
retainDatasetCite: z.boolean().optional().default(false).meta({
description: '是否保留数据集引用'
}),
showSkillReferences: z.boolean().optional().default(false).meta({
description: '是否显示技能引用'
})
});
export type CompletionsProps = z.infer<typeof CompletionsPropsSchema>;
/* =============== Response =============== */
const ChatCompletionResponseMessageSchema = z.object({
role: z.literal('assistant').meta({ description: '消息角色' }),
content: z.any().meta({
description:
'消息内容。普通对话为字符串;detail=true 或工作流命中交互节点时,可能为带 type 字段的对象数组(type 取值: text / interactive / tool / file / reasoning)'
}),
reasoning_content: z.string().optional().meta({ description: '思考过程内容(仅推理模型有)' })
});
const ChatCompletionChoiceSchema = z.object({
message: ChatCompletionResponseMessageSchema.meta({ description: '助手消息' }),
finish_reason: z.string().meta({ description: '完成原因,例如 stop' }),
index: z.number().meta({ description: '选项索引' })
});
const ChatCompletionUsageSchema = z.object({
prompt_tokens: z
.literal(1)
.meta({ description: '固定为 1,需要从 detail 中计算每一个节点的token数' }),
completion_tokens: z
.literal(1)
.meta({ description: '固定为 1,需要从 detail 中计算每一个节点的token数' }),
total_tokens: z
.literal(1)
.meta({ description: '固定为 1,需要从 detail 中计算每一个节点的token数' })
});
export const CompletionsResponseSchema = z.object({
id: z.string().meta({ description: '对话 ID(chatId)' }),
model: z.literal('').meta({ description: '模型名称,v1 接口固定为空字符串' }),
usage: ChatCompletionUsageSchema.meta({
description: 'Token 用量。v1 接口为占位值,需要时请从 responseData 计算'
}),
choices: z.array(ChatCompletionChoiceSchema).meta({ description: '回复选项列表' }),
responseData: z.array(z.any()).optional().meta({
description:
'各节点详细响应数据(仅 detail=true 时返回)。每项是一个节点的执行结果,常见字段如 moduleName / moduleType / runningTime / quoteList 等'
}),
newVariables: z
.record(z.string(), z.any())
.optional()
.meta({ description: '工作流执行后更新的变量(仅 detail=true 时返回)' })
});
export type CompletionsResponseType = z.infer<typeof CompletionsResponseSchema>;
export const AuthResponseSchema = z.object({
teamId: ObjectIdSchema.meta({ description: '团队ID' }),
tmbId: ObjectIdSchema.meta({ description: '团队成员ID' }),
app: AppSchemaTypeSchema.meta({ description: '应用' }),
showCite: z.boolean().default(false).optional().meta({
description: '是否显示引用'
}),
showRunningStatus: z.boolean().default(false).optional().meta({
description: '是否显示运行状态'
}),
showSkillReferences: z.boolean().default(false).optional().meta({
description: '是否显示技能引用'
}),
authType: z.enum(AuthUserTypeEnum).meta({ description: '认证类型' }),
apikey: z.string().optional().meta({ description: 'API密钥' }),
responseAllData: z.boolean().meta({
description: '是否返回所有数据'
}),
outLinkUserId: z.string().optional().meta({ description: '外部链接用户ID' }),
sourceName: z.string().optional().meta({ description: '来源名称' })
});
export type AuthResponseType = z.infer<typeof AuthResponseSchema>;
/* ====== Chat test ====== */
export const ChatTestPropsSchema = z.object({
messages: z.array(ChatCompletionMessageParamSchema).meta({ description: '消息列表' }),
responseChatItemId: z
.string()
.optional()
.meta({ description: '自定义响应的 assistant 的消息 ID,如果不传入,则自动生成一个' }),
nodes: z.array(StoreNodeItemTypeSchema).meta({ description: '节点列表' }),
edges: z.array(StoreEdgeItemTypeSchema).meta({ description: '边列表' }),
chatConfig: AppChatConfigTypeSchema.meta({ description: '聊天配置' }),
variables: z.record(z.string(), z.any()).optional().default({}).meta({
description: '全局变量或插件输入'
}),
appId: ObjectIdSchema.meta({ description: '应用ID' }),
appName: z.string().meta({ description: '应用名称' }),
chatId: z.string().meta({ description: '聊天ID' })
});
export type ChatTestPropsType = z.infer<typeof ChatTestPropsSchema>;
import type { OpenAPIPath } from '../../../type'; import type { OpenAPIPath } from '../../../type';
import { TagsMap } from '../../../tag'; import { TagsMap } from '../../../tag';
import { PresignChatFilePostUrlSchema, PresignChatFileGetUrlSchema } from './api'; import { PresignChatFilePostUrlSchema, PresignChatFileGetUrlSchema } from './api';
import { CreatePostPresignedUrlResultSchema } from '../../../../../service/common/s3/type'; import { CreatePostPresignedUrlResponseSchema } from '../../../../common/file/s3/type';
import { z } from 'zod'; import { z } from 'zod';
export const ChatFilePath: OpenAPIPath = { export const ChatFilePath: OpenAPIPath = {
...@@ -22,7 +22,7 @@ export const ChatFilePath: OpenAPIPath = { ...@@ -22,7 +22,7 @@ export const ChatFilePath: OpenAPIPath = {
description: '成功上传对话文件预签名 URL', description: '成功上传对话文件预签名 URL',
content: { content: {
'application/json': { 'application/json': {
schema: CreatePostPresignedUrlResultSchema schema: CreatePostPresignedUrlResponseSchema
} }
} }
} }
......
...@@ -11,6 +11,7 @@ import { ChatInputGuidePath } from './inputGuide/index'; ...@@ -11,6 +11,7 @@ import { ChatInputGuidePath } from './inputGuide/index';
import { OutLinkChatPath } from './outLink/index'; import { OutLinkChatPath } from './outLink/index';
import { ChatRecordPath } from './record/index'; import { ChatRecordPath } from './record/index';
import { ChatFilePath } from './file'; import { ChatFilePath } from './file';
import { ChatCompletionPath } from './completion';
export const ChatPath: OpenAPIPath = { export const ChatPath: OpenAPIPath = {
...ChatFeedbackPath, ...ChatFeedbackPath,
...@@ -23,6 +24,7 @@ export const ChatPath: OpenAPIPath = { ...@@ -23,6 +24,7 @@ export const ChatPath: OpenAPIPath = {
...ChatInputGuidePath, ...ChatInputGuidePath,
...OutLinkChatPath, ...OutLinkChatPath,
...ChatRecordPath, ...ChatRecordPath,
...ChatCompletionPath,
'/core/chat/recentlyUsed': { '/core/chat/recentlyUsed': {
get: { get: {
......
...@@ -411,3 +411,41 @@ export const ExportDatasetQuerySchema = z.object({ ...@@ -411,3 +411,41 @@ export const ExportDatasetQuerySchema = z.object({
}) })
}); });
export type ExportDatasetQuery = z.infer<typeof ExportDatasetQuerySchema>; export type ExportDatasetQuery = z.infer<typeof ExportDatasetQuerySchema>;
/* ============================================================================
* API: 获取知识库引用权限
* Route: GET /api/core/dataset/getPermission
* ============================================================================ */
export const GetDatasetPermissionQuerySchema = z.object({
id: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
})
});
export type GetDatasetPermissionQuery = z.infer<typeof GetDatasetPermissionQuerySchema>;
export const GetDatasetPermissionResponseSchema = z.object({
datasetName: z.string().meta({
example: '产品文档知识库',
description: '知识库名称'
}),
permission: z.object({
hasWritePer: z.boolean().meta({
example: true,
description: '是否有写权限'
}),
hasReadPer: z.boolean().meta({
example: true,
description: '是否有读权限'
})
})
});
export type GetDatasetPermissionResponse = z.infer<typeof GetDatasetPermissionResponseSchema>;
/* ============================================================================
* 数据集同步入参
* ============================================================================ */
export const PostDatasetSyncBodySchema = z.object({
datasetId: z.string().meta({ description: '数据集 ID' })
});
export type PostDatasetSyncParams = z.infer<typeof PostDatasetSyncBodySchema>;
import { z } from 'zod';
import { ObjectIdSchema } from '../../../../common/type/mongo';
import { ParentIdSchema } from '../../../../common/parentFolder/type';
import {
APIFileItemSchema,
ApiDatasetServerSchema
} from '../../../../core/dataset/apiDataset/type';
/* ============================================================================
* API: 获取第三方知识库目录(仅文件夹)
* Route: POST /api/core/dataset/apiDataset/getCatalog
* ============================================================================ */
export const GetApiDatasetCatalogBodySchema = z.object({
searchKey: z.string().optional().meta({
example: '产品文档',
description: '搜索关键词'
}),
parentId: ParentIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '父级节点 ID,不传或 null 表示根目录'
}),
apiDatasetServer: ApiDatasetServerSchema.optional().meta({
description: '第三方知识库服务器配置(API/飞书/语雀)'
})
});
export type GetApiDatasetCatalogBody = z.infer<typeof GetApiDatasetCatalogBodySchema>;
export const GetApiDatasetCatalogResponseSchema = z.array(APIFileItemSchema).meta({
description: '目录列表(仅包含 hasChild = true 的节点)'
});
export type GetApiDatasetCatalogResponse = z.infer<typeof GetApiDatasetCatalogResponseSchema>;
/* ============================================================================
* API: 获取第三方知识库节点完整路径
* Route: POST /api/core/dataset/apiDataset/getPathNames
* ============================================================================ */
export const GetApiDatasetPathNamesBodySchema = z.object({
datasetId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID,传入时从知识库配置中读取 apiDatasetServer'
}),
parentId: ParentIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '当前节点 ID,不传或 null 时返回空字符串'
}),
apiDatasetServer: ApiDatasetServerSchema.optional().meta({
description: '第三方知识库服务器配置,datasetId 不传时必须提供'
})
});
export type GetApiDatasetPathNamesBody = z.infer<typeof GetApiDatasetPathNamesBodySchema>;
export const GetApiDatasetPathNamesResponseSchema = z.string().meta({
example: '/根目录/产品文档/介绍',
description: '节点的完整路径字符串'
});
export type GetApiDatasetPathNamesResponse = z.infer<typeof GetApiDatasetPathNamesResponseSchema>;
/* ============================================================================
* API: 获取第三方知识库文件列表
* Route: POST /api/core/dataset/apiDataset/list
* ============================================================================ */
export const GetApiDatasetFileListBodySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
searchKey: z.string().optional().meta({
example: '产品文档',
description: '搜索关键词'
}),
parentId: ParentIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '父级节点 ID,不传或 null 表示根目录'
})
});
export type GetApiDatasetFileListBody = z.infer<typeof GetApiDatasetFileListBodySchema>;
export const GetApiDatasetFileListResponseSchema = z.array(APIFileItemSchema).meta({
description: '文件/文件夹列表'
});
export type GetApiDatasetFileListResponse = z.infer<typeof GetApiDatasetFileListResponseSchema>;
/* ============================================================================
* API: 获取第三方知识库已存在的 apiFileId 列表
* Route: GET /api/core/dataset/apiDataset/listExistId
* ============================================================================ */
export const GetApiDatasetFileListExistIdQuerySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
})
});
export type GetApiDatasetFileListExistIdQuery = z.infer<
typeof GetApiDatasetFileListExistIdQuerySchema
>;
export const GetApiDatasetFileListExistIdResponseSchema = z.array(z.string()).meta({
description: '已存在集合对应的 apiFileId 列表'
});
export type GetApiDatasetFileListExistIdResponse = z.infer<
typeof GetApiDatasetFileListExistIdResponseSchema
>;
import type { OpenAPIPath } from '../../../type';
import { TagsMap } from '../../../tag';
import {
GetApiDatasetCatalogBodySchema,
GetApiDatasetFileListBodySchema,
GetApiDatasetFileListExistIdQuerySchema,
GetApiDatasetPathNamesBodySchema
} from './api';
export const ApiDatasetPath: OpenAPIPath = {
'/core/dataset/apiDataset/getCatalog': {
post: {
summary: '获取第三方知识库目录',
description:
'列出第三方知识库(API/飞书/语雀)的目录节点,仅返回包含子节点的文件夹,用于构建目录选择器',
tags: [TagsMap.datasetApiDataset],
requestBody: {
content: {
'application/json': {
schema: GetApiDatasetCatalogBodySchema
}
}
},
responses: {
200: {
description: '成功返回目录节点列表'
}
}
}
},
'/core/dataset/apiDataset/getPathNames': {
post: {
summary: '获取第三方知识库节点路径',
description: '根据节点 ID 沿父级链向上查找,拼接出完整路径字符串,用于面包屑或路径展示',
tags: [TagsMap.datasetApiDataset],
requestBody: {
content: {
'application/json': {
schema: GetApiDatasetPathNamesBodySchema
}
}
},
responses: {
200: {
description: '成功返回节点的完整路径'
}
}
}
},
'/core/dataset/apiDataset/list': {
post: {
summary: '获取第三方知识库文件列表',
description: '列出指定知识库下的第三方文件/文件夹,支持关键词与父级筛选',
tags: [TagsMap.datasetApiDataset],
requestBody: {
content: {
'application/json': {
schema: GetApiDatasetFileListBodySchema
}
}
},
responses: {
200: {
description: '成功返回文件/文件夹列表'
}
}
}
},
'/core/dataset/apiDataset/listExistId': {
get: {
summary: '获取已导入的第三方文件 ID 列表',
description: '返回指定知识库下已创建集合所对应的 apiFileId,用于导入时的去重判断',
tags: [TagsMap.datasetApiDataset],
requestParams: {
query: GetApiDatasetFileListExistIdQuerySchema
},
responses: {
200: {
description: '成功返回已存在的 apiFileId 列表'
}
}
}
}
};
import { ParentIdSchema } from '../../../../common/parentFolder/type'; import {
GetPathPropsSchema,
ParentIdSchema,
ParentTreePathItemSchema
} from '../../../../common/parentFolder/type';
import { ObjectIdSchema } from '../../../../common/type/mongo'; import { ObjectIdSchema } from '../../../../common/type/mongo';
import { OutLinkChatAuthSchema } from '../../../../support/permission/chat'; import { OutLinkChatAuthSchema } from '../../../../support/permission/chat';
import {
DatasetCollectionSyncResultEnum,
DatasetCollectionTypeEnum,
DatasetCollectionDataProcessModeEnum,
TrainingModeEnum
} from '../../../../core/dataset/constants';
import {
ChunkSettingsSchema,
DatasetCollectionItemSchema,
DatasetCollectionSchema
} from '../../../../core/dataset/type';
import { PermissionSchema } from '../../../../support/permission/controller';
import { PaginationResponseSchema, PaginationSchema } from '../../../api';
import z from 'zod'; import z from 'zod';
// ============= Scroll Collections ============= // ============= Scroll Collections =============
...@@ -62,3 +79,130 @@ const ChatExportSchema = OutLinkChatAuthSchema.extend({ ...@@ -62,3 +79,130 @@ const ChatExportSchema = OutLinkChatAuthSchema.extend({
export const ExportCollectionBodySchema = z.union([BasicExportSchema, ChatExportSchema]); export const ExportCollectionBodySchema = z.union([BasicExportSchema, ChatExportSchema]);
export type ExportCollectionBodyType = z.infer<typeof ExportCollectionBodySchema>; export type ExportCollectionBodyType = z.infer<typeof ExportCollectionBodySchema>;
// ============= Delete Collection =============
export const DeleteCollectionQuerySchema = z.object({
id: z.string().optional().meta({ description: '单个集合 ID(与 body.collectionIds 二选一)' })
});
export type DeleteCollectionQueryType = z.infer<typeof DeleteCollectionQuerySchema>;
export const DeleteCollectionBodySchema = z.object({
collectionIds: z.array(z.string()).optional().meta({ description: '集合 ID 列表' })
});
export type DeleteCollectionBodyType = z.infer<typeof DeleteCollectionBodySchema>;
// ============= Get Collection Detail =============
export const GetCollectionDetailQuerySchema = z.object({
id: z.string().meta({ description: '集合 ID' })
});
export type GetCollectionDetailQueryType = z.infer<typeof GetCollectionDetailQuerySchema>;
export const GetCollectionDetailResponseSchema = DatasetCollectionItemSchema.meta({
description: '集合详情'
});
export type GetCollectionDetailResponseType = z.infer<typeof GetCollectionDetailResponseSchema>;
// ============= List Collections V2 =============
export const ListCollectionV2BodySchema = PaginationSchema.extend({
datasetId: z.string().meta({ description: '数据集 ID' }),
parentId: z.string().nullable().optional().default(null).meta({ description: '父级目录 ID' }),
searchText: z.string().max(100).optional().default('').meta({ description: '搜索文本' }),
selectFolder: z.boolean().optional().default(false).meta({ description: '只返回文件夹' }),
filterTags: z.array(z.string()).optional().default([]).meta({ description: '过滤标签' }),
simple: z.boolean().optional().default(false).meta({ description: '简单模式(不统计数量)' })
});
export type ListCollectionV2BodyType = z.infer<typeof ListCollectionV2BodySchema>;
// ============= List Collections V2 Response =============
export const DatasetCollectionsListItemSchema = z.object({
_id: ObjectIdSchema.meta({ description: '集合 ID' }),
parentId: DatasetCollectionSchema.shape.parentId,
tmbId: DatasetCollectionSchema.shape.tmbId,
name: DatasetCollectionSchema.shape.name,
type: DatasetCollectionSchema.shape.type,
createTime: DatasetCollectionSchema.shape.createTime,
updateTime: DatasetCollectionSchema.shape.updateTime,
forbid: DatasetCollectionSchema.shape.forbid,
trainingType: DatasetCollectionSchema.shape.trainingType,
tags: z.array(z.string()).optional().meta({ description: '标签' }),
externalFileId: z.string().optional().meta({ description: '外部文件 ID' }),
fileId: z.string().optional().meta({ description: '文件 ID' }),
rawLink: z.string().optional().meta({ description: '原始链接' }),
permission: PermissionSchema,
dataAmount: z.number().meta({ description: '数据数量' }),
trainingAmount: z.number().meta({ description: '训练数量' }),
hasError: z.boolean().optional().meta({ description: '是否错误' })
});
export type DatasetCollectionsListItemType = z.infer<typeof DatasetCollectionsListItemSchema>;
export const ListCollectionV2ResponseSchema = PaginationResponseSchema(
DatasetCollectionsListItemSchema
);
export type ListCollectionV2ResponseType = z.infer<typeof ListCollectionV2ResponseSchema>;
// ============= Get Collection Paths =============
export const GetCollectionPathsQuerySchema = GetPathPropsSchema;
export type GetCollectionPathsQueryType = z.infer<typeof GetCollectionPathsQuerySchema>;
export const GetCollectionPathsResponseSchema = z.array(ParentTreePathItemSchema);
export type GetCollectionPathsResponseType = z.infer<typeof GetCollectionPathsResponseSchema>;
// ============= Read Collection Source =============
export const ReadCollectionSourceBodySchema = OutLinkChatAuthSchema.extend({
collectionId: ObjectIdSchema.meta({ description: '集合 ID' }),
appId: ObjectIdSchema.optional().meta({ description: '应用 ID(对话中使用)' }),
chatId: ObjectIdSchema.optional().meta({ description: '对话 ID(对话中使用)' }),
chatItemDataId: z.string().optional().meta({ description: '对话消息 ID(对话中使用)' })
});
export type ReadCollectionSourceBodyType = z.infer<typeof ReadCollectionSourceBodySchema>;
export const ReadCollectionSourceResponseSchema = z.object({
type: z.literal('url').meta({ description: '资源类型' }),
value: z.string().meta({ description: '资源 URL' })
});
export type ReadCollectionSourceResponseType = z.infer<typeof ReadCollectionSourceResponseSchema>;
// ============= Training Detail =============
export const GetCollectionTrainingDetailQuerySchema = z.object({
collectionId: z.string().meta({ description: '集合 ID' })
});
export type GetCollectionTrainingDetailQueryType = z.infer<
typeof GetCollectionTrainingDetailQuerySchema
>;
const TrainingCountsSchema = z
.record(z.enum(TrainingModeEnum), z.number())
.meta({ description: '各训练模式数量' });
export const GetCollectionTrainingDetailResponseSchema = z.object({
trainingType: z
.enum(DatasetCollectionDataProcessModeEnum)
.optional()
.meta({ description: '训练类型' }),
advancedTraining: z
.object({
customPdfParse: z.boolean().meta({ description: '自定义 PDF 解析' }),
imageIndex: z.boolean().meta({ description: '图片索引' }),
autoIndexes: z.boolean().meta({ description: '自动索引' })
})
.meta({ description: '高级训练配置' }),
queuedCounts: TrainingCountsSchema.meta({ description: '排队中数量' }),
trainingCounts: TrainingCountsSchema.meta({ description: '训练中数量' }),
errorCounts: TrainingCountsSchema.meta({ description: '错误数量' }),
trainedCount: z.number().meta({ description: '已训练数据量' })
});
export type GetCollectionTrainingDetailResponseType = z.infer<
typeof GetCollectionTrainingDetailResponseSchema
>;
// ============= Sync Collection =============
export const SyncCollectionBodySchema = z.object({
collectionId: ObjectIdSchema.meta({ description: '集合 ID' })
});
export type SyncCollectionBodyType = z.infer<typeof SyncCollectionBodySchema>;
export const SyncCollectionResponseSchema = z.enum(DatasetCollectionSyncResultEnum).meta({
description: '同步结果'
});
export type SyncCollectionResponseType = z.infer<typeof SyncCollectionResponseSchema>;
import { TagsMap } from '../../../tag';
import type { OpenAPIPath } from '../../../type';
import {
CreateApiCollectionBodySchema,
CreateApiCollectionV2BodySchema,
CreateBackupCollectionMultipartSchema,
CreateCollectionBodySchema,
CreateCollectionByFileIdBodySchema,
CreateCollectionByLocalFileFormSchema,
CreateImageCollectionMultipartSchema,
CreateLinkCollectionBodySchema,
CreateTemplateCollectionMultipartSchema,
CreateTextCollectionBodySchema,
ReTrainingCollectionBodySchema
} from './createApi';
export const DatasetCollectionCreatePath: OpenAPIPath = {
/* ============================================================
* 通用创建(直接写入集合记录,不触发训练)
* ============================================================ */
'/core/dataset/collection/create': {
post: {
summary: '创建空集合/目录',
description: '创建空数据集合或者目录',
tags: [TagsMap.datasetCollectionCrteate],
requestBody: {
content: {
'application/json': {
schema: CreateCollectionBodySchema
}
}
},
responses: {
200: {
description: '成功返回新创建的集合 ID'
}
}
}
},
/* ============================================================
* 重新训练已有集合
* ============================================================ */
'/core/dataset/collection/create/reTrainingCollection': {
post: {
summary: '重新训练集合',
description: '删除原集合并以新参数重新创建并训练,适用于调整分块策略后的重处理场景',
tags: [TagsMap.datasetCollectionCrteate],
requestBody: {
content: {
'application/json': {
schema: ReTrainingCollectionBodySchema
}
}
},
responses: {
200: {
description: '成功返回新集合 ID'
}
}
}
},
/* ============================================================
* 通过已上传的文件 ID 创建集合
* ============================================================ */
'/core/dataset/collection/create/fileId': {
post: {
summary: '通过文件 ID 创建集合',
description: '使用已上传至 S3 的文件对象键创建集合并触发训练',
tags: [TagsMap.datasetCollectionCrteate],
requestBody: {
content: {
'application/json': {
schema: CreateCollectionByFileIdBodySchema
}
}
},
responses: {
200: {
description: '成功返回集合 ID 及数据插入结果'
}
}
}
},
/* ============================================================
* 上传本地文件创建集合(multipart/form-data)
* ============================================================ */
'/core/dataset/collection/create/localFile': {
post: {
summary: '上传本地文件创建集合',
description:
'通过 multipart/form-data 上传文件,自动存储至 S3 后创建集合并触发训练。`file` 字段为二进制文件,`data` 字段为 JSON 序列化的集合参数对象',
tags: [TagsMap.datasetCollectionCrteate],
requestBody: {
content: {
'multipart/form-data': {
schema: CreateCollectionByLocalFileFormSchema,
encoding: {
data: { contentType: 'application/json' }
}
}
}
},
responses: {
200: {
description: '成功返回集合 ID 及数据插入结果'
}
}
}
},
/* ============================================================
* 通过链接创建集合
* ============================================================ */
'/core/dataset/collection/create/link': {
post: {
summary: '通过链接创建集合',
description: '抓取指定 URL 内容创建集合并触发训练',
tags: [TagsMap.datasetCollectionCrteate],
requestBody: {
content: {
'application/json': {
schema: CreateLinkCollectionBodySchema
}
}
},
responses: {
200: {
description: '成功返回集合 ID 及数据插入结果'
}
}
}
},
/* ============================================================
* 通过文本创建集合
* ============================================================ */
'/core/dataset/collection/create/text': {
post: {
summary: '通过文本创建集合',
description: '将文本内容存储为文件后创建集合并触发训练',
tags: [TagsMap.datasetCollectionCrteate],
requestBody: {
content: {
'application/json': {
schema: CreateTextCollectionBodySchema
}
}
},
responses: {
200: {
description: '成功返回集合 ID 及数据插入结果'
}
}
}
},
/* ============================================================
* 通过 API 数据集创建集合(V1,单文件)
* ============================================================ */
'/core/dataset/collection/create/apiCollection': {
post: {
summary: '通过 API 数据集创建集合(V1)',
description: '根据 apiFileId 从第三方 API 数据源拉取单个文件并创建集合',
deprecated: true,
tags: [TagsMap.datasetCollectionCrteate],
requestBody: {
content: {
'application/json': {
schema: CreateApiCollectionBodySchema
}
}
},
responses: {
200: {
description: '成功创建集合'
}
}
}
},
/* ============================================================
* 通过 API 数据集创建集合(V2,批量/文件夹递归)
* ============================================================ */
'/core/dataset/collection/create/apiCollectionV2': {
post: {
summary: '通过 API 数据集批量创建集合(V2)',
description: '支持传入文件列表或选择根目录,递归拉取 API 数据源文件并批量创建集合',
tags: [TagsMap.datasetCollectionCrteate],
requestBody: {
content: {
'application/json': {
schema: CreateApiCollectionV2BodySchema
}
}
},
responses: {
200: {
description: '成功批量创建集合'
}
}
}
},
/* ============================================================
* 上传图片集创建集合(multipart/form-data)
* ============================================================ */
'/core/dataset/collection/create/images': {
post: {
summary: '上传图片集创建集合',
description:
'通过 multipart/form-data 批量上传图片,使用 VLM 模型解析后创建集合。`file` 为多个图片文件(多选),`data` 为 JSON 序列化的集合参数对象',
tags: [TagsMap.datasetCollectionCrteate],
requestBody: {
content: {
'multipart/form-data': {
schema: CreateImageCollectionMultipartSchema,
encoding: {
data: { contentType: 'application/json' }
}
}
}
},
responses: {
200: {
description: '成功返回集合 ID 及数据插入结果'
}
}
}
},
/* ============================================================
* 导入备份 CSV 文件创建集合(multipart/form-data)
* ============================================================ */
'/core/dataset/collection/create/backup': {
post: {
summary: '导入备份 CSV 创建集合',
description:
'上传格式为 q,a,indexes 的 CSV 备份文件,恢复数据到知识库集合。`file` 为 CSV 文件,`data` 为 JSON 序列化的集合参数对象',
tags: [TagsMap.datasetCollectionCrteate],
requestBody: {
content: {
'multipart/form-data': {
schema: CreateBackupCollectionMultipartSchema,
encoding: {
data: { contentType: 'application/json' }
}
}
}
},
responses: {
200: {
description: '成功导入备份数据'
}
}
}
},
/* ============================================================
* 导入模板 CSV 文件创建集合(multipart/form-data)
* ============================================================ */
'/core/dataset/collection/create/template': {
post: {
summary: '导入模板 CSV 创建集合',
description:
'上传格式为 q,a,indexes 的 CSV 模板文件,批量导入数据到知识库集合。`file` 为 CSV 文件,`data` 为 JSON 序列化的集合参数对象',
tags: [TagsMap.datasetCollectionCrteate],
requestBody: {
content: {
'multipart/form-data': {
schema: CreateTemplateCollectionMultipartSchema,
encoding: {
data: { contentType: 'application/json' }
}
}
}
},
responses: {
200: {
description: '成功导入模板数据'
}
}
}
}
};
import type { OpenAPIPath } from '../../../type'; import type { OpenAPIPath } from '../../../type';
import { TagsMap } from '../../../tag'; import { TagsMap } from '../../../tag';
import { import {
DeleteCollectionBodySchema,
DeleteCollectionQuerySchema,
ExportCollectionBodySchema, ExportCollectionBodySchema,
GetCollectionDetailQuerySchema,
GetCollectionPathsQuerySchema,
GetCollectionTrainingDetailQuerySchema,
GetCollectionTrainingDetailResponseSchema,
ListCollectionV2BodySchema,
ReadCollectionSourceBodySchema,
ScrollCollectionsBodySchema, ScrollCollectionsBodySchema,
SyncCollectionBodySchema,
UpdateDatasetCollectionBodySchema UpdateDatasetCollectionBodySchema
} from './api'; } from './api';
import { DatasetCollectionCreatePath } from './createPath';
export const DatasetCollectionPath: OpenAPIPath = { export const DatasetCollectionPath: OpenAPIPath = {
...DatasetCollectionCreatePath,
'/core/dataset/collection/delete': {
delete: {
summary: '删除集合',
description: '删除一个或多个集合及其子集合,支持通过 query.id 或 body.collectionIds 指定',
tags: [TagsMap.datasetCollection],
requestParams: {
query: DeleteCollectionQuerySchema
},
requestBody: {
content: {
'application/json': {
schema: DeleteCollectionBodySchema
}
}
},
responses: {
200: {
description: '成功删除集合'
}
}
}
},
'/core/dataset/collection/detail': {
get: {
summary: '获取集合详情',
description: '获取集合详细信息,包括索引数量、错误数量、文件信息等',
tags: [TagsMap.datasetCollection],
requestParams: {
query: GetCollectionDetailQuerySchema
},
responses: {
200: {
description: '成功返回集合详情'
}
}
}
},
'/core/dataset/collection/listV2': {
post: {
summary: '获取集合列表(分页)',
description: '获取数据集集合列表,支持分页、搜索、标签过滤',
tags: [TagsMap.datasetCollection],
requestBody: {
content: {
'application/json': {
schema: ListCollectionV2BodySchema
}
}
},
responses: {
200: {
description: '成功返回集合列表和总数'
}
}
}
},
'/core/dataset/collection/scrollList': { '/core/dataset/collection/scrollList': {
post: { post: {
summary: '获取数据集集合列表(滚动分页)', summary: '获取数据集集合列表(滚动分页)',
...@@ -45,6 +112,59 @@ export const DatasetCollectionPath: OpenAPIPath = { ...@@ -45,6 +112,59 @@ export const DatasetCollectionPath: OpenAPIPath = {
} }
} }
}, },
'/core/dataset/collection/paths': {
get: {
summary: '获取集合面包屑路径',
description: '从指定集合向上递归获取父级路径链,用于面包屑导航',
tags: [TagsMap.datasetCollection],
requestParams: {
query: GetCollectionPathsQuerySchema
},
responses: {
200: {
description: '成功返回路径列表'
}
}
}
},
'/core/dataset/collection/read': {
post: {
summary: '获取集合资源 URL',
description: '获取集合原始文件的访问 URL,支持直接鉴权和对话中鉴权两种模式',
tags: [TagsMap.datasetCollection],
requestBody: {
content: {
'application/json': {
schema: ReadCollectionSourceBodySchema
}
}
},
responses: {
200: {
description: '成功返回资源 URL'
}
}
}
},
'/core/dataset/collection/sync': {
post: {
summary: '同步集合',
description: '重新拉取集合原始内容并更新数据,支持链接类型和 API 数据集类型',
tags: [TagsMap.datasetCollection],
requestBody: {
content: {
'application/json': {
schema: SyncCollectionBodySchema
}
}
},
responses: {
200: {
description: '成功返回同步结果(success / sameRaw / failed)'
}
}
}
},
'/core/dataset/collection/export': { '/core/dataset/collection/export': {
post: { post: {
summary: '下载集合的所有数据块', summary: '下载集合的所有数据块',
...@@ -63,5 +183,25 @@ export const DatasetCollectionPath: OpenAPIPath = { ...@@ -63,5 +183,25 @@ export const DatasetCollectionPath: OpenAPIPath = {
} }
} }
} }
},
'/core/dataset/collection/trainingDetail': {
get: {
summary: '获取集合训练详情',
description: '获取集合的训练状态,包括排队中、训练中、错误数量及已完成的数据量',
tags: [TagsMap.datasetCollection],
requestParams: {
query: GetCollectionTrainingDetailQuerySchema
},
responses: {
200: {
description: '成功返回训练详情',
content: {
'application/json': {
schema: GetCollectionTrainingDetailResponseSchema
}
}
}
}
}
} }
}; };
import z from 'zod';
/* ============================================================================
* API: 创建集合标签
* Route: POST /proApi/core/dataset/tag/create
* ============================================================================ */
export const CreateDatasetCollectionTagBodySchema = z.object({
datasetId: z.string().meta({ description: '数据集 ID' }),
tag: z.string().meta({ description: '标签名称' })
});
export type CreateDatasetCollectionTagParams = z.infer<typeof CreateDatasetCollectionTagBodySchema>;
/* ============================================================================
* API: 批量为集合添加标签
* Route: POST /proApi/core/dataset/tag/addToCollections
* ============================================================================ */
export const AddTagsToCollectionsBodySchema = z.object({
originCollectionIds: z
.array(z.string())
.meta({ description: '来源集合 ID 列表(用于复制标签)' }),
collectionIds: z.array(z.string()).meta({ description: '目标集合 ID 列表' }),
datasetId: z.string().meta({ description: '数据集 ID' }),
tag: z.string().meta({ description: '标签名称' })
});
export type AddTagsToCollectionsParams = z.infer<typeof AddTagsToCollectionsBodySchema>;
/* ============================================================================
* API: 更新集合标签
* Route: POST /proApi/core/dataset/tag/update
* ============================================================================ */
export const UpdateDatasetCollectionTagBodySchema = z.object({
datasetId: z.string().meta({ description: '数据集 ID' }),
tagId: z.string().meta({ description: '标签 ID' }),
tag: z.string().meta({ description: '新标签名称' })
});
export type UpdateDatasetCollectionTagParams = z.infer<typeof UpdateDatasetCollectionTagBodySchema>;
import { z } from 'zod'; import { z } from 'zod';
import { ObjectIdSchema } from '../../../../common/type/mongo';
import {
DatasetCollectionSchema,
DatasetDataIndexItemSchema,
DatasetDataItemSchema,
UpdateDatasetDataPropsSchema
} from '../../../../core/dataset/type';
import { DatasetCollectionDataProcessModeEnum } from '../../../../core/dataset/constants';
import { OutLinkChatAuthSchema } from '../../../../support/permission/chat';
import { PaginationSchema, PaginationResponseSchema } from '../../../api';
const PushDataChunkSchema = z.object({
q: z.string().optional().meta({
example: '什么是 FastGPT?',
description: '问题/主文本'
}),
a: z.string().optional().meta({
description: '回答/补充文本'
}),
imageId: z.string().optional().meta({
description: '图片 ID'
}),
chunkIndex: z.number().optional().meta({
example: 0,
description: '块索引'
}),
indexes: z
.array(DatasetDataIndexItemSchema.omit({ dataId: true }))
.optional()
.meta({ description: '额外向量索引' })
});
export type PushDataChunkType = z.infer<typeof PushDataChunkSchema>;
/* ============================================================================
* API: 获取数据集数据详情
* Route: GET /api/core/dataset/data/detail
* ============================================================================ */
export const GetDatasetDataDetailQuerySchema = z.object({
id: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '数据 ID'
})
});
export type GetDatasetDataDetailQuery = z.infer<typeof GetDatasetDataDetailQuerySchema>;
export const GetDatasetDataDetailResponseSchema = DatasetDataItemSchema;
export type GetDatasetDataDetailResponse = z.infer<typeof GetDatasetDataDetailResponseSchema>;
/* ============================================================================
* API: 更新数据集数据
* Route: PUT /api/core/dataset/data/update
* ============================================================================ */
export const UpdateDatasetDataBodySchema = UpdateDatasetDataPropsSchema;
export type UpdateDatasetDataBody = z.infer<typeof UpdateDatasetDataBodySchema>;
/* ============================================================================
* API: 删除数据集数据
* Route: DELETE /api/core/dataset/data/delete
* ============================================================================ */
export const DeleteDatasetDataQuerySchema = z.object({
id: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '数据 ID'
})
});
export type DeleteDatasetDataQuery = z.infer<typeof DeleteDatasetDataQuerySchema>;
/* ============================================================================
* API: 获取引用数据
* Route: POST /api/core/dataset/data/getQuoteData
* ============================================================================ */
export const GetQuoteDataBodySchema = z.union([
z.object({
id: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '数据 ID'
})
}),
OutLinkChatAuthSchema.extend({
id: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '数据 ID'
}),
// 对话模式下的额外字段(可选)
appId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a10',
description: '应用 ID(对话模式必填)'
}),
chatId: z.string().meta({
example: '68ad85a7463006c963799a11',
description: '对话 ID(对话模式必填)'
}),
chatItemDataId: z.string().meta({
example: '68ad85a7463006c963799a12',
description: '对话条目数据 ID(对话模式必填)'
})
})
]);
export type GetQuoteDataBody = z.infer<typeof GetQuoteDataBodySchema>;
export const GetQuoteDataResponseSchema = z.object({
q: z.string().meta({
example: '什么是 FastGPT?',
description: '问题/主文本'
}),
a: z.string().optional().meta({
example: 'FastGPT 是一个 AI Agent 构建平台',
description: '回答/补充文本'
}),
collection: DatasetCollectionSchema.meta({
description: '所属集合信息'
})
});
export type GetQuoteDataResponse = z.infer<typeof GetQuoteDataResponseSchema>;
/* ============================================================================
* API: 插入单条数据
* Route: POST /api/core/dataset/data/insertData
* ============================================================================ */
export const InsertDataBodySchema = PushDataChunkSchema.omit({ q: true }).extend({
q: z.string().nonempty().meta({
example: '什么是 FastGPT?',
description: '问题/主文本'
}),
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
})
});
export type InsertDataBody = z.infer<typeof InsertDataBodySchema>;
export const InsertDataResponseSchema = ObjectIdSchema.meta({
example: '68ad85a7463006c963799a07',
description: '新插入的数据 ID'
});
export type InsertDataResponse = z.infer<typeof InsertDataResponseSchema>;
/* ============================================================================
* API: 插入图片
* Route: POST /api/core/dataset/data/insertImages (multipart/form-data)
* ============================================================================ */
export const InsertImagesBodySchema = z.object({
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
}),
file: z
.array(z.any())
.optional()
.meta({
description: '图片文件列表,multipart/form-data 上传,每个 item 为 binary 文件',
items: { type: 'string', format: 'binary' }
})
});
export type InsertImagesBody = z.infer<typeof InsertImagesBodySchema>;
export const InsertImagesResponseSchema = z.object({});
export type InsertImagesResponse = z.infer<typeof InsertImagesResponseSchema>;
/* ============================================================================
* API: 推送数据到训练队列
* Route: POST /api/core/dataset/data/pushData
* ============================================================================ */
export const PushDataBodySchema = z.object({
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
}),
data: z.array(PushDataChunkSchema).max(200).meta({
description: '数据列表,最多 200 条'
}),
trainingType: z.enum(DatasetCollectionDataProcessModeEnum).optional().meta({
description: '训练类型'
}),
indexSize: z.number().optional().meta({
description: '索引大小限制'
}),
autoIndexes: z.boolean().optional().meta({
description: '是否自动生成索引'
}),
imageIndex: z.boolean().optional().meta({
description: '是否生成图片索引'
}),
prompt: z.string().optional().meta({
description: '自定义提示词'
}),
billId: z.string().optional().meta({
description: '账单 ID'
}),
trainingMode: z.enum(DatasetCollectionDataProcessModeEnum).optional().meta({
description: '训练类型',
deprecated: true
})
});
export type PushDataBody = z.infer<typeof PushDataBodySchema>;
export const PushDataResponseSchema = z.object({
insertLen: z.number().meta({
example: 10,
description: '成功插入的数据条数'
})
});
export type PushDataResponseType = z.infer<typeof PushDataResponseSchema>;
/* ============================================================================
* API: 获取数据列表 V2(推荐)
* Route: POST /api/core/dataset/data/v2/list
* ============================================================================ */
export const GetDataListItemSchema = z.object({
_id: ObjectIdSchema.meta({ description: '数据 ID' }),
datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }),
collectionId: ObjectIdSchema.meta({ description: '集合 ID' }),
q: z.string().optional().meta({ description: '问题/主文本' }),
a: z.string().optional().meta({ description: '回答/补充文本' }),
imageId: z.string().optional().meta({ description: '图片 ID' }),
imageSize: z.number().optional().meta({ description: '图片大小(字节)' }),
imagePreviewUrl: z.string().optional().meta({ description: '图片预览 URL' }),
chunkIndex: z.number().optional().meta({ description: '块索引' }),
updated: z.boolean().optional().meta({ description: '是否已更新' })
});
export const GetDatasetDataListBodySchema = PaginationSchema.extend({
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
}),
searchText: z.string().optional().meta({
example: 'FastGPT',
description: '搜索关键词,按 q/a 字段模糊匹配'
})
});
export type GetDatasetDataListBody = z.infer<typeof GetDatasetDataListBodySchema>;
export const GetDatasetDataListResponseSchema = PaginationResponseSchema(GetDataListItemSchema);
export type GetDatasetDataListResponse = z.infer<typeof GetDatasetDataListResponseSchema>;
/* ============================================================================
* API: 获取数据列表(已废弃,使用 v2/list)
* Route: POST /api/core/dataset/data/list
* @deprecated
* ============================================================================ */
export const GetDatasetDataListLegacyBodySchema = PaginationSchema.extend({
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
}),
searchText: z.string().optional().meta({
example: 'FastGPT',
description: '搜索关键词'
})
});
export type GetDatasetDataListLegacyBody = z.infer<typeof GetDatasetDataListLegacyBodySchema>;
import type { OpenAPIPath } from '../../../type'; import type { OpenAPIPath } from '../../../type';
import { TagsMap } from '../../../tag'; import { TagsMap } from '../../../tag';
import {
GetDatasetDataDetailQuerySchema,
UpdateDatasetDataBodySchema,
DeleteDatasetDataQuerySchema,
GetQuoteDataBodySchema,
InsertDataBodySchema,
InsertImagesBodySchema,
PushDataBodySchema,
GetDatasetDataListBodySchema,
GetDatasetDataListResponseSchema
} from './api';
export const DatasetDataPath: OpenAPIPath = {}; export const DatasetDataPath: OpenAPIPath = {
'/core/dataset/data/v2/list': {
post: {
summary: '获取数据列表',
description: '分页查询集合内的数据列表,支持关键词搜索,包含图片预览 URL',
tags: [TagsMap.datasetData],
requestBody: {
content: {
'application/json': {
schema: GetDatasetDataListBodySchema
}
}
},
responses: {
200: {
description: '成功返回分页数据列表',
content: {
'application/json': {
schema: GetDatasetDataListResponseSchema
}
}
}
}
}
},
'/core/dataset/data/detail': {
get: {
summary: '获取数据详情',
description: '获取单条数据集数据的详细信息,包括向量索引',
tags: [TagsMap.datasetData],
requestParams: {
query: GetDatasetDataDetailQuerySchema
},
responses: {
200: {
description: '成功返回数据详情'
}
}
}
},
'/core/dataset/data/update': {
put: {
summary: '更新数据',
description: '更新数据集数据的 q、a 和向量索引,触发重新向量化',
tags: [TagsMap.datasetData],
requestBody: {
content: {
'application/json': {
schema: UpdateDatasetDataBodySchema
}
}
},
responses: {
200: {
description: '更新成功'
}
}
}
},
'/core/dataset/data/delete': {
delete: {
summary: '删除数据',
description: '删除指定数据集数据,需要写权限',
tags: [TagsMap.datasetData],
requestParams: {
query: DeleteDatasetDataQuerySchema
},
responses: {
200: {
description: '删除成功'
}
}
}
},
'/core/dataset/data/getQuoteData': {
post: {
summary: '获取引用数据',
description: '获取数据详情用于展示引用,支持直接访问或通过对话鉴权',
tags: [TagsMap.datasetData],
requestBody: {
content: {
'application/json': {
schema: GetQuoteDataBodySchema
}
}
},
responses: {
200: {
description: '成功返回引用数据和所属集合信息'
}
}
}
},
'/core/dataset/data/insertData': {
post: {
summary: '插入单条数据',
description: '立即插入一条数据到数据集并生成向量索引',
tags: [TagsMap.datasetData],
requestBody: {
content: {
'application/json': {
schema: InsertDataBodySchema
}
}
},
responses: {
200: {
description: '成功返回新数据的 ID'
}
}
}
},
'/core/dataset/data/insertImages': {
post: {
summary: '插入图片',
description: '上传图片文件并推送到训练队列(multipart/form-data)',
tags: [TagsMap.datasetData],
requestBody: {
content: {
'multipart/form-data': {
schema: InsertImagesBodySchema
}
}
},
responses: {
200: {
description: '上传成功,图片已加入训练队列'
}
}
}
},
'/core/dataset/data/pushData': {
post: {
summary: '推送数据到训练队列',
description: '批量推送数据到训练队列,最多 200 条',
tags: [TagsMap.datasetData],
requestBody: {
content: {
'application/json': {
schema: PushDataBodySchema
}
}
},
responses: {
200: {
description: '成功返回插入条数'
}
}
}
}
};
import { z } from 'zod';
import { ObjectIdSchema } from '../../../../common/type/mongo';
import { DatasetSourceReadTypeEnum } from '../../../../core/dataset/constants';
import { ChunkSettingsSchema } from '../../../../core/dataset/type';
import { CreatePostPresignedUrlResponseSchema } from '../../../../common/file/s3/type';
/* ============================================================================
* API: 预览文件分块
* Route: POST /api/core/dataset/file/getPreviewChunks
* ============================================================================ */
export const GetPreviewChunksBodySchema = ChunkSettingsSchema.extend({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
type: z.enum(DatasetSourceReadTypeEnum).meta({
example: DatasetSourceReadTypeEnum.fileLocal,
description: '数据源读取类型'
}),
sourceId: z.string().nonempty().meta({
example: '68ad85a7463006c963799a05',
description: '数据源 ID(文件 ID / 链接 / 外部文件 / API 文件等)'
}),
customPdfParse: z.boolean().optional().meta({
description: '是否启用自定义 PDF 解析'
}),
overlapRatio: z.number().meta({
example: 0.2,
description: '分块重叠比例'
}),
selector: z.string().optional().meta({
example: 'body',
description: '网页抓取的 CSS 选择器'
}),
externalFileId: z.string().optional().meta({
description: '外部文件标识'
})
});
export type GetPreviewChunksBody = z.infer<typeof GetPreviewChunksBodySchema>;
const PreviewChunkItemSchema = z.object({
q: z.string().meta({ description: '主要文本' }),
a: z.string().meta({ description: '辅助文本' })
});
export const GetPreviewChunksResponseSchema = z.object({
chunks: z.array(PreviewChunkItemSchema).meta({
description: '预览分块列表(最多 10 条)'
}),
total: z.number().meta({
example: 42,
description: '分块总数'
})
});
export type GetPreviewChunksResponse = z.infer<typeof GetPreviewChunksResponseSchema>;
/* ============================================================================
* API: 获取知识库文件上传预签名 URL
* Route: POST /api/core/dataset/file/presignDatasetFilePostUrl
* ============================================================================ */
export const PresignDatasetFilePostUrlBodySchema = z.object({
filename: z.string().min(1).meta({
example: '产品文档.pdf',
description: '待上传的文件名,不能为空'
}),
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '目标知识库 ID'
})
});
export type PresignDatasetFilePostUrlBody = z.infer<typeof PresignDatasetFilePostUrlBodySchema>;
export const PresignDatasetFilePostUrlResponseSchema = CreatePostPresignedUrlResponseSchema.meta({
description: 'S3 预签名上传 URL 及相关头信息'
});
export type PresignDatasetFilePostUrlResponse = z.infer<
typeof PresignDatasetFilePostUrlResponseSchema
>;
import type { OpenAPIPath } from '../../../type';
import { TagsMap } from '../../../tag';
import {
GetPreviewChunksBodySchema,
GetPreviewChunksResponseSchema,
PresignDatasetFilePostUrlBodySchema,
PresignDatasetFilePostUrlResponseSchema
} from './api';
export const DatasetFilePath: OpenAPIPath = {
'/core/dataset/file/getPreviewChunks': {
post: {
summary: '预览文件分块',
description: '读取数据源并按给定分块参数预览生成的前 10 个分块,用于导入前校验',
tags: [TagsMap.datasetFile],
requestBody: {
content: {
'application/json': {
schema: GetPreviewChunksBodySchema
}
}
},
responses: {
200: {
description: '成功返回预览分块列表及总数',
content: {
'application/json': {
schema: GetPreviewChunksResponseSchema
}
}
}
}
}
},
'/core/dataset/file/presignDatasetFilePostUrl': {
post: {
summary: '获取知识库文件上传预签名 URL',
description: '为指定知识库生成 S3 上传预签名 URL,同时校验写权限并对上传频率进行限制',
tags: [TagsMap.datasetFile],
requestBody: {
content: {
'application/json': {
schema: PresignDatasetFilePostUrlBodySchema
}
}
},
responses: {
200: {
description: '成功返回预签名上传 URL、key、请求头和最大文件大小',
content: {
'application/json': {
schema: PresignDatasetFilePostUrlResponseSchema
}
}
}
}
}
}
};
...@@ -2,6 +2,9 @@ import type { OpenAPIPath } from '../../type'; ...@@ -2,6 +2,9 @@ import type { OpenAPIPath } from '../../type';
import { TagsMap } from '../../tag'; import { TagsMap } from '../../tag';
import { DatasetDataPath } from './data'; import { DatasetDataPath } from './data';
import { DatasetCollectionPath } from './collection'; import { DatasetCollectionPath } from './collection';
import { ApiDatasetPath } from './apiDataset';
import { DatasetFilePath } from './file';
import { DatasetTrainingPath } from './training';
import { import {
CreateDatasetBodySchema, CreateDatasetBodySchema,
CreateDatasetWithFilesBodySchema, CreateDatasetWithFilesBodySchema,
...@@ -13,7 +16,8 @@ import { ...@@ -13,7 +16,8 @@ import {
ResumeDatasetInheritPermissionBodySchema, ResumeDatasetInheritPermissionBodySchema,
CreateDatasetFolderBodySchema, CreateDatasetFolderBodySchema,
SearchDatasetTestBodySchema, SearchDatasetTestBodySchema,
ExportDatasetQuerySchema ExportDatasetQuerySchema,
GetDatasetPermissionQuerySchema
} from './api'; } from './api';
export const DatasetPath: OpenAPIPath = { export const DatasetPath: OpenAPIPath = {
...@@ -123,7 +127,6 @@ export const DatasetPath: OpenAPIPath = { ...@@ -123,7 +127,6 @@ export const DatasetPath: OpenAPIPath = {
} }
} }
}, },
'/core/dataset/delete': { '/core/dataset/delete': {
delete: { delete: {
summary: '删除知识库', summary: '删除知识库',
...@@ -177,7 +180,6 @@ export const DatasetPath: OpenAPIPath = { ...@@ -177,7 +180,6 @@ export const DatasetPath: OpenAPIPath = {
} }
} }
}, },
'/core/dataset/searchTest': { '/core/dataset/searchTest': {
post: { post: {
summary: '搜索测试', summary: '搜索测试',
...@@ -197,7 +199,6 @@ export const DatasetPath: OpenAPIPath = { ...@@ -197,7 +199,6 @@ export const DatasetPath: OpenAPIPath = {
} }
} }
}, },
'/core/dataset/exportAll': { '/core/dataset/exportAll': {
get: { get: {
summary: '导出知识库全部数据', summary: '导出知识库全部数据',
...@@ -213,7 +214,25 @@ export const DatasetPath: OpenAPIPath = { ...@@ -213,7 +214,25 @@ export const DatasetPath: OpenAPIPath = {
} }
} }
}, },
'/core/dataset/getPermission': {
get: {
summary: '获取知识库引用权限',
description: '获取当前用户对指定知识库的读写权限,鉴权失败时返回无权限结果而非报错',
tags: [TagsMap.datasetCommon],
requestParams: {
query: GetDatasetPermissionQuerySchema
},
responses: {
200: {
description: '成功返回知识库名称和读写权限,无访问权限时返回空名称和 false'
}
}
}
},
...DatasetCollectionPath, ...DatasetCollectionPath,
...DatasetDataPath ...DatasetDataPath,
...ApiDatasetPath,
...DatasetFilePath,
...DatasetTrainingPath
}; };
import { z } from 'zod';
import { ObjectIdSchema } from '../../../../common/type/mongo';
import { TrainingModeEnum } from '../../../../core/dataset/constants';
import { DatasetTrainingSchema } from '../../../../core/dataset/type';
import { PaginationSchema, PaginationResponseSchema } from '../../../api';
/* ============================================================================
* API: 更新训练数据(或重试所有错误数据)
* Route: PUT /api/core/dataset/training/updateTrainingData
* ============================================================================ */
export const UpdateTrainingDataBodySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
}),
dataId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a07',
description: '训练数据 ID,不传则重试集合内所有错误数据'
}),
q: z.string().optional().meta({
example: '什么是 FastGPT?',
description: '问题/主文本'
}),
a: z.string().optional().meta({
example: 'FastGPT 是一个 AI Agent 构建平台',
description: '回答/补充文本'
}),
chunkIndex: z.int().min(0).optional().meta({
example: 0,
description: '块索引'
})
});
export type UpdateTrainingDataBody = z.infer<typeof UpdateTrainingDataBodySchema>;
export const UpdateTrainingDataResponseSchema = z.object({});
export type UpdateTrainingDataResponse = z.infer<typeof UpdateTrainingDataResponseSchema>;
/* ============================================================================
* API: 重建数据集向量索引
* Route: POST /api/core/dataset/training/rebuildEmbedding
* ============================================================================ */
export const RebuildEmbeddingBodySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
vectorModel: z.string().meta({
example: 'text-embedding-3-small',
description: '新的向量模型名称,不能与当前模型相同'
})
});
export type RebuildEmbeddingBody = z.infer<typeof RebuildEmbeddingBodySchema>;
export const RebuildEmbeddingResponseSchema = z.object({});
export type RebuildEmbeddingResponse = z.infer<typeof RebuildEmbeddingResponseSchema>;
/* ============================================================================
* API: 删除训练数据
* Route: POST /api/core/dataset/training/deleteTrainingData
* ============================================================================ */
export const DeleteTrainingDataBodySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
}),
dataId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a07',
description: '训练数据 ID'
})
});
export type DeleteTrainingDataBody = z.infer<typeof DeleteTrainingDataBodySchema>;
export const DeleteTrainingDataResponseSchema = z.object({});
export type DeleteTrainingDataResponse = z.infer<typeof DeleteTrainingDataResponseSchema>;
/* ============================================================================
* API: 获取训练数据详情
* Route: POST /api/core/dataset/training/getTrainingDataDetail
* ============================================================================ */
export const GetTrainingDataDetailBodySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
}),
dataId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a07',
description: '训练数据 ID'
})
});
export type GetTrainingDataDetailBody = z.infer<typeof GetTrainingDataDetailBodySchema>;
export const GetTrainingDataDetailResponseSchema = z
.object({
_id: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a07',
description: '训练数据 ID'
}),
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
mode: z.enum(TrainingModeEnum).meta({
example: TrainingModeEnum.chunk,
description: '训练模式'
}),
q: z.string().optional().meta({
example: '什么是 FastGPT?',
description: '问题/主文本'
}),
a: z.string().optional().meta({
example: 'FastGPT 是一个 AI Agent 构建平台',
description: '回答/补充文本'
}),
imagePreviewUrl: z.string().optional().meta({
example: 'https://example.com/image.png',
description: '图片预览 URL(S3 签名链接,有效期30分钟)'
})
})
.nullish()
.meta({ description: '训练数据详情,数据不存在时为 null' });
export type GetTrainingDataDetailResponse = z.infer<typeof GetTrainingDataDetailResponseSchema>;
/* ============================================================================
* API: 获取训练错误列表(分页)
* Route: POST /api/core/dataset/training/getTrainingError
* ============================================================================ */
export const GetTrainingErrorBodySchema = PaginationSchema.extend({
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
})
});
export type GetTrainingErrorBody = z.infer<typeof GetTrainingErrorBodySchema>;
export const GetTrainingErrorResponseSchema = PaginationResponseSchema(
DatasetTrainingSchema.omit({ billId: true }).extend({
billId: z.string().optional()
})
);
export type GetTrainingErrorResponse = z.infer<typeof GetTrainingErrorResponseSchema>;
/* ============================================================================
* API: 获取数据集训练队列状态
* Route: GET /api/core/dataset/training/getDatasetTrainingQueue
* ============================================================================ */
export const GetDatasetTrainingQueueQuerySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
})
});
export type GetDatasetTrainingQueueQuery = z.infer<typeof GetDatasetTrainingQueueQuerySchema>;
export const GetDatasetTrainingQueueResponseSchema = z.object({
rebuildingCount: z.number().meta({
example: 5,
description: '正在重建向量的数据条数'
}),
trainingCount: z.number().meta({
example: 12,
description: '训练队列中的数据条数'
})
});
export type GetDatasetTrainingQueueResponse = z.infer<typeof GetDatasetTrainingQueueResponseSchema>;
import type { OpenAPIPath } from '../../../type';
import { TagsMap } from '../../../tag';
import {
UpdateTrainingDataBodySchema,
RebuildEmbeddingBodySchema,
DeleteTrainingDataBodySchema,
GetTrainingDataDetailBodySchema,
GetTrainingErrorBodySchema,
GetDatasetTrainingQueueQuerySchema
} from './api';
export const DatasetTrainingPath: OpenAPIPath = {
'/core/dataset/training/updateTrainingData': {
put: {
summary: '更新训练数据',
description: '更新单条训练数据,或批量重试集合内所有错误数据(不传 dataId)',
tags: [TagsMap.datasetTraining],
requestBody: {
content: {
'application/json': {
schema: UpdateTrainingDataBodySchema
}
}
},
responses: {
200: {
description: '更新成功'
}
}
}
},
'/core/dataset/training/rebuildEmbedding': {
post: {
summary: '重建数据集向量索引',
description: '切换向量模型并重建知识库所有数据的向量索引,需要所有者权限',
tags: [TagsMap.datasetTraining],
requestBody: {
content: {
'application/json': {
schema: RebuildEmbeddingBodySchema
}
}
},
responses: {
200: {
description: '重建任务已启动'
}
}
}
},
'/core/dataset/training/deleteTrainingData': {
post: {
summary: '删除训练数据',
description: '删除指定的训练数据条目,需要管理权限',
tags: [TagsMap.datasetTraining],
requestBody: {
content: {
'application/json': {
schema: DeleteTrainingDataBodySchema
}
}
},
responses: {
200: {
description: '删除成功'
}
}
}
},
'/core/dataset/training/getTrainingDataDetail': {
post: {
summary: '获取训练数据详情',
description: '获取单条训练数据的详细信息,包括图片预览 URL',
tags: [TagsMap.datasetTraining],
requestBody: {
content: {
'application/json': {
schema: GetTrainingDataDetailBodySchema
}
}
},
responses: {
200: {
description: '成功返回训练数据详情,数据不存在时返回空'
}
}
}
},
'/core/dataset/training/getTrainingError': {
post: {
summary: '获取训练错误列表',
description: '分页查询集合内训练失败的数据列表',
tags: [TagsMap.datasetTraining],
requestBody: {
content: {
'application/json': {
schema: GetTrainingErrorBodySchema
}
}
},
responses: {
200: {
description: '成功返回错误数据分页列表'
}
}
}
},
'/core/dataset/training/getDatasetTrainingQueue': {
get: {
summary: '获取训练队列状态',
description: '获取知识库当前的重建数量和训练队列数量',
tags: [TagsMap.datasetTraining],
requestParams: {
query: GetDatasetTrainingQueueQuerySchema
},
responses: {
200: {
description: '成功返回重建数量和训练队列数量'
}
}
}
}
};
...@@ -54,7 +54,15 @@ export const openAPIDocument = createDocument({ ...@@ -54,7 +54,15 @@ export const openAPIDocument = createDocument({
}, },
{ {
name: '知识库', name: '知识库',
tags: [TagsMap.datasetCommon, TagsMap.datasetCollection] tags: [
TagsMap.datasetCommon,
TagsMap.datasetCollection,
TagsMap.datasetCollectionCrteate,
TagsMap.datasetData,
TagsMap.datasetFile,
TagsMap.datasetTraining,
TagsMap.datasetApiDataset
]
}, },
{ {
name: '插件系统', name: '插件系统',
......
import { UserInformPath } from './inform'; import { UserInformPath } from './inform';
import type { OpenAPIPath } from '../../type'; import type { OpenAPIPath } from '../../type';
import { UserAccountPath } from './account'; import { UserAccountPath } from './account';
import { TeamPath } from './team';
export const UserPath: OpenAPIPath = { export const UserPath: OpenAPIPath = {
...UserInformPath, ...UserInformPath,
...UserAccountPath ...UserAccountPath,
...TeamPath
}; };
import z from 'zod'; import z from 'zod';
import { LafAccountSchema, OpenaiAccountSchema } from '../../../../support/user/team/type';
export const TeamChangeOwnerBodySchema = z.object({ export const TeamChangeOwnerBodySchema = z.object({
userId: z.string().describe("the New Owner's UserId.") userId: z.string().describe("the New Owner's UserId.")
...@@ -8,3 +9,48 @@ export const TeamChangeOwnerResponseSchema = z.object(); ...@@ -8,3 +9,48 @@ export const TeamChangeOwnerResponseSchema = z.object();
export type TeamChangeOwnerBodyType = z.infer<typeof TeamChangeOwnerBodySchema>; export type TeamChangeOwnerBodyType = z.infer<typeof TeamChangeOwnerBodySchema>;
export type TeamChangeOwnerResponseType = z.infer<typeof TeamChangeOwnerResponseSchema>; export type TeamChangeOwnerResponseType = z.infer<typeof TeamChangeOwnerResponseSchema>;
/* ============================================================================
* API: 更新团队信息
* Route: POST /api/support/user/team/update
* ============================================================================ */
export const UpdateTeamBodySchema = z.object({
name: z.string().max(100).optional().meta({
example: '我的团队',
description: '团队名称'
}),
avatar: z.string().optional().meta({
description: '团队头像 URL'
}),
teamDomain: z.string().optional().meta({
description: '团队域名'
}),
lafAccount: LafAccountSchema.optional().meta({
description: 'Laf 账号配置'
}),
openaiAccount: OpenaiAccountSchema.optional().meta({
description: 'OpenAI 账号配置'
}),
externalWorkflowVariable: z
.object({
key: z
.string()
.regex(/^[a-zA-Z_]\w*$/, 'key 仅允许字母、数字、下划线,且不能以数字开头')
.meta({
example: 'myVar',
description: '变量名,仅允许字母、数字、下划线,且不以数字开头'
}),
value: z.string().meta({
example: 'myValue',
description: '变量值,为空字符串时删除该变量'
})
})
.optional()
.meta({
description: '外部工作流变量(单次更新一个变量)'
})
});
export type UpdateTeamBodyType = z.infer<typeof UpdateTeamBodySchema>;
export const UpdateTeamResponseSchema = z.object({});
export type UpdateTeamResponseType = z.infer<typeof UpdateTeamResponseSchema>;
import type { OpenAPIPath } from '../../../type';
import { TagsMap } from '../../../tag';
import { UpdateTeamBodySchema } from './api';
export const TeamPath: OpenAPIPath = {
'/api/support/user/team/update': {
post: {
summary: '更新团队信息',
description: '更新团队名称、头像、域名、第三方账号(Laf/OpenAI)及外部工作流变量',
tags: [TagsMap.teamManage],
requestBody: {
content: {
'application/json': {
schema: UpdateTeamBodySchema
}
}
},
responses: {
200: {
description: '更新成功'
}
}
}
}
};
...@@ -31,9 +31,11 @@ export const TagsMap = { ...@@ -31,9 +31,11 @@ export const TagsMap = {
// Dataset // Dataset
datasetCommon: '知识库管理', datasetCommon: '知识库管理',
datasetCollection: '集合管理', datasetCollection: '集合管理',
datasetCollectionController: '集合操作', datasetCollectionCrteate: '知识库集合创建',
datasetData: '数据管理', datasetData: '数据管理',
datasetTraining: '训练管理', datasetTraining: '训练管理',
datasetApiDataset: 'API 数据集管理',
datasetFile: '文件管理',
// Plugin // Plugin
pluginToolTag: '工具标签', pluginToolTag: '工具标签',
...@@ -43,6 +45,8 @@ export const TagsMap = { ...@@ -43,6 +45,8 @@ export const TagsMap = {
publishChannel: '发布渠道', publishChannel: '发布渠道',
/* Support */ /* Support */
// Team
teamManage: '团队管理',
// Wallet // Wallet
walletBill: '订单', walletBill: '订单',
walletDiscountCoupon: '优惠券', walletDiscountCoupon: '优惠券',
......
...@@ -64,7 +64,15 @@ export const LogCategories = { ...@@ -64,7 +64,15 @@ export const LogCategories = {
FILE: ['dataset', 'file'], FILE: ['dataset', 'file'],
FOLDER: ['dataset', 'folder'], FOLDER: ['dataset', 'folder'],
QUEUES: ['dataset', 'queues'], QUEUES: ['dataset', 'queues'],
TRAINING: ['dataset', 'training'] TRAINING: ['dataset', 'training'],
FILE_PARSE: ['dataset', 'training', 'file-parse'],
EMBEDDING: ['dataset', 'training', 'embedding'],
QA: ['dataset', 'training', 'qa'],
IMAGE_PARSE: ['dataset', 'training', 'image-parse'],
IMAGE_INDEX: ['dataset', 'training', 'image-index'],
INDEX_EXTEND: ['dataset', 'training', 'index-extend'],
LLM_PARGRAPH: ['dataset', 'training', 'llm-pargraph']
}), }),
AI: Object.assign(['ai'], { AI: Object.assign(['ai'], {
AGENT: ['ai', 'agent'], AGENT: ['ai', 'agent'],
......
...@@ -9,10 +9,10 @@ import { ...@@ -9,10 +9,10 @@ import {
import { import {
type CreatePostPresignedUrlOptions, type CreatePostPresignedUrlOptions,
type CreatePostPresignedUrlParams, type CreatePostPresignedUrlParams,
type CreatePostPresignedUrlResult,
type createPreviewUrlParams, type createPreviewUrlParams,
CreateGetPresignedUrlParamsSchema CreateGetPresignedUrlParamsSchema
} from '../type'; } from '../type';
import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type';
import { getSystemMaxFileSize, Mimes } from '../constants'; import { getSystemMaxFileSize, Mimes } from '../constants';
import path from 'node:path'; import path from 'node:path';
import { MongoS3TTL } from '../schema'; import { MongoS3TTL } from '../schema';
...@@ -131,7 +131,7 @@ export class S3BaseBucket { ...@@ -131,7 +131,7 @@ export class S3BaseBucket {
async createPresignedPutUrl( async createPresignedPutUrl(
params: CreatePostPresignedUrlParams, params: CreatePostPresignedUrlParams,
options: CreatePostPresignedUrlOptions = {} options: CreatePostPresignedUrlOptions = {}
): Promise<CreatePostPresignedUrlResult> { ): Promise<CreatePostPresignedUrlResponseType> {
try { try {
const { expiredHours, maxFileSize = getSystemMaxFileSize() } = options; const { expiredHours, maxFileSize = getSystemMaxFileSize() } = options;
const formatMaxFileSize = maxFileSize * 1024 * 1024; const formatMaxFileSize = maxFileSize * 1024 * 1024;
......
...@@ -40,14 +40,6 @@ export const CreatePostPresignedUrlOptionsSchema = z.object({ ...@@ -40,14 +40,6 @@ export const CreatePostPresignedUrlOptionsSchema = z.object({
}); });
export type CreatePostPresignedUrlOptions = z.infer<typeof CreatePostPresignedUrlOptionsSchema>; export type CreatePostPresignedUrlOptions = z.infer<typeof CreatePostPresignedUrlOptionsSchema>;
export const CreatePostPresignedUrlResultSchema = z.object({
url: z.string().nonempty(),
key: z.string().nonempty(),
headers: z.record(z.string(), z.string()),
maxSize: z.number().positive().optional() // bytes
});
export type CreatePostPresignedUrlResult = z.infer<typeof CreatePostPresignedUrlResultSchema>;
export const CreateGetPresignedUrlParamsSchema = z.object({ export const CreateGetPresignedUrlParamsSchema = z.object({
key: z.string().nonempty(), key: z.string().nonempty(),
expiredHours: z.number().positive().optional() expiredHours: z.number().positive().optional()
......
...@@ -3,7 +3,7 @@ import { ...@@ -3,7 +3,7 @@ import {
type ChatCompletionCreateParams, type ChatCompletionCreateParams,
type ChatCompletionMessageParam, type ChatCompletionMessageParam,
type ChatCompletionTool type ChatCompletionTool
} from '@fastgpt/global/core/ai/type'; } from '@fastgpt/global/core/ai/llm/type';
import { chats2GPTMessages } from '@fastgpt/global/core/chat/adapt'; import { chats2GPTMessages } from '@fastgpt/global/core/chat/adapt';
import { type ChatItemMiniType } from '@fastgpt/global/core/chat/type'; import { type ChatItemMiniType } from '@fastgpt/global/core/chat/type';
import { WorkerNameEnum, getWorkerController } from '../../../worker/utils'; import { WorkerNameEnum, getWorkerController } from '../../../worker/utils';
......
/* pg vector crud */ /* pg vector crud */
import { DatasetVectorTableName, VectorVQ } from '../constants'; import { DatasetVectorTableName, VectorVQ } from '../constants';
import { PgClient, connectPg } from './controller'; import { PgClient, connectPg } from './controller';
import { type PgSearchRawType } from '@fastgpt/global/core/dataset/api';
import type { VectorControllerType } from '../type'; import type { VectorControllerType } from '../type';
import dayjs from 'dayjs'; import dayjs from 'dayjs';
import { getLogger, LogCategories } from '../../logger'; import { getLogger, LogCategories } from '../../logger';
...@@ -168,7 +167,11 @@ export class PgVectorCtrl implements VectorControllerType { ...@@ -168,7 +167,11 @@ export class PgVectorCtrl implements VectorControllerType {
) SELECT id, collection_id, score FROM relaxed_results ORDER BY score; ) SELECT id, collection_id, score FROM relaxed_results ORDER BY score;
COMMIT;` COMMIT;`
); );
const rows = results?.[results.length - 2]?.rows as PgSearchRawType[]; const rows = results?.[results.length - 2]?.rows as {
id: string;
collection_id: string;
score: number;
}[];
if (!Array.isArray(rows)) { if (!Array.isArray(rows)) {
return { return {
......
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
*/ */
import { createLLMResponse } from '../ai/llm/request'; import { createLLMResponse } from '../ai/llm/request';
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type';
import { sliceJsonStr } from '@fastgpt/global/common/string/tools'; import { sliceJsonStr } from '@fastgpt/global/common/string/tools';
import json5 from 'json5'; import json5 from 'json5';
......
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type';
import { import {
QuestionGuidePrompt, QuestionGuidePrompt,
QuestionGuideFooterPrompt QuestionGuideFooterPrompt
......
...@@ -3,7 +3,7 @@ import type { ...@@ -3,7 +3,7 @@ import type {
ChatCompletionTool, ChatCompletionTool,
ChatCompletionMessageToolCall, ChatCompletionMessageToolCall,
CompletionFinishReason CompletionFinishReason
} from '@fastgpt/global/core/ai/type'; } from '@fastgpt/global/core/ai/llm/type';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
import type { import type {
ToolCallChildrenInteractive, ToolCallChildrenInteractive,
......
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type';
export const filterEmptyAssistantMessages = (messages: ChatCompletionMessageParam[]) => { export const filterEmptyAssistantMessages = (messages: ChatCompletionMessageParam[]) => {
return messages.filter((item) => { return messages.filter((item) => {
......
...@@ -4,7 +4,7 @@ import { calculateCompressionThresholds } from './constants'; ...@@ -4,7 +4,7 @@ import { calculateCompressionThresholds } from './constants';
import type { CreateLLMResponseProps } from '../request'; import type { CreateLLMResponseProps } from '../request';
import { createLLMResponse } from '../request'; import { createLLMResponse } from '../request';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type';
import { getCompressRequestMessagesPrompt } from './prompt'; import { getCompressRequestMessagesPrompt } from './prompt';
import type { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type'; import type { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
import { formatModelChars2Points } from '../../../../support/wallet/usage/utils'; import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';
......
import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.schema'; import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.schema';
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type';
import { calculateCompressionThresholds } from './constants'; import { calculateCompressionThresholds } from './constants';
export const getCompressRequestMessagesPrompt = async ({ export const getCompressRequestMessagesPrompt = async ({
......
...@@ -5,7 +5,7 @@ import type { ...@@ -5,7 +5,7 @@ import type {
ChatCompletionMessageToolCall, ChatCompletionMessageToolCall,
ChatCompletionSystemMessageParam, ChatCompletionSystemMessageParam,
ChatCompletionTool ChatCompletionTool
} from '@fastgpt/global/core/ai/type'; } from '@fastgpt/global/core/ai/llm/type';
import { getPromptToolCallPrompt } from './prompt'; import { getPromptToolCallPrompt } from './prompt';
import { cloneDeep } from 'lodash'; import { cloneDeep } from 'lodash';
......
import { replaceVariable } from '@fastgpt/global/common/string/tools'; import { replaceVariable } from '@fastgpt/global/common/string/tools';
import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type';
export const getPromptToolCallPrompt = (tools: ChatCompletionTool['function'][]) => { export const getPromptToolCallPrompt = (tools: ChatCompletionTool['function'][]) => {
const prompt = `<ToolSkill> const prompt = `<ToolSkill>
......
import type { import type {
ChatCompletion, ChatCompletionCreateParams,
ChatCompletionCreateParamsNonStreaming, ChatCompletionCreateParamsNonStreaming,
ChatCompletionCreateParamsStreaming, ChatCompletionCreateParamsStreaming,
ChatCompletionMessageParam, ChatCompletionMessageParam,
...@@ -7,9 +7,9 @@ import type { ...@@ -7,9 +7,9 @@ import type {
CompletionFinishReason, CompletionFinishReason,
CompletionUsage, CompletionUsage,
OpenAI, OpenAI,
StreamChatType, StreamResponseType,
UnStreamChatType UnStreamResponseType
} from '@fastgpt/global/core/ai/type'; } from '@fastgpt/global/core/ai/llm/type';
import { import {
computedMaxToken, computedMaxToken,
computedTemperature, computedTemperature,
...@@ -45,7 +45,9 @@ export type ResponseEvents = { ...@@ -45,7 +45,9 @@ export type ResponseEvents = {
onToolParam?: (e: { tool: ChatCompletionMessageToolCall; params: string }) => void; onToolParam?: (e: { tool: ChatCompletionMessageToolCall; params: string }) => void;
}; };
export type CreateLLMResponseProps<T extends CompletionsBodyType = CompletionsBodyType> = { export type CreateLLMResponseProps<
T extends ChatCompletionCreateParams = ChatCompletionCreateParams
> = {
throwError?: boolean; throwError?: boolean;
userKey?: OpenaiAccountType; userKey?: OpenaiAccountType;
body: LLMRequestBodyType<T>; body: LLMRequestBodyType<T>;
...@@ -77,7 +79,7 @@ type LLMResponse = { ...@@ -77,7 +79,7 @@ type LLMResponse = {
底层封装 LLM 调用 帮助上层屏蔽 stream 和非 stream,以及 toolChoice 和 promptTool 模式。 底层封装 LLM 调用 帮助上层屏蔽 stream 和非 stream,以及 toolChoice 和 promptTool 模式。
工具调用无论哪种模式,都存 toolChoice 的格式,promptTool 通过修改 toolChoice 的结构,形成特定的 messages 进行调用。 工具调用无论哪种模式,都存 toolChoice 的格式,promptTool 通过修改 toolChoice 的结构,形成特定的 messages 进行调用。
*/ */
export const createLLMResponse = async <T extends CompletionsBodyType>( export const createLLMResponse = async <T extends ChatCompletionCreateParams>(
args: CreateLLMResponseProps<T> args: CreateLLMResponseProps<T>
): Promise<LLMResponse> => { ): Promise<LLMResponse> => {
// 生成唯一的请求追踪 ID // 生成唯一的请求追踪 ID
...@@ -264,8 +266,8 @@ export const createLLMResponse = async <T extends CompletionsBodyType>( ...@@ -264,8 +266,8 @@ export const createLLMResponse = async <T extends CompletionsBodyType>(
const outputTokens = const outputTokens =
usage?.completion_tokens || (await countGptMessagesTokens([assistantMessage])); usage?.completion_tokens || (await countGptMessagesTokens([assistantMessage]));
// 异步保存 LLM 请求追踪记录 // 异步保存 LLM 请求追踪记录(fire-and-forget)
saveLLMRequestRecord({ void saveLLMRequestRecord({
requestId, requestId,
body: requestBody, body: requestBody,
response: { response: {
...@@ -333,8 +335,8 @@ export const createLLMResponse = async <T extends CompletionsBodyType>( ...@@ -333,8 +335,8 @@ export const createLLMResponse = async <T extends CompletionsBodyType>(
completeMessages: [...requestMessages, assistantMessage] completeMessages: [...requestMessages, assistantMessage]
}; };
} catch (error) { } catch (error) {
// 异步保存 LLM 请求追踪记录 // 异步保存 LLM 请求追踪记录(fire-and-forget)
saveLLMRequestRecord({ void saveLLMRequestRecord({
requestId, requestId,
body: requestBody, body: requestBody,
response: { response: {
...@@ -363,7 +365,8 @@ export const createLLMResponse = async <T extends CompletionsBodyType>( ...@@ -363,7 +365,8 @@ export const createLLMResponse = async <T extends CompletionsBodyType>(
} }
}; };
type CompleteParams = Pick<CreateLLMResponseProps<CompletionsBodyType>, 'body'> & ResponseEvents; type CompleteParams = Pick<CreateLLMResponseProps<ChatCompletionCreateParams>, 'body'> &
ResponseEvents;
type CompleteResponse = Pick< type CompleteResponse = Pick<
LLMResponse, LLMResponse,
...@@ -382,7 +385,7 @@ export const createStreamResponse = async ({ ...@@ -382,7 +385,7 @@ export const createStreamResponse = async ({
onToolCall, onToolCall,
onToolParam onToolParam
}: CompleteParams & { }: CompleteParams & {
response: StreamChatType; response: StreamResponseType;
isAborted?: CreateLLMResponseProps['isAborted']; isAborted?: CreateLLMResponseProps['isAborted'];
}): Promise<CompleteResponse> => { }): Promise<CompleteResponse> => {
const { retainDatasetCite = true, tools, toolCallMode = 'toolChoice', model } = body; const { retainDatasetCite = true, tools, toolCallMode = 'toolChoice', model } = body;
...@@ -598,7 +601,7 @@ export const createCompleteResponse = async ({ ...@@ -598,7 +601,7 @@ export const createCompleteResponse = async ({
onStreaming, onStreaming,
onReasoning, onReasoning,
onToolCall onToolCall
}: CompleteParams & { response: ChatCompletion }): Promise<CompleteResponse> => { }: CompleteParams & { response: UnStreamResponseType }): Promise<CompleteResponse> => {
const { tools, toolCallMode = 'toolChoice', retainDatasetCite = true } = body; const { tools, toolCallMode = 'toolChoice', retainDatasetCite = true } = body;
const modelData = getLLMModel(body.model); const modelData = getLLMModel(body.model);
...@@ -674,14 +677,11 @@ export const createCompleteResponse = async ({ ...@@ -674,14 +677,11 @@ export const createCompleteResponse = async ({
}; };
}; };
type CompletionsBodyType =
| ChatCompletionCreateParamsNonStreaming
| ChatCompletionCreateParamsStreaming;
type InferCompletionsBody<T> = T extends { stream: true } type InferCompletionsBody<T> = T extends { stream: true }
? ChatCompletionCreateParamsStreaming ? ChatCompletionCreateParamsStreaming
: T extends { stream: false } : T extends { stream: false }
? ChatCompletionCreateParamsNonStreaming ? ChatCompletionCreateParamsNonStreaming
: ChatCompletionCreateParamsNonStreaming | ChatCompletionCreateParamsStreaming; : ChatCompletionCreateParams;
type LLMRequestBodyType<T> = Omit<T, 'model' | 'stop' | 'response_format' | 'messages'> & { type LLMRequestBodyType<T> = Omit<T, 'model' | 'stop' | 'response_format' | 'messages'> & {
model: string | LLMModelItemType; model: string | LLMModelItemType;
...@@ -698,7 +698,7 @@ type LLMRequestBodyType<T> = Omit<T, 'model' | 'stop' | 'response_format' | 'mes ...@@ -698,7 +698,7 @@ type LLMRequestBodyType<T> = Omit<T, 'model' | 'stop' | 'response_format' | 'mes
useVision?: boolean; useVision?: boolean;
requestOrigin?: string; requestOrigin?: string;
}; };
const llmCompletionsBodyFormat = async <T extends CompletionsBodyType>({ const llmCompletionsBodyFormat = async <T extends ChatCompletionCreateParams>({
retainDatasetCite, retainDatasetCite,
useVision, useVision,
requestOrigin, requestOrigin,
...@@ -822,11 +822,11 @@ const createChatCompletion = async ({ ...@@ -822,11 +822,11 @@ const createChatCompletion = async ({
options?: OpenAI.RequestOptions; options?: OpenAI.RequestOptions;
}): Promise< }): Promise<
| { | {
response: StreamChatType; response: StreamResponseType;
isStreamResponse: true; isStreamResponse: true;
} }
| { | {
response: UnStreamChatType; response: UnStreamResponseType;
isStreamResponse: false; isStreamResponse: false;
} }
> => { > => {
......
...@@ -4,9 +4,8 @@ import type { ...@@ -4,9 +4,8 @@ import type {
ChatCompletionContentPart, ChatCompletionContentPart,
ChatCompletionContentPartRefusal, ChatCompletionContentPartRefusal,
ChatCompletionContentPartText, ChatCompletionContentPartText,
ChatCompletionMessageParam, ChatCompletionMessageParam
SdkChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type';
} from '@fastgpt/global/core/ai/type';
import { axios } from '../../../common/api/axios'; import { axios } from '../../../common/api/axios';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
...@@ -376,9 +375,9 @@ export const loadRequestMessages = async ({ ...@@ -376,9 +375,9 @@ export const loadRequestMessages = async ({
const loadMessages = ( const loadMessages = (
await Promise.all( await Promise.all(
mergeMessages.map(async (item, i) => { mergeMessages.map(async (raw, i) => {
delete item.dataId; // 解构剥离系统内部字段,避免 mutate 调用方传入的 messages
delete item.hideInUI; const { dataId: _dataId, hideInUI: _hideInUI, ...item } = raw;
if (item.role === ChatCompletionRequestMessageRoleEnum.System) { if (item.role === ChatCompletionRequestMessageRoleEnum.System) {
const content = parseSystemMessage(item.content); const content = parseSystemMessage(item.content);
...@@ -443,5 +442,5 @@ export const loadRequestMessages = async ({ ...@@ -443,5 +442,5 @@ export const loadRequestMessages = async ({
) )
).filter(Boolean) as ChatCompletionMessageParam[]; ).filter(Boolean) as ChatCompletionMessageParam[];
return loadMessages as SdkChatCompletionMessageParam[]; return loadMessages;
}; };
import { type LLMModelItemType } from '@fastgpt/global/core/ai/model.schema'; import { type LLMModelItemType } from '@fastgpt/global/core/ai/model.schema';
import type { CompletionFinishReason, CompletionUsage } from '@fastgpt/global/core/ai/type'; import type { CompletionFinishReason, CompletionUsage } from '@fastgpt/global/core/ai/llm/type';
import { getLLMDefaultUsage } from '@fastgpt/global/core/ai/constants'; import { getLLMDefaultUsage } from '@fastgpt/global/core/ai/constants';
import { removeDatasetCiteText } from '@fastgpt/global/core/ai/llm/utils'; import { removeDatasetCiteText } from '@fastgpt/global/core/ai/llm/utils';
import json5 from 'json5'; import json5 from 'json5';
......
...@@ -2,7 +2,6 @@ import { ...@@ -2,7 +2,6 @@ import {
DatasetCollectionDataProcessModeEnum, DatasetCollectionDataProcessModeEnum,
DatasetCollectionTypeEnum DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants'; } from '@fastgpt/global/core/dataset/constants';
import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api';
import { MongoDatasetCollection } from './schema'; import { MongoDatasetCollection } from './schema';
import type { import type {
DatasetCollectionSchemaType, DatasetCollectionSchemaType,
...@@ -33,6 +32,10 @@ import { ...@@ -33,6 +32,10 @@ import {
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants'; import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
import { getS3DatasetSource } from '../../../common/s3/sources/dataset'; import { getS3DatasetSource } from '../../../common/s3/sources/dataset';
import { removeS3TTL, isS3ObjectKey } from '../../../common/s3/utils'; import { removeS3TTL, isS3ObjectKey } from '../../../common/s3/utils';
import type {
CreateCollectionWithResultResponseType,
ApiCreateDatasetCollectionParams
} from '@fastgpt/global/openapi/core/dataset/collection/createApi';
export const createCollectionAndInsertData = async ({ export const createCollectionAndInsertData = async ({
dataset, dataset,
...@@ -52,7 +55,7 @@ export const createCollectionAndInsertData = async ({ ...@@ -52,7 +55,7 @@ export const createCollectionAndInsertData = async ({
billId?: string; billId?: string;
session?: ClientSession; session?: ClientSession;
}) => { }): Promise<CreateCollectionWithResultResponseType> => {
// Adapter 4.9.0 // Adapter 4.9.0
if (createCollectionParams.trainingType === DatasetCollectionDataProcessModeEnum.auto) { if (createCollectionParams.trainingType === DatasetCollectionDataProcessModeEnum.auto) {
createCollectionParams.trainingType = DatasetCollectionDataProcessModeEnum.chunk; createCollectionParams.trainingType = DatasetCollectionDataProcessModeEnum.chunk;
...@@ -166,7 +169,7 @@ export const createCollectionAndInsertData = async ({ ...@@ -166,7 +169,7 @@ export const createCollectionAndInsertData = async ({
insertLen: predictDataLimitLength(trainingMode, chunks) insertLen: predictDataLimitLength(trainingMode, chunks)
}); });
const fn = async (session: ClientSession) => { const fn = async (session: ClientSession): Promise<CreateCollectionWithResultResponseType> => {
// 3. Create collection // 3. Create collection
const { _id: collectionId } = await createOneCollection({ const { _id: collectionId } = await createOneCollection({
...formatCreateCollectionParams, ...formatCreateCollectionParams,
...@@ -236,7 +239,9 @@ export const createCollectionAndInsertData = async ({ ...@@ -236,7 +239,9 @@ export const createCollectionAndInsertData = async ({
return { return {
collectionId: String(collectionId), collectionId: String(collectionId),
insertResults results: {
insertLen: insertResults.insertLen
}
}; };
}; };
...@@ -246,9 +251,21 @@ export const createCollectionAndInsertData = async ({ ...@@ -246,9 +251,21 @@ export const createCollectionAndInsertData = async ({
return mongoSessionRun(fn); return mongoSessionRun(fn);
}; };
export type CreateOneCollectionParams = CreateDatasetCollectionParams & { export type CreateOneCollectionParams = ApiCreateDatasetCollectionParams & {
teamId: string; teamId: string;
tmbId: string; tmbId: string;
name: string;
type: DatasetCollectionTypeEnum;
fileId?: string;
rawLink?: string;
externalFileId?: string;
externalFileUrl?: string;
apiFileId?: string;
apiFileParentId?: string;
rawTextLength?: number;
hashRawText?: string;
createTime?: Date;
updateTime?: Date;
session?: ClientSession; session?: ClientSession;
}; };
export async function createOneCollection({ session, ...props }: CreateOneCollectionParams) { export async function createOneCollection({ session, ...props }: CreateOneCollectionParams) {
......
...@@ -218,7 +218,7 @@ export const getTrainingModeByCollection = ({ ...@@ -218,7 +218,7 @@ export const getTrainingModeByCollection = ({
autoIndexes, autoIndexes,
imageIndex imageIndex
}: { }: {
trainingType: DatasetCollectionDataProcessModeEnum; trainingType?: DatasetCollectionDataProcessModeEnum;
autoIndexes?: boolean; autoIndexes?: boolean;
imageIndex?: boolean; imageIndex?: boolean;
}) => { }) => {
......
import { MongoDatasetTraining } from './schema'; import { MongoDatasetTraining } from './schema';
import type { import type {
PushDatasetDataChunkProps, PushDataChunkType,
PushDatasetDataResponse PushDataResponseType
} from '@fastgpt/global/core/dataset/api'; } from '@fastgpt/global/openapi/core/dataset/data/api';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { type ClientSession } from '../../../common/mongo'; import { type ClientSession } from '../../../common/mongo';
import { getLLMModel, getEmbeddingModel, getVlmModel } from '../../ai/model'; import { getLLMModel, getEmbeddingModel, getVlmModel } from '../../ai/model';
...@@ -27,7 +27,7 @@ export const lockTrainingDataByTeamId = async (teamId: string): Promise<any> => ...@@ -27,7 +27,7 @@ export const lockTrainingDataByTeamId = async (teamId: string): Promise<any> =>
} catch (error) {} } catch (error) {}
}; };
export async function pushDataListToTrainingQueue({ export const pushDataListToTrainingQueue = async ({
teamId, teamId,
tmbId, tmbId,
datasetId, datasetId,
...@@ -46,7 +46,7 @@ export async function pushDataListToTrainingQueue({ ...@@ -46,7 +46,7 @@ export async function pushDataListToTrainingQueue({
datasetId: string; datasetId: string;
collectionId: string; collectionId: string;
data: PushDatasetDataChunkProps[]; data: PushDataChunkType[];
mode?: TrainingModeEnum; mode?: TrainingModeEnum;
agentModel: string; agentModel: string;
...@@ -55,9 +55,9 @@ export async function pushDataListToTrainingQueue({ ...@@ -55,9 +55,9 @@ export async function pushDataListToTrainingQueue({
indexSize?: number; indexSize?: number;
billId?: string; billId: string;
session?: ClientSession; session?: ClientSession;
}): Promise<PushDatasetDataResponse> { }): Promise<PushDataResponseType> => {
const vectorModelData = getEmbeddingModel(vectorModel); const vectorModelData = getEmbeddingModel(vectorModel);
if (!vectorModelData) { if (!vectorModelData) {
return Promise.reject(i18nT('common:error_embedding_not_config')); return Promise.reject(i18nT('common:error_embedding_not_config'));
...@@ -209,7 +209,7 @@ export async function pushDataListToTrainingQueue({ ...@@ -209,7 +209,7 @@ export async function pushDataListToTrainingQueue({
logger.info('Single transaction completed', { durationMs: Date.now() - start }); logger.info('Single transaction completed', { durationMs: Date.now() - start });
return { insertLen: insertedCount }; return { insertLen: insertedCount };
} }
} };
export const pushDatasetToParseQueue = async ({ export const pushDatasetToParseQueue = async ({
teamId, teamId,
......
...@@ -35,7 +35,10 @@ const TrainingDataSchema = new Schema({ ...@@ -35,7 +35,10 @@ const TrainingDataSchema = new Schema({
ref: DatasetColCollectionName, ref: DatasetColCollectionName,
required: true required: true
}, },
billId: String, billId: {
type: String,
required: true
},
mode: { mode: {
type: String, type: String,
enum: Object.values(TrainingModeEnum), enum: Object.values(TrainingModeEnum),
......
import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type';
import type { AIChatItemValueItemType } from '@fastgpt/global/core/chat/type'; import type { AIChatItemValueItemType } from '@fastgpt/global/core/chat/type';
export type CapabilityToolCallResult = { export type CapabilityToolCallResult = {
......
...@@ -28,7 +28,7 @@ import type { DispatchPlanAgentResponse } from './sub/plan'; ...@@ -28,7 +28,7 @@ import type { DispatchPlanAgentResponse } from './sub/plan';
import { dispatchPlanAgent } from './sub/plan'; import { dispatchPlanAgent } from './sub/plan';
import { formatFileInput } from './sub/file/utils'; import { formatFileInput } from './sub/file/utils';
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type';
import { masterCall } from './master/call'; import { masterCall } from './master/call';
import type { SkillToolType } from '@fastgpt/global/core/ai/skill/type'; import type { SkillToolType } from '@fastgpt/global/core/ai/skill/type';
import { import {
......
import type { ChatCompletionMessageParam, ChatCompletionTool } from '@fastgpt/global/core/ai/type'; import type {
ChatCompletionMessageParam,
ChatCompletionTool
} from '@fastgpt/global/core/ai/llm/type';
import { runAgentCall } from '../../../../../ai/llm/agentCall'; import { runAgentCall } from '../../../../../ai/llm/agentCall';
import { chats2GPTMessages, runtimePrompt2ChatsValue } from '@fastgpt/global/core/chat/adapt'; import { chats2GPTMessages, runtimePrompt2ChatsValue } from '@fastgpt/global/core/chat/adapt';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants'; import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
......
import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type';
import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants'; import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants';
import z from 'zod'; import z from 'zod';
import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/type/io'; import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/type/io';
......
import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type';
import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants'; import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants';
import { parseUrlToFileType } from '../../../../../utils/context'; import { parseUrlToFileType } from '../../../../../utils/context';
import { getLogger, LogCategories } from '../../../../../../../common/logger'; import { getLogger, LogCategories } from '../../../../../../../common/logger';
......
import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type';
import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants'; import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants';
export const ModelAgentTool: ChatCompletionTool = { export const ModelAgentTool: ChatCompletionTool = {
......
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type';
import { createLLMResponse, type ResponseEvents } from '../../../../../../ai/llm/request'; import { createLLMResponse, type ResponseEvents } from '../../../../../../ai/llm/request';
import { getLLMModel } from '../../../../../../ai/model'; import { getLLMModel } from '../../../../../../ai/model';
import { formatModelChars2Points } from '../../../../../../../support/wallet/usage/utils'; import { formatModelChars2Points } from '../../../../../../../support/wallet/usage/utils';
......
import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type';
import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants'; import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants';
import z from 'zod'; import z from 'zod';
import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
......
import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type';
import { SubAppIds, systemSubInfo } from '@fastgpt/global/core/workflow/node/agent/constants'; import { SubAppIds, systemSubInfo } from '@fastgpt/global/core/workflow/node/agent/constants';
import type { InteractiveNodeResponseType } from '@fastgpt/global/core/workflow/template/system/interactive/type'; import type { InteractiveNodeResponseType } from '@fastgpt/global/core/workflow/template/system/interactive/type';
import { getNanoid } from '@fastgpt/global/common/string/tools'; import { getNanoid } from '@fastgpt/global/common/string/tools';
......
...@@ -2,7 +2,7 @@ import type { ...@@ -2,7 +2,7 @@ import type {
ChatCompletionMessageParam, ChatCompletionMessageParam,
ChatCompletionMessageToolCall, ChatCompletionMessageToolCall,
ChatCompletionTool ChatCompletionTool
} from '@fastgpt/global/core/ai/type'; } from '@fastgpt/global/core/ai/llm/type';
import { createLLMResponse } from '../../../../../../ai/llm/request'; import { createLLMResponse } from '../../../../../../ai/llm/request';
import { import {
getInitialPlanPrompt, getInitialPlanPrompt,
......
import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type';
import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants'; import { SubAppIds } from '@fastgpt/global/core/workflow/node/agent/constants';
import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/type/io'; import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/type/io';
import { AIAskTool } from './ask/constants'; import { AIAskTool } from './ask/constants';
......
...@@ -11,7 +11,7 @@ import { MongoApp } from '../../../../../../app/schema'; ...@@ -11,7 +11,7 @@ import { MongoApp } from '../../../../../../app/schema';
import { getMCPChildren } from '../../../../../../app/mcp'; import { getMCPChildren } from '../../../../../../app/mcp';
import { getMCPToolRuntimeNode } from '@fastgpt/global/core/app/tool/mcpTool/utils'; import { getMCPToolRuntimeNode } from '@fastgpt/global/core/app/tool/mcpTool/utils';
import { getHTTPToolRuntimeNode } from '@fastgpt/global/core/app/tool/httpTool/utils'; import { getHTTPToolRuntimeNode } from '@fastgpt/global/core/app/tool/httpTool/utils';
import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type';
import type { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io'; import type { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io';
import type { JSONSchemaInputType } from '@fastgpt/global/core/app/jsonschema'; import type { JSONSchemaInputType } from '@fastgpt/global/core/app/jsonschema';
import { import {
......
import type { StoreSecretValueType } from '@fastgpt/global/common/secret/type'; import type { StoreSecretValueType } from '@fastgpt/global/common/secret/type';
import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type';
import type { SystemToolSecretInputTypeEnum } from '@fastgpt/global/core/app/tool/systemTool/constants'; import type { SystemToolSecretInputTypeEnum } from '@fastgpt/global/core/app/tool/systemTool/constants';
import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type'; import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type';
......
...@@ -2,7 +2,7 @@ import type { localeType } from '@fastgpt/global/common/i18n/type'; ...@@ -2,7 +2,7 @@ import type { localeType } from '@fastgpt/global/common/i18n/type';
import type { SkillToolType } from '@fastgpt/global/core/ai/skill/type'; import type { SkillToolType } from '@fastgpt/global/core/ai/skill/type';
import type { SubAppRuntimeType } from './type'; import type { SubAppRuntimeType } from './type';
import { getAgentRuntimeTools } from './sub/tool/utils'; import { getAgentRuntimeTools } from './sub/tool/utils';
import type { ChatCompletionTool } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionTool } from '@fastgpt/global/core/ai/llm/type';
import { readFileTool } from './sub/file/utils'; import { readFileTool } from './sub/file/utils';
import { PlanAgentTool } from './sub/plan/constants'; import { PlanAgentTool } from './sub/plan/constants';
import { datasetSearchTool } from './sub/dataset/utils'; import { datasetSearchTool } from './sub/dataset/utils';
......
...@@ -23,7 +23,7 @@ const logger = getLogger(LogCategories.MODULE.WORKFLOW.AI); ...@@ -23,7 +23,7 @@ const logger = getLogger(LogCategories.MODULE.WORKFLOW.AI);
import { import {
type ChatCompletionMessageParam, type ChatCompletionMessageParam,
type ChatCompletionTool type ChatCompletionTool
} from '@fastgpt/global/core/ai/type'; } from '@fastgpt/global/core/ai/llm/type';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type'; import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
import { import {
......
...@@ -2,7 +2,7 @@ import type { ...@@ -2,7 +2,7 @@ import type {
ChatCompletionMessageParam, ChatCompletionMessageParam,
ChatCompletionTool, ChatCompletionTool,
CompletionFinishReason CompletionFinishReason
} from '@fastgpt/global/core/ai/type'; } from '@fastgpt/global/core/ai/llm/type';
import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants'; import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/utils'; import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/utils';
import { runWorkflow } from '../../index'; import { runWorkflow } from '../../index';
......
import type { import type {
ChatCompletionMessageParam, ChatCompletionMessageParam,
CompletionFinishReason CompletionFinishReason
} from '@fastgpt/global/core/ai/type'; } from '@fastgpt/global/core/ai/llm/type';
import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type'; import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type';
import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type'; import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type';
......
...@@ -14,7 +14,7 @@ import type { McpToolDataType } from '@fastgpt/global/core/app/tool/mcpTool/type ...@@ -14,7 +14,7 @@ import type { McpToolDataType } from '@fastgpt/global/core/app/tool/mcpTool/type
import type { JSONSchemaInputType } from '@fastgpt/global/core/app/jsonschema'; import type { JSONSchemaInputType } from '@fastgpt/global/core/app/jsonschema';
import type { ToolNodeItemType } from './tool/type'; import type { ToolNodeItemType } from './tool/type';
import json5 from 'json5'; import json5 from 'json5';
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
// Assistant process // Assistant process
......
...@@ -6,7 +6,7 @@ import { ...@@ -6,7 +6,7 @@ import {
type ChatCompletionContentPart, type ChatCompletionContentPart,
type ChatCompletionCreateParams, type ChatCompletionCreateParams,
type ChatCompletionTool type ChatCompletionTool
} from '@fastgpt/global/core/ai/type'; } from '@fastgpt/global/core/ai/llm/type';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
import { parentPort } from 'worker_threads'; import { parentPort } from 'worker_threads';
import { getLogger, LogCategories } from '../../common/logger'; import { getLogger, LogCategories } from '../../common/logger';
......
...@@ -3,11 +3,11 @@ import { compressBase64Img } from '../img'; ...@@ -3,11 +3,11 @@ import { compressBase64Img } from '../img';
import { useToast } from '../../../hooks/useToast'; import { useToast } from '../../../hooks/useToast';
import { useCallback, useRef, useTransition } from 'react'; import { useCallback, useRef, useTransition } from 'react';
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
import { type CreatePostPresignedUrlResult } from '../../../../service/common/s3/type';
import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants'; import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants';
import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type';
export const useUploadAvatar = ( export const useUploadAvatar = (
api: (params: { filename: string }) => Promise<CreatePostPresignedUrlResult>, api: (params: { filename: string }) => Promise<CreatePostPresignedUrlResponseType>,
{ {
onSuccess, onSuccess,
maxW = 300, maxW = 300,
......
...@@ -16,7 +16,7 @@ import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; ...@@ -16,7 +16,7 @@ import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
import React, { useMemo } from 'react'; import React, { useMemo } from 'react';
import { getQuoteData } from '@/web/core/dataset/api'; import { getQuoteData } from '@/web/core/dataset/api/data';
import MyBox from '@fastgpt/web/components/common/MyBox'; import MyBox from '@fastgpt/web/components/common/MyBox';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils'; import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
import Markdown from '.'; import Markdown from '.';
......
import React, { useState } from 'react'; import React, { useState } from 'react';
import MyModal from '@fastgpt/web/components/common/MyModal'; import MyModal from '@fastgpt/web/components/common/MyModal';
import { Box, Button, Flex, Grid, useTheme } from '@chakra-ui/react'; import { Box, Button, Flex, Grid, useTheme } from '@chakra-ui/react';
import { type PromptTemplateItem } from '@fastgpt/global/core/ai/type'; import { type PromptTemplateItem } from '@fastgpt/global/core/ai/llm/type';
import { ModalBody, ModalFooter } from '@chakra-ui/react'; import { ModalBody, ModalFooter } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
const PromptTemplate = ({ const PromptTemplate = ({
......
import type { StreamResponseType } from '@/web/common/api/fetch'; import type { StreamResponseType } from '@/web/common/api/fetch';
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type';
import type { import type {
ChatHistoryItemResType, ChatHistoryItemResType,
StepTitleItemType, StepTitleItemType,
......
...@@ -6,7 +6,7 @@ import type { ...@@ -6,7 +6,7 @@ import type {
GetHelperBotFilePresignParamsType, GetHelperBotFilePresignParamsType,
GetHelperBotFilePreviewParamsType GetHelperBotFilePreviewParamsType
} from '@fastgpt/global/openapi/core/chat/helperBot/api'; } from '@fastgpt/global/openapi/core/chat/helperBot/api';
import type { CreatePostPresignedUrlResult } from '@fastgpt/service/common/s3/type'; import type { CreatePostPresignedUrlResponseType } from '@fastgpt/global/common/file/s3/type';
export const getHelperBotChatRecords = (data: GetHelperBotChatRecordsParamsType) => export const getHelperBotChatRecords = (data: GetHelperBotChatRecordsParamsType) =>
GET<GetHelperBotChatRecordsResponseType>('/core/chat/helperBot/getRecords', data); GET<GetHelperBotChatRecordsResponseType>('/core/chat/helperBot/getRecords', data);
...@@ -15,7 +15,7 @@ export const deleteHelperBotChatRecord = (data: DeleteHelperBotChatParamsType) = ...@@ -15,7 +15,7 @@ export const deleteHelperBotChatRecord = (data: DeleteHelperBotChatParamsType) =
DELETE('/core/chat/helperBot/deleteRecord', data); DELETE('/core/chat/helperBot/deleteRecord', data);
export const getHelperBotFilePresign = (data: GetHelperBotFilePresignParamsType) => export const getHelperBotFilePresign = (data: GetHelperBotFilePresignParamsType) =>
POST<CreatePostPresignedUrlResult>('/core/chat/helperBot/getFilePresign', data); POST<CreatePostPresignedUrlResponseType>('/core/chat/helperBot/getFilePresign', data);
export const getHelperBotFilePreview = (data: GetHelperBotFilePreviewParamsType) => export const getHelperBotFilePreview = (data: GetHelperBotFilePreviewParamsType) =>
POST<string>('/core/chat/helperBot/getFilePreview', data); POST<string>('/core/chat/helperBot/getFilePreview', data);
...@@ -8,12 +8,8 @@ import { useTranslation } from 'next-i18next'; ...@@ -8,12 +8,8 @@ import { useTranslation } from 'next-i18next';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import dynamic from 'next/dynamic'; import dynamic from 'next/dynamic';
import MyBox from '@fastgpt/web/components/common/MyBox'; import MyBox from '@fastgpt/web/components/common/MyBox';
import { import { SearchScoreTypeEnum, SearchScoreTypeMap } from '@fastgpt/global/core/dataset/constants';
DatasetCollectionTypeEnum, import type { ReadCollectionSourceBodyType } from '@fastgpt/global/openapi/core/dataset/collection/api';
SearchScoreTypeEnum,
SearchScoreTypeMap
} from '@fastgpt/global/core/dataset/constants';
import type { readCollectionSourceBody } from '@/pages/api/core/dataset/collection/read';
import Markdown from '@/components/Markdown'; import Markdown from '@/components/Markdown';
const InputDataModal = dynamic(() => import('@/pageComponents/dataset/detail/InputDataModal')); const InputDataModal = dynamic(() => import('@/pageComponents/dataset/detail/InputDataModal'));
...@@ -100,7 +96,7 @@ const QuoteItem = ({ ...@@ -100,7 +96,7 @@ const QuoteItem = ({
canDownloadSource?: boolean; canDownloadSource?: boolean;
canEditData?: boolean; canEditData?: boolean;
canEditDataset?: boolean; canEditDataset?: boolean;
} & Omit<readCollectionSourceBody, 'collectionId'>) => { } & Omit<ReadCollectionSourceBodyType, 'collectionId'>) => {
const { t } = useTranslation(); const { t } = useTranslation();
const [editInputData, setEditInputData] = useState<{ dataId: string; collectionId: string }>(); const [editInputData, setEditInputData] = useState<{ dataId: string; collectionId: string }>();
......
...@@ -5,11 +5,11 @@ import { useTranslation } from 'next-i18next'; ...@@ -5,11 +5,11 @@ import { useTranslation } from 'next-i18next';
import { getCollectionSourceAndOpen } from '@/web/core/dataset/hooks/readCollectionSource'; import { getCollectionSourceAndOpen } from '@/web/core/dataset/hooks/readCollectionSource';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils'; import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import MyIcon from '@fastgpt/web/components/common/Icon'; import MyIcon from '@fastgpt/web/components/common/Icon';
import type { readCollectionSourceBody } from '@/pages/api/core/dataset/collection/read'; import type { ReadCollectionSourceBodyType } from '@fastgpt/global/openapi/core/dataset/collection/api';
import type { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import type { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
type Props = BoxProps & type Props = BoxProps &
readCollectionSourceBody & { ReadCollectionSourceBodyType & {
collectionType?: DatasetCollectionTypeEnum; collectionType?: DatasetCollectionTypeEnum;
sourceName?: string; sourceName?: string;
sourceId?: string; sourceId?: string;
......
...@@ -6,12 +6,10 @@ import { Box } from '@chakra-ui/react'; ...@@ -6,12 +6,10 @@ import { Box } from '@chakra-ui/react';
import FolderPath from '@/components/common/folder/Path'; import FolderPath from '@/components/common/folder/Path';
import MyBox from '@fastgpt/web/components/common/MyBox'; import MyBox from '@fastgpt/web/components/common/MyBox';
import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { useRequest } from '@fastgpt/web/hooks/useRequest';
import type { ParentIdType } from '@fastgpt/global/common/parentFolder/type'; import type {
ParentIdType,
type PathItemType = { ParentTreePathItemType
parentId: ParentIdType; } from '@fastgpt/global/common/parentFolder/type';
parentName: string;
};
const DatasetSelectContainer = ({ const DatasetSelectContainer = ({
isOpen, isOpen,
...@@ -24,7 +22,7 @@ const DatasetSelectContainer = ({ ...@@ -24,7 +22,7 @@ const DatasetSelectContainer = ({
}: { }: {
isOpen: boolean; isOpen: boolean;
setParentId: Dispatch<ParentIdType>; setParentId: Dispatch<ParentIdType>;
paths: PathItemType[]; paths: ParentTreePathItemType[];
onClose: () => void; onClose: () => void;
tips?: string | null; tips?: string | null;
isLoading?: boolean; isLoading?: boolean;
......
import type { import type { PushDataResponseType } from '@fastgpt/global/openapi/core/dataset/data/api';
PushDatasetDataChunkProps,
PushDatasetDataResponse
} from '@fastgpt/global/core/dataset/api';
import type { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import type { ApiDatasetServerType } from '@fastgpt/global/core/dataset/apiDataset/type';
/* ================= dataset ===================== */
export type RebuildEmbeddingProps = {
datasetId: string;
vectorModel: string;
};
/* ================= collection ===================== */ /* ================= collection ===================== */
export type CreateCollectionResponse = Promise<{ export type CreateCollectionResponse = Promise<{
collectionId: string; collectionId: string;
results: PushDatasetDataResponse; results: PushDataResponseType;
}>; }>;
/* ================= data ===================== */
export type InsertOneDatasetDataProps = PushDatasetDataChunkProps & {
collectionId: string;
};
import { ObjectIdSchema } from '@fastgpt/global/common/type/mongo';
import { DatasetCollectionSchema } from '@fastgpt/global/core/dataset/type';
import { PermissionSchema } from '@fastgpt/global/support/permission/controller';
import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
import z from 'zod';
/* ================= collection ===================== */
export const DatasetCollectionsListItemSchema = z.object({
_id: ObjectIdSchema.meta({ description: '集合 ID' }),
parentId: DatasetCollectionSchema.shape.parentId,
tmbId: DatasetCollectionSchema.shape.tmbId,
name: DatasetCollectionSchema.shape.name,
type: DatasetCollectionSchema.shape.type,
createTime: DatasetCollectionSchema.shape.createTime,
updateTime: DatasetCollectionSchema.shape.updateTime,
forbid: DatasetCollectionSchema.shape.forbid,
trainingType: DatasetCollectionSchema.shape.trainingType,
tags: z.array(z.string()).optional().meta({ description: '标签' }),
externalFileId: z.string().optional().meta({ description: '外部文件 ID' }),
fileId: z.string().optional().meta({ description: '文件 ID' }),
rawLink: z.string().optional().meta({ description: '原始链接' }),
permission: PermissionSchema,
dataAmount: z.number().meta({ description: '数据数量' }),
trainingAmount: z.number().meta({ description: '训练数量' }),
hasError: z.boolean().optional().meta({ description: '是否错误' })
});
export type DatasetCollectionsListItemType = z.infer<typeof DatasetCollectionsListItemSchema>;
/* ================= data ===================== */
export const DatasetDataListItemSchema = z.object({
_id: ObjectIdSchema.meta({ description: '数据 ID' }),
datasetId: ObjectIdSchema.meta({ description: '数据集 ID' }),
collectionId: ObjectIdSchema.meta({ description: '集合 ID' }),
q: z.string().optional().meta({ description: '问题' }),
a: z.string().optional().meta({ description: '答案' }),
imageId: z.string().optional().meta({ description: '图片 ID' }),
imageSize: z.number().optional().meta({ description: '图片大小' }),
imagePreviewUrl: z.string().optional().meta({ description: '图片预览 URL' }),
chunkIndex: z.number().optional().meta({ description: '块索引' }),
updated: z.boolean().optional().meta({ description: '是否更新' })
});
export type DatasetDataListItemType = z.infer<typeof DatasetDataListItemSchema>;
...@@ -21,7 +21,7 @@ import { ...@@ -21,7 +21,7 @@ import {
FlowNodeOutputTypeEnum FlowNodeOutputTypeEnum
} from '@fastgpt/global/core/workflow/node/constant'; } from '@fastgpt/global/core/workflow/node/constant';
import { nanoid } from 'nanoid'; import { nanoid } from 'nanoid';
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type'; import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type';
import { import {
JS_TEMPLATE, JS_TEMPLATE,
SandboxCodeTypeEnum SandboxCodeTypeEnum
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment