Skip to content
Toggle navigation
P
Projects
G
Groups
S
Snippets
Help
赵月辉
/
fastgpt-migrated
This project
Loading...
Sign in
Toggle navigation
Go to a project
Project
Repository
Issues
0
Merge Requests
0
Pipelines
Wiki
Snippets
Members
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Unverified
Commit
36fafd21
authored
Jun 18, 2025
by
Archer
Committed by
GitHub
Jun 18, 2025
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
feat: dataset index prefix (#5061)
parent
6b2ea696
Hide whitespace changes
Inline
Side-by-side
Showing
34 changed files
with
371 additions
and
259 deletions
+371
-259
deploy/docker/docker-compose-milvus.yml
+6
-6
deploy/docker/docker-compose-oceanbase/docker-compose.yml
+6
-6
deploy/docker/docker-compose-pgvector.yml
+6
-6
deploy/docker/docker-compose-zilliz.yml
+6
-6
docSite/content/zh-cn/docs/development/upgrading/4912.md
+5
-1
docSite/content/zh-cn/docs/development/upgrading/4913.md
+1
-1
docSite/content/zh-cn/docs/development/upgrading/4914.md
+20
-0
packages/global/core/dataset/controller.d.ts
+2
-0
packages/global/core/dataset/training/type.d.ts
+1
-1
packages/global/core/dataset/type.d.ts
+1
-2
packages/service/core/dataset/collection/controller.ts
+1
-1
packages/service/core/dataset/schema.ts
+1
-0
packages/service/core/dataset/training/controller.ts
+0
-20
packages/service/core/dataset/training/schema.ts
+11
-3
packages/web/i18n/en/dataset.json
+2
-0
packages/web/i18n/zh-CN/dataset.json
+2
-0
packages/web/i18n/zh-Hant/dataset.json
+2
-0
projects/app/src/components/core/chat/ChatContainer/ChatBox/Input/ChatInput.tsx
+25
-7
projects/app/src/pageComponents/dataset/detail/Form/CollectionChunkForm.tsx
+53
-44
projects/app/src/pageComponents/dataset/detail/Import/Context.tsx
+1
-0
projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx
+3
-2
projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx
+6
-3
projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx
+20
-12
projects/app/src/pages/api/core/dataset/data/insertData.ts
+4
-1
projects/app/src/pages/api/core/dataset/data/update.ts
+6
-2
projects/app/src/pages/dataset/detail/index.tsx
+1
-1
projects/app/src/service/core/dataset/data/controller.ts
+33
-8
projects/app/src/service/core/dataset/queues/datasetParse.ts
+2
-11
projects/app/src/service/core/dataset/queues/generateQA.ts
+32
-18
projects/app/src/service/core/dataset/queues/generateVector.ts
+110
-91
projects/app/test/api/core/dataset/training/deleteTrainingData.test.ts
+1
-2
projects/app/test/api/core/dataset/training/getTrainingDataDetail.test.ts
+0
-1
projects/app/test/api/core/dataset/training/getTrainingError.test.ts
+0
-1
projects/app/test/api/core/dataset/training/updateTrainingData.test.ts
+1
-2
No files found.
deploy/docker/docker-compose-milvus.yml
View file @
36fafd21
...
@@ -133,15 +133,15 @@ services:
...
@@ -133,15 +133,15 @@ services:
# fastgpt
# fastgpt
sandbox
:
sandbox
:
container_name
:
sandbox
container_name
:
sandbox
image
:
ghcr.io/labring/fastgpt-sandbox:v4.9.1
1
# git
image
:
ghcr.io/labring/fastgpt-sandbox:v4.9.1
3
# git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1
1
# 阿里云
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1
3
# 阿里云
networks
:
networks
:
-
fastgpt
-
fastgpt
restart
:
always
restart
:
always
fastgpt-mcp-server
:
fastgpt-mcp-server
:
container_name
:
fastgpt-mcp-server
container_name
:
fastgpt-mcp-server
image
:
ghcr.io/labring/fastgpt-mcp_server:v4.9.1
1
# git
image
:
ghcr.io/labring/fastgpt-mcp_server:v4.9.1
3
# git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.1
1
# 阿里云
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.1
3
# 阿里云
ports
:
ports
:
-
3005:3000
-
3005:3000
networks
:
networks
:
...
@@ -151,8 +151,8 @@ services:
...
@@ -151,8 +151,8 @@ services:
-
FASTGPT_ENDPOINT=http://fastgpt:3000
-
FASTGPT_ENDPOINT=http://fastgpt:3000
fastgpt
:
fastgpt
:
container_name
:
fastgpt
container_name
:
fastgpt
image
:
ghcr.io/labring/fastgpt:v4.9.1
1
# git
image
:
ghcr.io/labring/fastgpt:v4.9.1
3
# git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1
1
# 阿里云
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1
3
# 阿里云
ports
:
ports
:
-
3000:3000
-
3000:3000
networks
:
networks
:
...
...
deploy/docker/docker-compose-oceanbase/docker-compose.yml
View file @
36fafd21
...
@@ -109,15 +109,15 @@ services:
...
@@ -109,15 +109,15 @@ services:
# fastgpt
# fastgpt
sandbox
:
sandbox
:
container_name
:
sandbox
container_name
:
sandbox
image
:
ghcr.io/labring/fastgpt-sandbox:v4.9.1
1
# git
image
:
ghcr.io/labring/fastgpt-sandbox:v4.9.1
3
# git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1
1
# 阿里云
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1
3
# 阿里云
networks
:
networks
:
-
fastgpt
-
fastgpt
restart
:
always
restart
:
always
fastgpt-mcp-server
:
fastgpt-mcp-server
:
container_name
:
fastgpt-mcp-server
container_name
:
fastgpt-mcp-server
image
:
ghcr.io/labring/fastgpt-mcp_server:v4.9.1
1
# git
image
:
ghcr.io/labring/fastgpt-mcp_server:v4.9.1
3
# git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.1
1
# 阿里云
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.1
3
# 阿里云
ports
:
ports
:
-
3005:3000
-
3005:3000
networks
:
networks
:
...
@@ -127,8 +127,8 @@ services:
...
@@ -127,8 +127,8 @@ services:
-
FASTGPT_ENDPOINT=http://fastgpt:3000
-
FASTGPT_ENDPOINT=http://fastgpt:3000
fastgpt
:
fastgpt
:
container_name
:
fastgpt
container_name
:
fastgpt
image
:
ghcr.io/labring/fastgpt:v4.9.1
1
# git
image
:
ghcr.io/labring/fastgpt:v4.9.1
3
# git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1
1
# 阿里云
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1
3
# 阿里云
ports
:
ports
:
-
3000:3000
-
3000:3000
networks
:
networks
:
...
...
deploy/docker/docker-compose-pgvector.yml
View file @
36fafd21
...
@@ -96,15 +96,15 @@ services:
...
@@ -96,15 +96,15 @@ services:
# fastgpt
# fastgpt
sandbox
:
sandbox
:
container_name
:
sandbox
container_name
:
sandbox
image
:
ghcr.io/labring/fastgpt-sandbox:v4.9.1
1
# git
image
:
ghcr.io/labring/fastgpt-sandbox:v4.9.1
3
# git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1
1
# 阿里云
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1
3
# 阿里云
networks
:
networks
:
-
fastgpt
-
fastgpt
restart
:
always
restart
:
always
fastgpt-mcp-server
:
fastgpt-mcp-server
:
container_name
:
fastgpt-mcp-server
container_name
:
fastgpt-mcp-server
image
:
ghcr.io/labring/fastgpt-mcp_server:v4.9.1
1
# git
image
:
ghcr.io/labring/fastgpt-mcp_server:v4.9.1
3
# git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.1
1
# 阿里云
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.1
3
# 阿里云
ports
:
ports
:
-
3005:3000
-
3005:3000
networks
:
networks
:
...
@@ -114,8 +114,8 @@ services:
...
@@ -114,8 +114,8 @@ services:
-
FASTGPT_ENDPOINT=http://fastgpt:3000
-
FASTGPT_ENDPOINT=http://fastgpt:3000
fastgpt
:
fastgpt
:
container_name
:
fastgpt
container_name
:
fastgpt
image
:
ghcr.io/labring/fastgpt:v4.9.1
1
# git
image
:
ghcr.io/labring/fastgpt:v4.9.1
3
# git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1
1
# 阿里云
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1
3
# 阿里云
ports
:
ports
:
-
3000:3000
-
3000:3000
networks
:
networks
:
...
...
deploy/docker/docker-compose-zilliz.yml
View file @
36fafd21
...
@@ -72,15 +72,15 @@ services:
...
@@ -72,15 +72,15 @@ services:
sandbox
:
sandbox
:
container_name
:
sandbox
container_name
:
sandbox
image
:
ghcr.io/labring/fastgpt-sandbox:v4.9.1
1
# git
image
:
ghcr.io/labring/fastgpt-sandbox:v4.9.1
3
# git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1
1
# 阿里云
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1
3
# 阿里云
networks
:
networks
:
-
fastgpt
-
fastgpt
restart
:
always
restart
:
always
fastgpt-mcp-server
:
fastgpt-mcp-server
:
container_name
:
fastgpt-mcp-server
container_name
:
fastgpt-mcp-server
image
:
ghcr.io/labring/fastgpt-mcp_server:v4.9.1
1
# git
image
:
ghcr.io/labring/fastgpt-mcp_server:v4.9.1
3
# git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.1
1
# 阿里云
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.1
3
# 阿里云
ports
:
ports
:
-
3005:3000
-
3005:3000
networks
:
networks
:
...
@@ -90,8 +90,8 @@ services:
...
@@ -90,8 +90,8 @@ services:
-
FASTGPT_ENDPOINT=http://fastgpt:3000
-
FASTGPT_ENDPOINT=http://fastgpt:3000
fastgpt
:
fastgpt
:
container_name
:
fastgpt
container_name
:
fastgpt
image
:
ghcr.io/labring/fastgpt:v4.9.1
1
# git
image
:
ghcr.io/labring/fastgpt:v4.9.1
3
# git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1
1
# 阿里云
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1
3
# 阿里云
ports
:
ports
:
-
3000:3000
-
3000:3000
networks
:
networks
:
...
...
docSite/content/zh-cn/docs/development/upgrading/4912.md
View file @
36fafd21
...
@@ -9,7 +9,11 @@ weight: 788
...
@@ -9,7 +9,11 @@ weight: 788
## 更新指南
## 更新指南
### 1. 更新镜像:
### 1. 更新环境变量
在
`fastgpt`
和
`fastgpt-pro`
镜像环境变量中加入:
`AES256_SECRET_KEY=`
变量,用于密钥加密。
### 2. 更新镜像:
-
更新 FastGPT 镜像 tag: v4.9.12
-
更新 FastGPT 镜像 tag: v4.9.12
-
更新 FastGPT 商业版镜像 tag: v4.9.12
-
更新 FastGPT 商业版镜像 tag: v4.9.12
...
...
docSite/content/zh-cn/docs/development/upgrading/4913.md
View file @
36fafd21
---
---
title
:
'
V4.9.13
(进行中)
'
title
:
'
V4.9.13'
description
:
'
FastGPT
V4.9.13
更新说明'
description
:
'
FastGPT
V4.9.13
更新说明'
icon
:
'
upgrade'
icon
:
'
upgrade'
draft
:
false
draft
:
false
...
...
docSite/content/zh-cn/docs/development/upgrading/4914.md
0 → 100644
View file @
36fafd21
---
title
:
'
V4.9.14(进行中)'
description
:
'
FastGPT
V4.9.14
更新说明'
icon
:
'
upgrade'
draft
:
false
toc
:
true
weight
:
787
---
## 🚀 新增内容
1.
知识库导入,支持配置:自动将文件名加入系统索引中。
## ⚙️ 优化
1.
统一知识库训练队列代码逻辑。
2.
输入框 UX。
## 🐛 修复
packages/global/core/dataset/controller.d.ts
View file @
36fafd21
...
@@ -10,6 +10,7 @@ export type CreateDatasetDataProps = {
...
@@ -10,6 +10,7 @@ export type CreateDatasetDataProps = {
a
?:
string
;
a
?:
string
;
imageId
?:
string
;
imageId
?:
string
;
indexes
?:
Omit
<
DatasetDataIndexItemType
,
'dataId'
>
[];
indexes
?:
Omit
<
DatasetDataIndexItemType
,
'dataId'
>
[];
indexPrefix
?:
string
;
};
};
export
type
UpdateDatasetDataProps
=
{
export
type
UpdateDatasetDataProps
=
{
...
@@ -21,6 +22,7 @@ export type UpdateDatasetDataProps = {
...
@@ -21,6 +22,7 @@ export type UpdateDatasetDataProps = {
dataId
?:
string
;
// pg data id
dataId
?:
string
;
// pg data id
})[];
})[];
imageId
?:
string
;
imageId
?:
string
;
indexPrefix
?:
string
;
};
};
export
type
PatchIndexesProps
=
export
type
PatchIndexesProps
=
...
...
packages/global/core/dataset/training/type.d.ts
View file @
36fafd21
...
@@ -7,9 +7,9 @@ export type PushDataToTrainingQueueProps = {
...
@@ -7,9 +7,9 @@ export type PushDataToTrainingQueueProps = {
datasetId
:
string
;
datasetId
:
string
;
collectionId
:
string
;
collectionId
:
string
;
data
:
PushDatasetDataChunkProps
[];
mode
?:
TrainingModeEnum
;
mode
?:
TrainingModeEnum
;
data
:
PushDatasetDataChunkProps
[];
data
:
PushDatasetDataChunkProps
[];
prompt
?:
string
;
agentModel
:
string
;
agentModel
:
string
;
vectorModel
:
string
;
vectorModel
:
string
;
...
...
packages/global/core/dataset/type.d.ts
View file @
36fafd21
...
@@ -36,6 +36,7 @@ export type ChunkSettingsType = {
...
@@ -36,6 +36,7 @@ export type ChunkSettingsType = {
// Index enhance
// Index enhance
imageIndex
?:
boolean
;
imageIndex
?:
boolean
;
autoIndexes
?:
boolean
;
autoIndexes
?:
boolean
;
indexPrefixTitle
?:
boolean
;
// Chunk setting
// Chunk setting
chunkSettingMode
?:
ChunkSettingModeEnum
;
// 系统参数/自定义参数
chunkSettingMode
?:
ChunkSettingModeEnum
;
// 系统参数/自定义参数
...
@@ -184,8 +185,6 @@ export type DatasetTrainingSchemaType = {
...
@@ -184,8 +185,6 @@ export type DatasetTrainingSchemaType = {
expireAt
:
Date
;
expireAt
:
Date
;
lockTime
:
Date
;
lockTime
:
Date
;
mode
:
TrainingModeEnum
;
mode
:
TrainingModeEnum
;
model
?:
string
;
prompt
?:
string
;
dataId
?:
string
;
dataId
?:
string
;
q
:
string
;
q
:
string
;
a
:
string
;
a
:
string
;
...
...
packages/service/core/dataset/collection/controller.ts
View file @
36fafd21
...
@@ -103,6 +103,7 @@ export const createCollectionAndInsertData = async ({
...
@@ -103,6 +103,7 @@ export const createCollectionAndInsertData = async ({
delete
formatCreateCollectionParams
.
chunkSize
;
delete
formatCreateCollectionParams
.
chunkSize
;
delete
formatCreateCollectionParams
.
chunkSplitter
;
delete
formatCreateCollectionParams
.
chunkSplitter
;
delete
formatCreateCollectionParams
.
indexSize
;
delete
formatCreateCollectionParams
.
indexSize
;
delete
formatCreateCollectionParams
.
indexPrefixTitle
;
}
}
}
}
if
(
trainingType
!==
DatasetCollectionDataProcessModeEnum
.
qa
)
{
if
(
trainingType
!==
DatasetCollectionDataProcessModeEnum
.
qa
)
{
...
@@ -223,7 +224,6 @@ export const createCollectionAndInsertData = async ({
...
@@ -223,7 +224,6 @@ export const createCollectionAndInsertData = async ({
vlmModel
:
dataset
.
vlmModel
,
vlmModel
:
dataset
.
vlmModel
,
indexSize
,
indexSize
,
mode
:
trainingMode
,
mode
:
trainingMode
,
prompt
:
formatCreateCollectionParams
.
qaPrompt
,
billId
:
traingBillId
,
billId
:
traingBillId
,
data
:
chunks
.
map
((
item
,
index
)
=>
({
data
:
chunks
.
map
((
item
,
index
)
=>
({
...
item
,
...
item
,
...
...
packages/service/core/dataset/schema.ts
View file @
36fafd21
...
@@ -32,6 +32,7 @@ export const ChunkSettings = {
...
@@ -32,6 +32,7 @@ export const ChunkSettings = {
imageIndex
:
Boolean
,
imageIndex
:
Boolean
,
autoIndexes
:
Boolean
,
autoIndexes
:
Boolean
,
indexPrefixTitle
:
Boolean
,
chunkSettingMode
:
{
chunkSettingMode
:
{
type
:
String
,
type
:
String
,
...
...
packages/service/core/dataset/training/controller.ts
View file @
36fafd21
...
@@ -27,23 +27,6 @@ export const lockTrainingDataByTeamId = async (teamId: string): Promise<any> =>
...
@@ -27,23 +27,6 @@ export const lockTrainingDataByTeamId = async (teamId: string): Promise<any> =>
}
catch
(
error
)
{}
}
catch
(
error
)
{}
};
};
export
const
pushDataListToTrainingQueueByCollectionId
=
async
({
collectionId
,
...
props
}:
Omit
<
PushDataToTrainingQueueProps
,
'datasetId'
|
'agentModel'
|
'vectorModel'
|
'vlmModel'
>
)
=>
{
const
{
dataset
:
{
_id
:
datasetId
,
agentModel
,
vectorModel
,
vlmModel
}
}
=
await
getCollectionWithDataset
(
collectionId
);
return
pushDataListToTrainingQueue
({
...
props
,
datasetId
,
collectionId
,
vectorModel
,
agentModel
,
vlmModel
});
};
export
async
function
pushDataListToTrainingQueue
({
export
async
function
pushDataListToTrainingQueue
({
teamId
,
teamId
,
tmbId
,
tmbId
,
...
@@ -53,7 +36,6 @@ export async function pushDataListToTrainingQueue({
...
@@ -53,7 +36,6 @@ export async function pushDataListToTrainingQueue({
vectorModel
,
vectorModel
,
vlmModel
,
vlmModel
,
data
,
data
,
prompt
,
billId
,
billId
,
mode
=
TrainingModeEnum
.
chunk
,
mode
=
TrainingModeEnum
.
chunk
,
indexSize
,
indexSize
,
...
@@ -149,8 +131,6 @@ export async function pushDataListToTrainingQueue({
...
@@ -149,8 +131,6 @@ export async function pushDataListToTrainingQueue({
collectionId
:
collectionId
,
collectionId
:
collectionId
,
billId
,
billId
,
mode
:
formatTrainingMode
(
item
,
mode
),
mode
:
formatTrainingMode
(
item
,
mode
),
prompt
,
model
,
...(
item
.
q
&&
{
q
:
item
.
q
}),
...(
item
.
q
&&
{
q
:
item
.
q
}),
...(
item
.
a
&&
{
a
:
item
.
a
}),
...(
item
.
a
&&
{
a
:
item
.
a
}),
...(
item
.
imageId
&&
{
imageId
:
item
.
imageId
}),
...(
item
.
imageId
&&
{
imageId
:
item
.
imageId
}),
...
...
packages/service/core/dataset/training/schema.ts
View file @
36fafd21
...
@@ -10,6 +10,7 @@ import {
...
@@ -10,6 +10,7 @@ import {
TeamMemberCollectionName
TeamMemberCollectionName
}
from
'@fastgpt/global/support/user/team/constant'
;
}
from
'@fastgpt/global/support/user/team/constant'
;
import
{
DatasetDataIndexTypeEnum
}
from
'@fastgpt/global/core/dataset/data/constants'
;
import
{
DatasetDataIndexTypeEnum
}
from
'@fastgpt/global/core/dataset/data/constants'
;
import
{
DatasetDataCollectionName
}
from
'../data/schema'
;
export
const
DatasetTrainingCollectionName
=
'dataset_trainings'
;
export
const
DatasetTrainingCollectionName
=
'dataset_trainings'
;
...
@@ -54,8 +55,6 @@ const TrainingDataSchema = new Schema({
...
@@ -54,8 +55,6 @@ const TrainingDataSchema = new Schema({
default
:
5
default
:
5
},
},
model
:
String
,
prompt
:
String
,
q
:
{
q
:
{
type
:
String
,
type
:
String
,
default
:
''
default
:
''
...
@@ -74,7 +73,10 @@ const TrainingDataSchema = new Schema({
...
@@ -74,7 +73,10 @@ const TrainingDataSchema = new Schema({
type
:
Number
,
type
:
Number
,
default
:
0
default
:
0
},
},
dataId
:
Schema
.
Types
.
ObjectId
,
dataId
:
{
type
:
Schema
.
Types
.
ObjectId
,
ref
:
DatasetDataCollectionName
},
indexes
:
{
indexes
:
{
type
:
[
type
:
[
{
{
...
@@ -105,6 +107,12 @@ TrainingDataSchema.virtual('collection', {
...
@@ -105,6 +107,12 @@ TrainingDataSchema.virtual('collection', {
foreignField
:
'_id'
,
foreignField
:
'_id'
,
justOne
:
true
justOne
:
true
});
});
TrainingDataSchema
.
virtual
(
'data'
,
{
ref
:
DatasetDataCollectionName
,
localField
:
'dataId'
,
foreignField
:
'_id'
,
justOne
:
true
});
try
{
try
{
// lock training data(teamId); delete training data
// lock training data(teamId); delete training data
...
...
packages/web/i18n/en/dataset.json
View file @
36fafd21
...
@@ -111,6 +111,8 @@
...
@@ -111,6 +111,8 @@
"import_param_setting"
:
"Parameter settings"
,
"import_param_setting"
:
"Parameter settings"
,
"import_select_file"
:
"Select a file"
,
"import_select_file"
:
"Select a file"
,
"import_select_link"
:
"Enter link"
,
"import_select_link"
:
"Enter link"
,
"index_prefix_title"
:
"Index add title"
,
"index_prefix_title_tips"
:
"Automatically add title names to all indexes"
,
"index_size"
:
"Index size"
,
"index_size"
:
"Index size"
,
"index_size_tips"
:
"When vectorized, the system will automatically further segment the blocks according to this size."
,
"index_size_tips"
:
"When vectorized, the system will automatically further segment the blocks according to this size."
,
"input_required_field_to_select_baseurl"
:
"Please enter the required information first"
,
"input_required_field_to_select_baseurl"
:
"Please enter the required information first"
,
...
...
packages/web/i18n/zh-CN/dataset.json
View file @
36fafd21
...
@@ -111,6 +111,8 @@
...
@@ -111,6 +111,8 @@
"import_param_setting"
:
"参数设置"
,
"import_param_setting"
:
"参数设置"
,
"import_select_file"
:
"选择文件"
,
"import_select_file"
:
"选择文件"
,
"import_select_link"
:
"输入链接"
,
"import_select_link"
:
"输入链接"
,
"index_prefix_title"
:
"将标题加入索引"
,
"index_prefix_title_tips"
:
"自动给索引所有索引加标题名"
,
"index_size"
:
"索引大小"
,
"index_size"
:
"索引大小"
,
"index_size_tips"
:
"向量化时内容的长度,系统会自动按该大小对分块进行进一步的分割。"
,
"index_size_tips"
:
"向量化时内容的长度,系统会自动按该大小对分块进行进一步的分割。"
,
"input_required_field_to_select_baseurl"
:
"请先输入必填信息"
,
"input_required_field_to_select_baseurl"
:
"请先输入必填信息"
,
...
...
packages/web/i18n/zh-Hant/dataset.json
View file @
36fafd21
...
@@ -110,6 +110,8 @@
...
@@ -110,6 +110,8 @@
"import_param_setting"
:
"參數設定"
,
"import_param_setting"
:
"參數設定"
,
"import_select_file"
:
"選擇文件"
,
"import_select_file"
:
"選擇文件"
,
"import_select_link"
:
"輸入連結"
,
"import_select_link"
:
"輸入連結"
,
"index_prefix_title"
:
"將標題加入索引"
,
"index_prefix_title_tips"
:
"自動給索引所有索引加標題名"
,
"index_size"
:
"索引大小"
,
"index_size"
:
"索引大小"
,
"index_size_tips"
:
"向量化時內容的長度,系統會自動按該大小對分塊進行進一步的分割。"
,
"index_size_tips"
:
"向量化時內容的長度,系統會自動按該大小對分塊進行進一步的分割。"
,
"input_required_field_to_select_baseurl"
:
"請先輸入必填信息"
,
"input_required_field_to_select_baseurl"
:
"請先輸入必填信息"
,
...
...
projects/app/src/components/core/chat/ChatContainer/ChatBox/Input/ChatInput.tsx
View file @
36fafd21
import
{
Box
,
Flex
,
Textarea
}
from
'@chakra-ui/react'
;
import
type
{
FlexProps
}
from
'@chakra-ui/react'
;
import
{
Box
,
Flex
,
Textarea
,
useBoolean
}
from
'@chakra-ui/react'
;
import
React
,
{
useRef
,
useCallback
,
useMemo
,
useState
}
from
'react'
;
import
React
,
{
useRef
,
useCallback
,
useMemo
,
useState
}
from
'react'
;
import
{
useTranslation
}
from
'next-i18next'
;
import
{
useTranslation
}
from
'next-i18next'
;
import
MyTooltip
from
'@fastgpt/web/components/common/MyTooltip'
;
import
MyTooltip
from
'@fastgpt/web/components/common/MyTooltip'
;
...
@@ -48,6 +49,8 @@ const ChatInput = ({
...
@@ -48,6 +49,8 @@ const ChatInput = ({
const
{
setValue
,
watch
,
control
}
=
chatForm
;
const
{
setValue
,
watch
,
control
}
=
chatForm
;
const
inputValue
=
watch
(
'input'
);
const
inputValue
=
watch
(
'input'
);
const
[
focusing
,
{
on
:
onFocus
,
off
:
offFocus
}]
=
useBoolean
();
// Check voice input state
// Check voice input state
const
[
mobilePreSpeak
,
setMobilePreSpeak
]
=
useState
(
false
);
const
[
mobilePreSpeak
,
setMobilePreSpeak
]
=
useState
(
false
);
...
@@ -207,6 +210,8 @@ const ChatInput = ({
...
@@ -207,6 +210,8 @@ const ChatInput = ({
}
}
}
}
}
}
}
}
onFocus=
{
onFocus
}
onBlur=
{
offFocus
}
/>
/>
</
Flex
>
</
Flex
>
</
Flex
>
</
Flex
>
...
@@ -254,7 +259,8 @@ const ChatInput = ({
...
@@ -254,7 +259,8 @@ const ChatInput = ({
borderRadius=
{
'sm'
}
borderRadius=
{
'sm'
}
cursor=
{
'pointer'
}
cursor=
{
'pointer'
}
_hover=
{
{
bg
:
'rgba(0, 0, 0, 0.04)'
}
}
_hover=
{
{
bg
:
'rgba(0, 0, 0, 0.04)'
}
}
onClick=
{
()
=>
{
onClick=
{
(
e
)
=>
{
e
.
stopPropagation
();
onOpenSelectFile
();
onOpenSelectFile
();
}
}
}
}
>
>
...
@@ -276,7 +282,8 @@ const ChatInput = ({
...
@@ -276,7 +282,8 @@ const ChatInput = ({
borderRadius=
{
'sm'
}
borderRadius=
{
'sm'
}
cursor=
{
'pointer'
}
cursor=
{
'pointer'
}
_hover=
{
{
bg
:
'rgba(0, 0, 0, 0.04)'
}
}
_hover=
{
{
bg
:
'rgba(0, 0, 0, 0.04)'
}
}
onClick=
{
()
=>
{
onClick=
{
(
e
)
=>
{
e
.
stopPropagation
();
VoiceInputRef
.
current
?.
onSpeak
?.();
VoiceInputRef
.
current
?.
onSpeak
?.();
}
}
}
}
>
>
...
@@ -307,7 +314,8 @@ const ChatInput = ({
...
@@ -307,7 +314,8 @@ const ChatInput = ({
}
}
borderRadius=
{
[
'md'
,
'lg'
]
}
borderRadius=
{
[
'md'
,
'lg'
]
}
cursor=
{
isChatting
?
'pointer'
:
canSendMessage
?
'pointer'
:
'not-allowed'
}
cursor=
{
isChatting
?
'pointer'
:
canSendMessage
?
'pointer'
:
'not-allowed'
}
onClick=
{
()
=>
{
onClick=
{
(
e
)
=>
{
e
.
stopPropagation
();
if
(
isChatting
)
{
if
(
isChatting
)
{
return
onStop
();
return
onStop
();
}
}
...
@@ -343,6 +351,11 @@ const ChatInput = ({
...
@@ -343,6 +351,11 @@ const ChatInput = ({
onStop
onStop
]);
]);
const
activeStyles
:
FlexProps
=
{
boxShadow
:
'0px 5px 20px -4px rgba(19, 51, 107, 0.13)'
,
border
:
'0.5px solid rgba(0, 0, 0, 0.24)'
};
return
(
return
(
<
Box
<
Box
m=
{
[
'0 auto 10px'
,
'10px auto'
]
}
m=
{
[
'0 auto 10px'
,
'10px auto'
]
}
...
@@ -381,12 +394,17 @@ const ChatInput = ({
...
@@ -381,12 +394,17 @@ const ChatInput = ({
pt=
{
fileList
.
length
>
0
?
'0'
:
mobilePreSpeak
?
[
0
,
4
]
:
[
3
,
4
]
}
pt=
{
fileList
.
length
>
0
?
'0'
:
mobilePreSpeak
?
[
0
,
4
]
:
[
3
,
4
]
}
pb=
{
[
2
,
4
]
}
pb=
{
[
2
,
4
]
}
position=
{
'relative'
}
position=
{
'relative'
}
boxShadow=
{
`0px 5px 16px -4px rgba(19, 51, 107, 0.08)`
}
borderRadius=
{
[
'xl'
,
'xxl'
]
}
borderRadius=
{
[
'xl'
,
'xxl'
]
}
bg=
{
'white'
}
bg=
{
'white'
}
overflow=
{
'display'
}
overflow=
{
'display'
}
border=
{
'0.5px solid rgba(0, 0, 0, 0.15)'
}
{
...
(
focusing
borderColor=
{
'rgba(0,0,0,0.12)'
}
?
activeStyles
:
{
_hover
:
activeStyles
,
border
:
'0
.
5
px
solid
rgba
(0,
0,
0,
0
.
18)',
boxShadow
:
`0
px
5
px
16
px
-4
px
rgba
(19,
51,
107,
0
.
08)`
})}
onClick=
{
()
=>
TextareaDom
?.
current
?.
focus
()
}
>
>
<
Box
flex=
{
1
}
>
<
Box
flex=
{
1
}
>
{
/* Chat input guide box */
}
{
/* Chat input guide box */
}
...
...
projects/app/src/pageComponents/dataset/detail/Form/CollectionChunkForm.tsx
View file @
36fafd21
...
@@ -13,7 +13,8 @@ import {
...
@@ -13,7 +13,8 @@ import {
Textarea
,
Textarea
,
useDisclosure
,
useDisclosure
,
Checkbox
,
Checkbox
,
HStack
HStack
,
Grid
}
from
'@chakra-ui/react'
;
}
from
'@chakra-ui/react'
;
import
MyIcon
from
'@fastgpt/web/components/common/Icon'
;
import
MyIcon
from
'@fastgpt/web/components/common/Icon'
;
import
LeftRadio
from
'@fastgpt/web/components/common/Radio/LeftRadio'
;
import
LeftRadio
from
'@fastgpt/web/components/common/Radio/LeftRadio'
;
...
@@ -35,7 +36,6 @@ import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContex
...
@@ -35,7 +36,6 @@ import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContex
import
MySelect
from
'@fastgpt/web/components/common/MySelect'
;
import
MySelect
from
'@fastgpt/web/components/common/MySelect'
;
import
{
import
{
chunkAutoChunkSize
,
chunkAutoChunkSize
,
getAutoIndexSize
,
getIndexSizeSelectList
,
getIndexSizeSelectList
,
getLLMDefaultChunkSize
,
getLLMDefaultChunkSize
,
getLLMMaxChunkSize
,
getLLMMaxChunkSize
,
...
@@ -44,7 +44,6 @@ import {
...
@@ -44,7 +44,6 @@ import {
minChunkSize
minChunkSize
}
from
'@fastgpt/global/core/dataset/training/utils'
;
}
from
'@fastgpt/global/core/dataset/training/utils'
;
import
RadioGroup
from
'@fastgpt/web/components/common/Radio/RadioGroup'
;
import
RadioGroup
from
'@fastgpt/web/components/common/Radio/RadioGroup'
;
import
type
{
LLMModelItemType
,
EmbeddingModelItemType
}
from
'@fastgpt/global/core/ai/model.d'
;
const
PromptTextarea
=
({
const
PromptTextarea
=
({
defaultValue
=
''
,
defaultValue
=
''
,
...
@@ -98,6 +97,7 @@ export type CollectionChunkFormType = {
...
@@ -98,6 +97,7 @@ export type CollectionChunkFormType = {
// Index enhance
// Index enhance
imageIndex
:
boolean
;
imageIndex
:
boolean
;
autoIndexes
:
boolean
;
autoIndexes
:
boolean
;
indexPrefixTitle
:
boolean
;
// Chunk setting
// Chunk setting
chunkSettingMode
:
ChunkSettingModeEnum
;
// 系统参数/自定义参数
chunkSettingMode
:
ChunkSettingModeEnum
;
// 系统参数/自定义参数
...
@@ -133,6 +133,7 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
...
@@ -133,6 +133,7 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
const
autoIndexes
=
watch
(
'autoIndexes'
);
const
autoIndexes
=
watch
(
'autoIndexes'
);
const
indexSize
=
watch
(
'indexSize'
);
const
indexSize
=
watch
(
'indexSize'
);
const
imageIndex
=
watch
(
'imageIndex'
);
const
imageIndex
=
watch
(
'imageIndex'
);
const
indexPrefixTitle
=
watch
(
'indexPrefixTitle'
);
const
paragraphChunkAIMode
=
watch
(
'paragraphChunkAIMode'
);
const
paragraphChunkAIMode
=
watch
(
'paragraphChunkAIMode'
);
const
trainingModeList
=
useMemo
(()
=>
{
const
trainingModeList
=
useMemo
(()
=>
{
...
@@ -282,48 +283,56 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
...
@@ -282,48 +283,56 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
</
Box
>
</
Box
>
)
}
)
}
{
trainingType
===
DatasetCollectionDataProcessModeEnum
.
chunk
&&
<
Box
mt=
{
6
}
>
feConfigs
?.
show_dataset_enhance
!==
false
&&
(
<
Box
fontSize=
{
'sm'
}
mb=
{
2
}
color=
{
'myGray.600'
}
>
<
Box
mt=
{
6
}
>
{
t
(
'dataset:enhanced_indexes'
)
}
<
Box
fontSize=
{
'sm'
}
mb=
{
2
}
color=
{
'myGray.600'
}
>
</
Box
>
{
t
(
'dataset:enhanced_indexes'
)
}
<
Grid
gridTemplateColumns=
{
'1fr 1fr'
}
rowGap=
{
[
2
,
4
]
}
columnGap=
{
[
3
,
7
]
}
>
</
Box
>
<
HStack
flex=
{
'1'
}
spacing=
{
1
}
>
<
HStack
gap=
{
[
3
,
7
]
}
>
<
Checkbox
isChecked=
{
indexPrefixTitle
}
{
...
register
('
indexPrefixTitle
')}
>
<
HStack
flex=
{
'1'
}
spacing=
{
1
}
>
<
FormLabel
>
{
t
(
'dataset:index_prefix_title'
)
}
</
FormLabel
>
<
MyTooltip
label=
{
!
feConfigs
?.
isPlus
?
t
(
'common:commercial_function_tip'
)
:
''
}
>
</
Checkbox
>
<
Checkbox
<
QuestionTip
label=
{
t
(
'dataset:index_prefix_title_tips'
)
}
/>
isDisabled=
{
!
feConfigs
?.
isPlus
}
</
HStack
>
isChecked=
{
autoIndexes
}
{
trainingType
===
DatasetCollectionDataProcessModeEnum
.
chunk
&&
{
...
register
('
autoIndexes
')}
feConfigs
?.
show_dataset_enhance
!==
false
&&
(
>
<>
<
FormLabel
>
{
t
(
'dataset:auto_indexes'
)
}
</
FormLabel
>
<
HStack
flex=
{
'1'
}
spacing=
{
1
}
>
</
Checkbox
>
<
MyTooltip
label=
{
!
feConfigs
?.
isPlus
?
t
(
'common:commercial_function_tip'
)
:
''
}
>
</
MyTooltip
>
<
Checkbox
<
QuestionTip
label=
{
t
(
'dataset:auto_indexes_tips'
)
}
/>
isDisabled=
{
!
feConfigs
?.
isPlus
}
</
HStack
>
isChecked=
{
autoIndexes
}
<
HStack
flex=
{
'1'
}
spacing=
{
1
}
>
{
...
register
('
autoIndexes
')}
<
MyTooltip
>
label=
{
<
FormLabel
>
{
t
(
'dataset:auto_indexes'
)
}
</
FormLabel
>
!
feConfigs
?.
isPlus
</
Checkbox
>
?
t
(
'common:commercial_function_tip'
)
</
MyTooltip
>
:
!
datasetDetail
?.
vlmModel
<
QuestionTip
label=
{
t
(
'dataset:auto_indexes_tips'
)
}
/>
?
t
(
'common:error_vlm_not_config'
)
</
HStack
>
:
''
<
HStack
flex=
{
'1'
}
spacing=
{
1
}
>
}
<
MyTooltip
>
label=
{
<
Checkbox
!
feConfigs
?.
isPlus
isDisabled=
{
!
feConfigs
?.
isPlus
||
!
datasetDetail
?.
vlmModel
}
?
t
(
'common:commercial_function_tip'
)
isChecked=
{
imageIndex
}
:
!
datasetDetail
?.
vlmModel
{
...
register
('
imageIndex
')}
?
t
(
'common:error_vlm_not_config'
)
:
''
}
>
>
<
FormLabel
>
{
t
(
'dataset:image_auto_parse'
)
}
</
FormLabel
>
<
Checkbox
</
Checkbox
>
isDisabled=
{
!
feConfigs
?.
isPlus
||
!
datasetDetail
?.
vlmModel
}
</
MyTooltip
>
isChecked=
{
imageIndex
}
<
QuestionTip
label=
{
t
(
'dataset:image_auto_parse_tips'
)
}
/>
{
...
register
('
imageIndex
')}
</
HStack
>
>
</
HStack
>
<
FormLabel
>
{
t
(
'dataset:image_auto_parse'
)
}
</
FormLabel
>
</
Box
>
</
Checkbox
>
)
}
</
MyTooltip
>
<
QuestionTip
label=
{
t
(
'dataset:image_auto_parse_tips'
)
}
/>
</
HStack
>
</>
)
}
</
Grid
>
</
Box
>
<
Box
mt=
{
6
}
>
<
Box
mt=
{
6
}
>
<
Box
fontSize=
{
'sm'
}
mb=
{
2
}
color=
{
'myGray.600'
}
>
<
Box
fontSize=
{
'sm'
}
mb=
{
2
}
color=
{
'myGray.600'
}
>
{
t
(
'dataset:chunk_process_params'
)
}
{
t
(
'dataset:chunk_process_params'
)
}
...
...
projects/app/src/pageComponents/dataset/detail/Import/Context.tsx
View file @
36fafd21
...
@@ -49,6 +49,7 @@ export const defaultFormData: ImportFormType = {
...
@@ -49,6 +49,7 @@ export const defaultFormData: ImportFormType = {
imageIndex
:
false
,
imageIndex
:
false
,
autoIndexes
:
false
,
autoIndexes
:
false
,
indexPrefixTitle
:
true
,
chunkSettingMode
:
ChunkSettingModeEnum
.
auto
,
chunkSettingMode
:
ChunkSettingModeEnum
.
auto
,
chunkSplitMode
:
DataChunkSplitModeEnum
.
paragraph
,
chunkSplitMode
:
DataChunkSplitModeEnum
.
paragraph
,
...
...
projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx
View file @
36fafd21
...
@@ -55,8 +55,9 @@ const ReTraining = () => {
...
@@ -55,8 +55,9 @@ const ReTraining = () => {
dataEnhanceCollectionName
:
dataEnhanceCollectionName
:
collection
.
dataEnhanceCollectionName
||
defaultFormData
.
dataEnhanceCollectionName
,
collection
.
dataEnhanceCollectionName
||
defaultFormData
.
dataEnhanceCollectionName
,
imageIndex
:
collection
.
imageIndex
||
defaultFormData
.
imageIndex
,
imageIndex
:
collection
.
imageIndex
??
defaultFormData
.
imageIndex
,
autoIndexes
:
collection
.
autoIndexes
||
defaultFormData
.
autoIndexes
,
autoIndexes
:
collection
.
autoIndexes
??
defaultFormData
.
autoIndexes
,
indexPrefixTitle
:
collection
.
indexPrefixTitle
??
defaultFormData
.
indexPrefixTitle
,
chunkSettingMode
:
collection
.
chunkSettingMode
||
defaultFormData
.
chunkSettingMode
,
chunkSettingMode
:
collection
.
chunkSettingMode
||
defaultFormData
.
chunkSettingMode
,
chunkSplitMode
:
collection
.
chunkSplitMode
||
defaultFormData
.
chunkSplitMode
,
chunkSplitMode
:
collection
.
chunkSplitMode
||
defaultFormData
.
chunkSplitMode
,
...
...
projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx
View file @
36fafd21
...
@@ -84,15 +84,13 @@ const InputDataModal = ({
...
@@ -84,15 +84,13 @@ const InputDataModal = ({
onSuccess
(
res
)
{
onSuccess
(
res
)
{
if
(
res
.
type
===
DatasetCollectionTypeEnum
.
images
)
{
if
(
res
.
type
===
DatasetCollectionTypeEnum
.
images
)
{
setCurrentTab
(
TabEnum
.
image
);
setCurrentTab
(
TabEnum
.
image
);
}
else
{
setCurrentTab
(
TabEnum
.
chunk
);
}
}
}
}
}
}
);
);
// Get data
// Get data
const
{
loading
:
isFetchingData
}
=
useRequest2
(
const
{
data
:
dataItem
,
loading
:
isFetchingData
}
=
useRequest2
(
async
()
=>
{
async
()
=>
{
if
(
dataId
)
return
getDatasetDataItemById
(
dataId
);
if
(
dataId
)
return
getDatasetDataItemById
(
dataId
);
return
null
;
return
null
;
...
@@ -125,6 +123,11 @@ const InputDataModal = ({
...
@@ -125,6 +123,11 @@ const InputDataModal = ({
}
}
);
);
useEffect
(()
=>
{
if
(
currentTab
||
!
dataItem
)
return
;
setCurrentTab
(
dataItem
.
a
?
TabEnum
.
qa
:
TabEnum
.
chunk
);
},
[
collection
,
dataItem
,
currentTab
]);
// Import new data
// Import new data
const
{
runAsync
:
sureImportData
,
loading
:
isImporting
}
=
useRequest2
(
const
{
runAsync
:
sureImportData
,
loading
:
isImporting
}
=
useRequest2
(
async
(
e
:
InputDataType
)
=>
{
async
(
e
:
InputDataType
)
=>
{
...
...
projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx
View file @
36fafd21
...
@@ -47,6 +47,10 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
...
@@ -47,6 +47,10 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
return
[
return
[
{
{
label
:
t
(
'common:core.dataset.collection.id'
),
value
:
collection
?.
_id
},
{
label
:
t
(
'common:core.dataset.collection.metadata.source'
),
label
:
t
(
'common:core.dataset.collection.metadata.source'
),
value
:
t
(
DatasetCollectionTypeMap
[
collection
.
type
]?.
name
as
any
)
value
:
t
(
DatasetCollectionTypeMap
[
collection
.
type
]?.
name
as
any
)
},
},
...
@@ -94,6 +98,14 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
...
@@ -94,6 +98,14 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
}
}
]
]
:
[]),
:
[]),
...(
collection
.
indexPrefixTitle
!==
undefined
?
[
{
label
:
t
(
'dataset:index_prefix_title'
),
value
:
collection
.
indexPrefixTitle
?
'Yes'
:
'No'
}
]
:
[]),
...(
collection
.
imageIndex
!==
undefined
...(
collection
.
imageIndex
!==
undefined
?
[
?
[
{
{
...
@@ -146,26 +158,22 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
...
@@ -146,26 +158,22 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
},
[
collection
,
t
]);
},
[
collection
,
t
]);
return
(
return
(
<
MyBox
isLoading=
{
isLoading
}
w=
{
'100%'
}
h=
{
'100%'
}
p=
{
6
}
>
<
MyBox
isLoading=
{
isLoading
}
w=
{
'100%'
}
h=
{
'100%'
}
p=
{
6
}
overflow=
{
'auto'
}
>
<
Box
fontSize=
{
'md'
}
pb=
{
4
}
>
<
Box
fontSize=
{
'md'
}
fontWeight=
{
'bold'
}
color=
{
'myGray.900'
}
pb=
{
4
}
>
{
t
(
'common:core.dataset.collection.metadata.metadata'
)
}
{
t
(
'common:core.dataset.collection.metadata.metadata'
)
}
</
Box
>
</
Box
>
<
Flex
mb=
{
3
}
wordBreak=
{
'break-all'
}
fontSize=
{
'sm'
}
>
<
Box
color=
{
'myGray.500'
}
flex=
{
'0 0 90px'
}
>
{
t
(
'common:core.dataset.collection.id'
)
}
:
</
Box
>
<
Box
>
{
collection
?.
_id
}
</
Box
>
</
Flex
>
{
metadataList
.
map
(
{
metadataList
.
map
(
(
item
,
i
)
=>
(
item
,
i
)
=>
item
.
label
&&
item
.
label
&&
item
.
value
&&
(
item
.
value
&&
(
<
Flex
key=
{
i
}
alignItems=
{
'center'
}
mb=
{
3
}
wordBreak=
{
'break-all'
}
fontSize=
{
'sm
'
}
>
<
Box
key=
{
i
}
mb=
{
3
}
wordBreak=
{
'break-all
'
}
>
<
Box
color=
{
'myGray.500'
}
f
lex=
{
'0 0 90px
'
}
>
<
Box
color=
{
'myGray.500'
}
f
ontSize=
{
'xs
'
}
>
{
item
.
label
}
{
item
.
label
}
</
Box
>
</
Box
>
<
Box
>
{
item
.
value
}
</
Box
>
<
Box
color=
{
'myGray.900'
}
fontSize=
{
'sm'
}
>
</
Flex
>
{
item
.
value
}
</
Box
>
</
Box
>
)
)
)
}
)
}
{
collection
?.
sourceId
&&
(
{
collection
?.
sourceId
&&
(
...
...
projects/app/src/pages/api/core/dataset/data/insertData.ts
View file @
36fafd21
...
@@ -48,7 +48,9 @@ async function handler(req: NextApiRequest) {
...
@@ -48,7 +48,9 @@ async function handler(req: NextApiRequest) {
const
[
const
[
{
{
dataset
:
{
_id
:
datasetId
,
vectorModel
,
agentModel
}
dataset
:
{
_id
:
datasetId
,
vectorModel
,
agentModel
},
indexPrefixTitle
,
name
}
}
]
=
await
Promise
.
all
([
getCollectionWithDataset
(
collectionId
)]);
]
=
await
Promise
.
all
([
getCollectionWithDataset
(
collectionId
)]);
...
@@ -84,6 +86,7 @@ async function handler(req: NextApiRequest) {
...
@@ -84,6 +86,7 @@ async function handler(req: NextApiRequest) {
q
:
formatQ
,
q
:
formatQ
,
a
:
formatA
,
a
:
formatA
,
chunkIndex
:
0
,
chunkIndex
:
0
,
indexPrefix
:
indexPrefixTitle
?
`#
${
name
}
`
:
undefined
,
embeddingModel
:
vectorModelData
.
model
,
embeddingModel
:
vectorModelData
.
model
,
indexes
:
formatIndexes
indexes
:
formatIndexes
});
});
...
...
projects/app/src/pages/api/core/dataset/data/update.ts
View file @
36fafd21
...
@@ -8,13 +8,16 @@ import { type ApiRequestProps } from '@fastgpt/service/type/next';
...
@@ -8,13 +8,16 @@ import { type ApiRequestProps } from '@fastgpt/service/type/next';
import
{
addOperationLog
}
from
'@fastgpt/service/support/operationLog/addOperationLog'
;
import
{
addOperationLog
}
from
'@fastgpt/service/support/operationLog/addOperationLog'
;
import
{
OperationLogEventEnum
}
from
'@fastgpt/global/support/operationLog/constants'
;
import
{
OperationLogEventEnum
}
from
'@fastgpt/global/support/operationLog/constants'
;
import
{
getI18nDatasetType
}
from
'@fastgpt/service/support/operationLog/util'
;
import
{
getI18nDatasetType
}
from
'@fastgpt/service/support/operationLog/util'
;
async
function
handler
(
req
:
ApiRequestProps
<
UpdateDatasetDataProps
>
)
{
async
function
handler
(
req
:
ApiRequestProps
<
UpdateDatasetDataProps
>
)
{
const
{
dataId
,
q
,
a
,
indexes
=
[]
}
=
req
.
body
;
const
{
dataId
,
q
,
a
,
indexes
=
[]
}
=
req
.
body
;
// auth data permission
// auth data permission
const
{
const
{
collection
:
{
collection
:
{
dataset
:
{
vectorModel
}
dataset
:
{
vectorModel
},
name
,
indexPrefixTitle
},
},
teamId
,
teamId
,
tmbId
,
tmbId
,
...
@@ -33,7 +36,8 @@ async function handler(req: ApiRequestProps<UpdateDatasetDataProps>) {
...
@@ -33,7 +36,8 @@ async function handler(req: ApiRequestProps<UpdateDatasetDataProps>) {
q
,
q
,
a
,
a
,
indexes
,
indexes
,
model
:
vectorModel
model
:
vectorModel
,
indexPrefix
:
indexPrefixTitle
?
`#
${
name
}
`
:
undefined
});
});
pushGenerateVectorUsage
({
pushGenerateVectorUsage
({
...
...
projects/app/src/pages/dataset/detail/index.tsx
View file @
36fafd21
...
@@ -41,7 +41,7 @@ type Props = { datasetId: string; currentTab: TabEnum };
...
@@ -41,7 +41,7 @@ type Props = { datasetId: string; currentTab: TabEnum };
const
sliderStyles
:
FlexProps
=
{
const
sliderStyles
:
FlexProps
=
{
bg
:
'white'
,
bg
:
'white'
,
borderRadius
:
'md'
,
borderRadius
:
'md'
,
overflowY
:
'
scroll
'
,
overflowY
:
'
auto
'
,
boxShadow
:
2
boxShadow
:
2
};
};
...
...
projects/app/src/service/core/dataset/data/controller.ts
View file @
36fafd21
...
@@ -25,13 +25,15 @@ const formatIndexes = async ({
...
@@ -25,13 +25,15 @@ const formatIndexes = async ({
q
,
q
,
a
=
''
,
a
=
''
,
indexSize
,
indexSize
,
maxIndexSize
maxIndexSize
,
indexPrefix
}:
{
}:
{
indexes
?:
(
Omit
<
DatasetDataIndexItemType
,
'dataId'
>
&
{
dataId
?:
string
})[];
indexes
?:
(
Omit
<
DatasetDataIndexItemType
,
'dataId'
>
&
{
dataId
?:
string
})[];
q
:
string
;
q
:
string
;
a
?:
string
;
a
?:
string
;
indexSize
:
number
;
indexSize
:
number
;
maxIndexSize
:
number
;
maxIndexSize
:
number
;
indexPrefix
?:
string
;
}):
Promise
<
}):
Promise
<
{
{
type
:
`
${
DatasetDataIndexTypeEnum
}
`
;
type
:
`
${
DatasetDataIndexTypeEnum
}
`
;
...
@@ -39,6 +41,12 @@ const formatIndexes = async ({
...
@@ -39,6 +41,12 @@ const formatIndexes = async ({
dataId
?:
string
;
dataId
?:
string
;
}[]
}[]
>
=>
{
>
=>
{
const
formatText
=
(
text
:
string
)
=>
{
if
(
indexPrefix
&&
!
text
.
startsWith
(
indexPrefix
))
{
return
`
${
indexPrefix
}
\n
${
text
}
`
;
}
return
text
;
};
/* get dataset data default index */
/* get dataset data default index */
const
getDefaultIndex
=
async
({
const
getDefaultIndex
=
async
({
q
=
''
,
q
=
''
,
...
@@ -62,11 +70,11 @@ const formatIndexes = async ({
...
@@ -62,11 +70,11 @@ const formatIndexes = async ({
return
[
return
[
...
qChunks
.
map
((
text
)
=>
({
...
qChunks
.
map
((
text
)
=>
({
text
,
text
:
formatText
(
text
)
,
type
:
DatasetDataIndexTypeEnum
.
default
type
:
DatasetDataIndexTypeEnum
.
default
})),
})),
...
aChunks
.
map
((
text
)
=>
({
...
aChunks
.
map
((
text
)
=>
({
text
,
text
:
formatText
(
text
)
,
type
:
DatasetDataIndexTypeEnum
.
default
type
:
DatasetDataIndexTypeEnum
.
default
}))
}))
];
];
...
@@ -130,9 +138,22 @@ const formatIndexes = async ({
...
@@ -130,9 +138,22 @@ const formatIndexes = async ({
return
item
;
return
item
;
})
})
)
)
).
flat
();
)
.
flat
()
.
filter
((
item
)
=>
!!
item
.
text
.
trim
());
// Add prefix
const
prefixIndexes
=
indexPrefix
?
chekcIndexes
.
map
((
index
)
=>
{
if
(
index
.
type
===
DatasetDataIndexTypeEnum
.
custom
)
return
index
;
return
{
...
index
,
text
:
formatText
(
index
.
text
)
};
})
:
chekcIndexes
;
return
chekcIndexes
.
filter
((
item
)
=>
!!
item
.
text
.
trim
())
;
return
prefixIndexes
;
};
};
/* insert data.
/* insert data.
* 1. create data id
* 1. create data id
...
@@ -150,6 +171,7 @@ export async function insertData2Dataset({
...
@@ -150,6 +171,7 @@ export async function insertData2Dataset({
chunkIndex
=
0
,
chunkIndex
=
0
,
indexSize
=
512
,
indexSize
=
512
,
indexes
,
indexes
,
indexPrefix
,
embeddingModel
,
embeddingModel
,
session
session
}:
CreateDatasetDataProps
&
{
}:
CreateDatasetDataProps
&
{
...
@@ -174,7 +196,8 @@ export async function insertData2Dataset({
...
@@ -174,7 +196,8 @@ export async function insertData2Dataset({
q
,
q
,
a
,
a
,
indexSize
,
indexSize
,
maxIndexSize
:
embModel
.
maxToken
maxIndexSize
:
embModel
.
maxToken
,
indexPrefix
});
});
// insert to vector store
// insert to vector store
...
@@ -255,7 +278,8 @@ export async function updateData2Dataset({
...
@@ -255,7 +278,8 @@ export async function updateData2Dataset({
a
,
a
,
indexes
,
indexes
,
model
,
model
,
indexSize
=
512
indexSize
=
512
,
indexPrefix
}:
UpdateDatasetDataProps
&
{
model
:
string
;
indexSize
?:
number
})
{
}:
UpdateDatasetDataProps
&
{
model
:
string
;
indexSize
?:
number
})
{
if
(
!
Array
.
isArray
(
indexes
))
{
if
(
!
Array
.
isArray
(
indexes
))
{
return
Promise
.
reject
(
'indexes is required'
);
return
Promise
.
reject
(
'indexes is required'
);
...
@@ -271,7 +295,8 @@ export async function updateData2Dataset({
...
@@ -271,7 +295,8 @@ export async function updateData2Dataset({
q
,
q
,
a
,
a
,
indexSize
,
indexSize
,
maxIndexSize
:
getEmbeddingModel
(
model
).
maxToken
maxIndexSize
:
getEmbeddingModel
(
model
).
maxToken
,
indexPrefix
});
});
// 3. Patch indexes, create, update, delete
// 3. Patch indexes, create, update, delete
...
...
projects/app/src/service/core/dataset/queues/datasetParse.ts
View file @
36fafd21
...
@@ -101,21 +101,13 @@ export const datasetParseQueue = async (): Promise<any> => {
...
@@ -101,21 +101,13 @@ export const datasetParseQueue = async (): Promise<any> => {
$inc
:
{
retryCount
:
-
1
}
$inc
:
{
retryCount
:
-
1
}
}
}
)
)
.
select
({
_id
:
1
,
teamId
:
1
,
tmbId
:
1
,
datasetId
:
1
,
collectionId
:
1
,
billId
:
1
,
q
:
1
})
.
populate
<
{
.
populate
<
{
dataset
:
DatasetSchemaType
;
dataset
:
DatasetSchemaType
;
collection
:
DatasetCollectionSchemaType
;
collection
:
DatasetCollectionSchemaType
;
}
>
([
}
>
([
{
{
path
:
'collection'
path
:
'collection'
,
select
:
'-qaPrompt'
},
},
{
{
path
:
'dataset'
path
:
'dataset'
...
@@ -300,7 +292,6 @@ export const datasetParseQueue = async (): Promise<any> => {
...
@@ -300,7 +292,6 @@ export const datasetParseQueue = async (): Promise<any> => {
vlmModel
:
dataset
.
vlmModel
,
vlmModel
:
dataset
.
vlmModel
,
indexSize
:
collection
.
indexSize
,
indexSize
:
collection
.
indexSize
,
mode
:
trainingMode
,
mode
:
trainingMode
,
prompt
:
collection
.
qaPrompt
,
billId
:
data
.
billId
,
billId
:
data
.
billId
,
data
:
chunks
.
map
((
item
,
index
)
=>
({
data
:
chunks
.
map
((
item
,
index
)
=>
({
...
item
,
...
item
,
...
...
projects/app/src/service/core/dataset/queues/generateQA.ts
View file @
36fafd21
...
@@ -14,7 +14,6 @@ import {
...
@@ -14,7 +14,6 @@ import {
countGptMessagesTokens
,
countGptMessagesTokens
,
countPromptTokens
countPromptTokens
}
from
'@fastgpt/service/common/string/tiktoken/index'
;
}
from
'@fastgpt/service/common/string/tiktoken/index'
;
import
{
pushDataListToTrainingQueueByCollectionId
}
from
'@fastgpt/service/core/dataset/training/controller'
;
import
{
loadRequestMessages
}
from
'@fastgpt/service/core/chat/utils'
;
import
{
loadRequestMessages
}
from
'@fastgpt/service/core/chat/utils'
;
import
{
llmCompletionsBodyFormat
,
formatLLMResponse
}
from
'@fastgpt/service/core/ai/utils'
;
import
{
llmCompletionsBodyFormat
,
formatLLMResponse
}
from
'@fastgpt/service/core/ai/utils'
;
import
type
{
LLMModelItemType
}
from
'@fastgpt/global/core/ai/model.d'
;
import
type
{
LLMModelItemType
}
from
'@fastgpt/global/core/ai/model.d'
;
...
@@ -24,6 +23,7 @@ import {
...
@@ -24,6 +23,7 @@ import {
}
from
'@fastgpt/global/core/dataset/training/utils'
;
}
from
'@fastgpt/global/core/dataset/training/utils'
;
import
{
getErrText
}
from
'@fastgpt/global/common/error/utils'
;
import
{
getErrText
}
from
'@fastgpt/global/common/error/utils'
;
import
{
text2Chunks
}
from
'@fastgpt/service/worker/function'
;
import
{
text2Chunks
}
from
'@fastgpt/service/worker/function'
;
import
{
pushDataListToTrainingQueue
}
from
'@fastgpt/service/core/dataset/training/controller'
;
const
reduceQueue
=
()
=>
{
const
reduceQueue
=
()
=>
{
global
.
qaQueueLen
=
global
.
qaQueueLen
>
0
?
global
.
qaQueueLen
-
1
:
0
;
global
.
qaQueueLen
=
global
.
qaQueueLen
>
0
?
global
.
qaQueueLen
-
1
:
0
;
...
@@ -41,6 +41,11 @@ const reduceQueueAndReturn = (delay = 0) => {
...
@@ -41,6 +41,11 @@ const reduceQueueAndReturn = (delay = 0) => {
}
}
};
};
type
PopulateType
=
{
dataset
:
{
vectorModel
:
string
;
agentModel
:
string
;
vlmModel
:
string
};
collection
:
{
qaPrompt
?:
string
};
};
export
async
function
generateQA
():
Promise
<
any
>
{
export
async
function
generateQA
():
Promise
<
any
>
{
const
max
=
global
.
systemEnv
?.
qaMaxProcess
||
10
;
const
max
=
global
.
systemEnv
?.
qaMaxProcess
||
10
;
addLog
.
debug
(
`[QA Queue] Queue size:
${
global
.
qaQueueLen
}
`
);
addLog
.
debug
(
`[QA Queue] Queue size:
${
global
.
qaQueueLen
}
`
);
...
@@ -68,18 +73,16 @@ export async function generateQA(): Promise<any> {
...
@@ -68,18 +73,16 @@ export async function generateQA(): Promise<any> {
$inc
:
{
retryCount
:
-
1
}
$inc
:
{
retryCount
:
-
1
}
}
}
)
)
.
select
({
.
populate
<
PopulateType
>
([
_id
:
1
,
{
teamId
:
1
,
path
:
'dataset'
,
tmbId
:
1
,
select
:
'agentModel vectorModel vlmModel'
datasetId
:
1
,
},
collectionId
:
1
,
{
q
:
1
,
path
:
'collection'
,
model
:
1
,
select
:
'qaPrompt'
chunkIndex
:
1
,
}
billId
:
1
,
])
prompt
:
1
})
.
lean
();
.
lean
();
// task preemption
// task preemption
...
@@ -110,6 +113,13 @@ export async function generateQA(): Promise<any> {
...
@@ -110,6 +113,13 @@ export async function generateQA(): Promise<any> {
return
reduceQueueAndReturn
();
return
reduceQueueAndReturn
();
}
}
if
(
!
data
.
dataset
||
!
data
.
collection
)
{
addLog
.
info
(
`[QA Queue] Dataset or collection not found`
,
data
);
// Delete data
await
MongoDatasetTraining
.
deleteOne
({
_id
:
data
.
_id
});
return
reduceQueueAndReturn
();
}
// auth balance
// auth balance
if
(
!
(
await
checkTeamAiPointsAndLock
(
data
.
teamId
)))
{
if
(
!
(
await
checkTeamAiPointsAndLock
(
data
.
teamId
)))
{
return
reduceQueueAndReturn
();
return
reduceQueueAndReturn
();
...
@@ -117,8 +127,8 @@ export async function generateQA(): Promise<any> {
...
@@ -117,8 +127,8 @@ export async function generateQA(): Promise<any> {
addLog
.
info
(
`[QA Queue] Start`
);
addLog
.
info
(
`[QA Queue] Start`
);
try
{
try
{
const
modelData
=
getLLMModel
(
data
.
m
odel
);
const
modelData
=
getLLMModel
(
data
.
dataset
.
agentM
odel
);
const
prompt
=
`
${
data
.
p
rompt
||
Prompt_AgentQA
.
description
}
const
prompt
=
`
${
data
.
collection
.
qaP
rompt
||
Prompt_AgentQA
.
description
}
${
replaceVariable
(
Prompt_AgentQA
.
fixedText
,
{
text
})}
`
;
${
replaceVariable
(
Prompt_AgentQA
.
fixedText
,
{
text
})}
`
;
// request LLM to get QA
// request LLM to get QA
...
@@ -147,16 +157,20 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`;
...
@@ -147,16 +157,20 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`;
const
qaArr
=
await
formatSplitText
({
answer
,
rawText
:
text
,
llmModel
:
modelData
});
// 格式化后的QA对
const
qaArr
=
await
formatSplitText
({
answer
,
rawText
:
text
,
llmModel
:
modelData
});
// 格式化后的QA对
// get vector and insert
// get vector and insert
await
pushDataListToTrainingQueue
ByCollectionId
({
await
pushDataListToTrainingQueue
({
teamId
:
data
.
teamId
,
teamId
:
data
.
teamId
,
tmbId
:
data
.
tmbId
,
tmbId
:
data
.
tmbId
,
datasetId
:
data
.
datasetId
,
collectionId
:
data
.
collectionId
,
collectionId
:
data
.
collectionId
,
mode
:
TrainingModeEnum
.
chunk
,
mode
:
TrainingModeEnum
.
chunk
,
data
:
qaArr
.
map
((
item
)
=>
({
data
:
qaArr
.
map
((
item
)
=>
({
...
item
,
...
item
,
chunkIndex
:
data
.
chunkIndex
chunkIndex
:
data
.
chunkIndex
})),
})),
billId
:
data
.
billId
billId
:
data
.
billId
,
vectorModel
:
data
.
dataset
.
vectorModel
,
agentModel
:
data
.
dataset
.
agentModel
,
vlmModel
:
data
.
dataset
.
vlmModel
});
});
// delete data from training
// delete data from training
...
@@ -192,7 +206,7 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`;
...
@@ -192,7 +206,7 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`;
}
}
);
);
return
reduceQueueAndReturn
(
10
00
);
return
reduceQueueAndReturn
(
5
00
);
}
}
}
}
...
...
projects/app/src/service/core/dataset/queues/generateVector.ts
View file @
36fafd21
...
@@ -12,10 +12,13 @@ import {
...
@@ -12,10 +12,13 @@ import {
}
from
'@fastgpt/service/common/vectorDB/controller'
;
}
from
'@fastgpt/service/common/vectorDB/controller'
;
import
{
getEmbeddingModel
}
from
'@fastgpt/service/core/ai/model'
;
import
{
getEmbeddingModel
}
from
'@fastgpt/service/core/ai/model'
;
import
{
mongoSessionRun
}
from
'@fastgpt/service/common/mongo/sessionRun'
;
import
{
mongoSessionRun
}
from
'@fastgpt/service/common/mongo/sessionRun'
;
import
{
type
DatasetTrainingSchemaType
}
from
'@fastgpt/global/core/dataset/type'
;
import
type
{
Document
}
from
'@fastgpt/service/common/mongo'
;
import
{
getErrText
}
from
'@fastgpt/global/common/error/utils'
;
import
{
getErrText
}
from
'@fastgpt/global/common/error/utils'
;
import
{
getMaxIndexSize
}
from
'@fastgpt/global/core/dataset/training/utils'
;
import
{
getMaxIndexSize
}
from
'@fastgpt/global/core/dataset/training/utils'
;
import
type
{
DatasetDataSchemaType
,
DatasetTrainingSchemaType
}
from
'@fastgpt/global/core/dataset/type'
;
import
{
retryFn
}
from
'@fastgpt/global/common/system/utils'
;
const
reduceQueue
=
()
=>
{
const
reduceQueue
=
()
=>
{
global
.
vectorQueueLen
=
global
.
vectorQueueLen
>
0
?
global
.
vectorQueueLen
-
1
:
0
;
global
.
vectorQueueLen
=
global
.
vectorQueueLen
>
0
?
global
.
vectorQueueLen
-
1
:
0
;
...
@@ -33,6 +36,13 @@ const reduceQueueAndReturn = (delay = 0) => {
...
@@ -33,6 +36,13 @@ const reduceQueueAndReturn = (delay = 0) => {
}
}
};
};
type
PopulateType
=
{
dataset
:
{
vectorModel
:
string
};
collection
:
{
name
:
string
;
indexPrefixTitle
:
boolean
};
data
:
{
_id
:
string
;
indexes
:
DatasetDataSchemaType
[
'indexes'
]
};
};
type
TrainingDataType
=
DatasetTrainingSchemaType
&
PopulateType
;
/* 索引生成队列。每导入一次,就是一个单独的线程 */
/* 索引生成队列。每导入一次,就是一个单独的线程 */
export
async
function
generateVector
():
Promise
<
any
>
{
export
async
function
generateVector
():
Promise
<
any
>
{
const
max
=
global
.
systemEnv
?.
vectorMaxProcess
||
10
;
const
max
=
global
.
systemEnv
?.
vectorMaxProcess
||
10
;
...
@@ -59,7 +69,22 @@ export async function generateVector(): Promise<any> {
...
@@ -59,7 +69,22 @@ export async function generateVector(): Promise<any> {
lockTime
:
new
Date
(),
lockTime
:
new
Date
(),
$inc
:
{
retryCount
:
-
1
}
$inc
:
{
retryCount
:
-
1
}
}
}
);
)
.
populate
<
PopulateType
>
([
{
path
:
'dataset'
,
select
:
'vectorModel'
},
{
path
:
'collection'
,
select
:
'name indexPrefixTitle'
},
{
path
:
'data'
,
select
:
'_id indexes'
}
])
.
lean
();
// task preemption
// task preemption
if
(
!
data
)
{
if
(
!
data
)
{
...
@@ -89,6 +114,13 @@ export async function generateVector(): Promise<any> {
...
@@ -89,6 +114,13 @@ export async function generateVector(): Promise<any> {
return
reduceQueueAndReturn
();
return
reduceQueueAndReturn
();
}
}
if
(
!
data
.
dataset
||
!
data
.
collection
)
{
addLog
.
info
(
`[Vector Queue] Dataset or collection not found`
,
data
);
// Delete data
await
MongoDatasetTraining
.
deleteOne
({
_id
:
data
.
_id
});
return
reduceQueueAndReturn
();
}
// auth balance
// auth balance
if
(
!
(
await
checkTeamAiPointsAndLock
(
data
.
teamId
)))
{
if
(
!
(
await
checkTeamAiPointsAndLock
(
data
.
teamId
)))
{
return
reduceQueueAndReturn
();
return
reduceQueueAndReturn
();
...
@@ -110,7 +142,7 @@ export async function generateVector(): Promise<any> {
...
@@ -110,7 +142,7 @@ export async function generateVector(): Promise<any> {
teamId
:
data
.
teamId
,
teamId
:
data
.
teamId
,
tmbId
:
data
.
tmbId
,
tmbId
:
data
.
tmbId
,
inputTokens
:
tokens
,
inputTokens
:
tokens
,
model
:
data
.
m
odel
,
model
:
data
.
dataset
.
vectorM
odel
,
billId
:
data
.
billId
billId
:
data
.
billId
});
});
...
@@ -131,75 +163,62 @@ export async function generateVector(): Promise<any> {
...
@@ -131,75 +163,62 @@ export async function generateVector(): Promise<any> {
errorMsg
:
getErrText
(
err
,
'unknown error'
)
errorMsg
:
getErrText
(
err
,
'unknown error'
)
}
}
);
);
return
reduceQueueAndReturn
(
10
00
);
return
reduceQueueAndReturn
(
5
00
);
}
}
}
}
const
rebuildData
=
async
({
const
rebuildData
=
async
({
trainingData
}:
{
trainingData
:
TrainingDataType
})
=>
{
trainingData
if
(
!
trainingData
.
data
)
{
}:
{
await
MongoDatasetTraining
.
deleteOne
({
_id
:
trainingData
.
_id
});
trainingData
:
Document
<
unknown
,
{},
DatasetTrainingSchemaType
>
&
Omit
<
DatasetTrainingSchemaType
&
Required
<
{
_id
:
string
;
}
>
,
never
>
;
})
=>
{
// find data
const
mongoData
=
await
MongoDatasetData
.
findById
(
trainingData
.
dataId
,
'indexes teamId datasetId collectionId'
);
if
(
!
mongoData
)
{
await
trainingData
.
deleteOne
();
return
Promise
.
reject
(
'Not data'
);
return
Promise
.
reject
(
'Not data'
);
}
}
const
deleteVectorIdList
=
mongoData
.
indexes
.
map
((
index
)
=>
index
.
dataId
);
// Old vectorId
const
deleteVectorIdList
=
trainingData
.
data
.
indexes
.
map
((
index
)
=>
index
.
dataId
);
// Find next rebuilding data to insert training queue
// Find next rebuilding data to insert training queue
await
mongoSessionRun
(
async
(
session
)
=>
{
try
{
// get new mongoData insert to training
await
retryFn
(()
=>
const
newRebuildingData
=
await
MongoDatasetData
.
findOneAndUpdate
(
mongoSessionRun
(
async
(
session
)
=>
{
{
// get new mongoData insert to training
rebuilding
:
true
,
const
newRebuildingData
=
await
MongoDatasetData
.
findOneAndUpdate
(
teamId
:
mongoData
.
teamId
,
datasetId
:
mongoData
.
datasetId
},
{
$unset
:
{
rebuilding
:
null
},
updateTime
:
new
Date
()
},
{
session
}
).
select
({
_id
:
1
,
collectionId
:
1
});
if
(
newRebuildingData
)
{
await
MongoDatasetTraining
.
create
(
[
{
{
teamId
:
mongoData
.
teamId
,
rebuilding
:
true
,
tmbId
:
trainingData
.
tmbId
,
teamId
:
trainingData
.
teamId
,
datasetId
:
mongoData
.
datasetId
,
datasetId
:
trainingData
.
datasetId
collectionId
:
newRebuildingData
.
collectionId
,
},
billId
:
trainingData
.
billId
,
{
mode
:
TrainingModeEnum
.
chunk
,
$unset
:
{
model
:
trainingData
.
model
,
rebuilding
:
null
dataId
:
newRebuildingData
.
_id
,
},
retryCount
:
50
updateTime
:
new
Date
()
}
},
],
{
session
}
{
session
,
ordered
:
true
}
).
select
({
);
_id
:
1
,
}
collectionId
:
1
});
});
if
(
newRebuildingData
)
{
await
MongoDatasetTraining
.
create
(
[
{
teamId
:
trainingData
.
teamId
,
tmbId
:
trainingData
.
tmbId
,
datasetId
:
trainingData
.
datasetId
,
collectionId
:
newRebuildingData
.
collectionId
,
billId
:
trainingData
.
billId
,
mode
:
TrainingModeEnum
.
chunk
,
dataId
:
newRebuildingData
.
_id
,
retryCount
:
50
}
],
{
session
,
ordered
:
true
}
);
}
})
);
}
catch
(
error
)
{}
// update vector, update dataset_data rebuilding status, delete data from training
// update vector, update dataset_data rebuilding status, delete data from training
// 1. Insert new vector to dataset_data
// 1. Insert new vector to dataset_data
...
@@ -208,28 +227,36 @@ const rebuildData = async ({
...
@@ -208,28 +227,36 @@ const rebuildData = async ({
insertId
:
string
;
insertId
:
string
;
}[]
=
[];
}[]
=
[];
let
i
=
0
;
let
i
=
0
;
for
await
(
const
index
of
mongoD
ata
.
indexes
)
{
for
await
(
const
index
of
trainingData
.
d
ata
.
indexes
)
{
const
result
=
await
insertDatasetDataVector
({
const
result
=
await
insertDatasetDataVector
({
query
:
index
.
text
,
query
:
index
.
text
,
model
:
getEmbeddingModel
(
trainingData
.
m
odel
),
model
:
getEmbeddingModel
(
trainingData
.
dataset
.
vectorM
odel
),
teamId
:
mongo
Data
.
teamId
,
teamId
:
training
Data
.
teamId
,
datasetId
:
mongo
Data
.
datasetId
,
datasetId
:
training
Data
.
datasetId
,
collectionId
:
mongo
Data
.
collectionId
collectionId
:
training
Data
.
collectionId
});
});
mongoD
ata
.
indexes
[
i
].
dataId
=
result
.
insertId
;
trainingData
.
d
ata
.
indexes
[
i
].
dataId
=
result
.
insertId
;
updateResult
.
push
(
result
);
updateResult
.
push
(
result
);
i
++
;
i
++
;
}
}
const
{
tokens
}
=
await
mongoSessionRun
(
async
(
session
)
=>
{
const
{
tokens
}
=
await
mongoSessionRun
(
async
(
session
)
=>
{
// 2. Ensure that the training data is deleted after the Mongo update is successful
// 2. Ensure that the training data is deleted after the Mongo update is successful
await
mongoData
.
save
({
session
});
await
MongoDatasetData
.
updateOne
(
{
_id
:
trainingData
.
data
.
_id
},
{
$set
:
{
indexes
:
trainingData
.
data
.
indexes
}
},
{
session
}
);
// 3. Delete the training data
// 3. Delete the training data
await
trainingData
.
deleteOne
(
{
session
});
await
MongoDatasetTraining
.
deleteOne
({
_id
:
trainingData
.
_id
},
{
session
});
// 4. Delete old vector
// 4. Delete old vector
await
deleteDatasetDataVector
({
await
deleteDatasetDataVector
({
teamId
:
mongo
Data
.
teamId
,
teamId
:
training
Data
.
teamId
,
idList
:
deleteVectorIdList
idList
:
deleteVectorIdList
});
});
...
@@ -241,19 +268,8 @@ const rebuildData = async ({
...
@@ -241,19 +268,8 @@ const rebuildData = async ({
return
{
tokens
};
return
{
tokens
};
};
};
const
insertData
=
async
({
const
insertData
=
async
({
trainingData
}:
{
trainingData
:
TrainingDataType
})
=>
{
trainingData
return
mongoSessionRun
(
async
(
session
)
=>
{
}:
{
trainingData
:
Document
<
unknown
,
{},
DatasetTrainingSchemaType
>
&
Omit
<
DatasetTrainingSchemaType
&
Required
<
{
_id
:
string
;
}
>
,
never
>
;
})
=>
{
const
{
tokens
}
=
await
mongoSessionRun
(
async
(
session
)
=>
{
// insert new data to dataset
// insert new data to dataset
const
{
tokens
}
=
await
insertData2Dataset
({
const
{
tokens
}
=
await
insertData2Dataset
({
teamId
:
trainingData
.
teamId
,
teamId
:
trainingData
.
teamId
,
...
@@ -264,18 +280,21 @@ const insertData = async ({
...
@@ -264,18 +280,21 @@ const insertData = async ({
a
:
trainingData
.
a
,
a
:
trainingData
.
a
,
imageId
:
trainingData
.
imageId
,
imageId
:
trainingData
.
imageId
,
chunkIndex
:
trainingData
.
chunkIndex
,
chunkIndex
:
trainingData
.
chunkIndex
,
indexSize
:
trainingData
.
indexSize
||
getMaxIndexSize
(
getEmbeddingModel
(
trainingData
.
model
)),
indexSize
:
trainingData
.
indexSize
||
getMaxIndexSize
(
getEmbeddingModel
(
trainingData
.
dataset
.
vectorModel
)),
indexes
:
trainingData
.
indexes
,
indexes
:
trainingData
.
indexes
,
embeddingModel
:
trainingData
.
model
,
indexPrefix
:
trainingData
.
collection
.
indexPrefixTitle
?
`#
${
trainingData
.
collection
.
name
}
`
:
undefined
,
embeddingModel
:
trainingData
.
dataset
.
vectorModel
,
session
session
});
});
// delete data from training
// delete data from training
await
trainingData
.
deleteOne
(
{
session
});
await
MongoDatasetTraining
.
deleteOne
({
_id
:
trainingData
.
_id
},
{
session
});
return
{
return
{
tokens
tokens
};
};
});
});
return
{
tokens
};
};
};
test/cases
/api/core/dataset/training/deleteTrainingData.test.ts
→
projects/app/test
/api/core/dataset/training/deleteTrainingData.test.ts
View file @
36fafd21
...
@@ -33,8 +33,7 @@ describe('delete training data test', () => {
...
@@ -33,8 +33,7 @@ describe('delete training data test', () => {
tmbId
:
root
.
tmbId
,
tmbId
:
root
.
tmbId
,
datasetId
:
dataset
.
_id
,
datasetId
:
dataset
.
_id
,
collectionId
:
collection
.
_id
,
collectionId
:
collection
.
_id
,
mode
:
TrainingModeEnum
.
chunk
,
mode
:
TrainingModeEnum
.
chunk
model
:
'test'
});
});
const
res
=
await
Call
<
deleteTrainingDataBody
,
{},
deleteTrainingDataResponse
>
(
handler
,
{
const
res
=
await
Call
<
deleteTrainingDataBody
,
{},
deleteTrainingDataResponse
>
(
handler
,
{
...
...
test/cases
/api/core/dataset/training/getTrainingDataDetail.test.ts
→
projects/app/test
/api/core/dataset/training/getTrainingDataDetail.test.ts
View file @
36fafd21
...
@@ -33,7 +33,6 @@ describe('get training data detail test', () => {
...
@@ -33,7 +33,6 @@ describe('get training data detail test', () => {
tmbId
:
root
.
tmbId
,
tmbId
:
root
.
tmbId
,
datasetId
:
dataset
.
_id
,
datasetId
:
dataset
.
_id
,
collectionId
:
collection
.
_id
,
collectionId
:
collection
.
_id
,
model
:
'test'
,
mode
:
TrainingModeEnum
.
chunk
,
mode
:
TrainingModeEnum
.
chunk
,
q
:
'test'
,
q
:
'test'
,
a
:
'test'
a
:
'test'
...
...
test/cases
/api/core/dataset/training/getTrainingError.test.ts
→
projects/app/test
/api/core/dataset/training/getTrainingError.test.ts
View file @
36fafd21
...
@@ -35,7 +35,6 @@ describe('training error list test', () => {
...
@@ -35,7 +35,6 @@ describe('training error list test', () => {
datasetId
:
dataset
.
_id
,
datasetId
:
dataset
.
_id
,
collectionId
:
collection
.
_id
,
collectionId
:
collection
.
_id
,
mode
:
TrainingModeEnum
.
chunk
,
mode
:
TrainingModeEnum
.
chunk
,
model
:
'test'
,
errorMsg
:
'test'
errorMsg
:
'test'
}))
}))
);
);
...
...
test/cases
/api/core/dataset/training/updateTrainingData.test.ts
→
projects/app/test
/api/core/dataset/training/updateTrainingData.test.ts
View file @
36fafd21
...
@@ -33,8 +33,7 @@ describe('update training data test', () => {
...
@@ -33,8 +33,7 @@ describe('update training data test', () => {
tmbId
:
root
.
tmbId
,
tmbId
:
root
.
tmbId
,
datasetId
:
dataset
.
_id
,
datasetId
:
dataset
.
_id
,
collectionId
:
collection
.
_id
,
collectionId
:
collection
.
_id
,
mode
:
TrainingModeEnum
.
chunk
,
mode
:
TrainingModeEnum
.
chunk
model
:
'test'
});
});
const
res
=
await
Call
<
updateTrainingDataBody
,
{},
updateTrainingDataResponse
>
(
handler
,
{
const
res
=
await
Call
<
updateTrainingDataBody
,
{},
updateTrainingDataResponse
>
(
handler
,
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment