Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit b3ba5de1 by light5980 Committed by GitHub
Options

feat: dataset error list (#7084)

parent 86a0ffe4
Showing with 3555 additions and 645 deletions
packages/global/core/dataset/constants.ts
......@@ -245,6 +245,12 @@ export enum TrainingModeEnum {
imageParse = 'imageParse'
}
export enum CollectionTrainingStatusEnum {
running = 'running',
error = 'error',
ready = 'ready'
}
/* ------------ search -------------- */
export enum DatasetSearchModeEnum {
embedding = 'embedding',
......
packages/global/core/dataset/type.ts
......@@ -7,6 +7,7 @@ import {
DatasetTypeEnum,
SearchScoreTypeEnum,
TrainingModeEnum,
CollectionTrainingStatusEnum,
ChunkSettingModeEnum,
ChunkTriggerConfigTypeEnum,
ParagraphChunkAIModeEnum
......@@ -262,6 +263,24 @@ export const CollectionWithDatasetSchema = DatasetCollectionSchema.extend({
});
export type CollectionWithDatasetType = z.infer<typeof CollectionWithDatasetSchema>;
export const CollectionTrainingStatusSchema = z.object({
trainingAmount: z.number().optional().default(0).meta({ description: '剩余训练数量' }),
activeTrainingAmount: z
.number()
.optional()
.default(0)
.meta({ description: '仍会继续处理的训练数量' }),
finalErrorAmount: z.number().optional().default(0).meta({ description: '最终/阻塞异常训练数量' }),
hasError: z.boolean().optional().default(false).meta({ description: '是否存在最终/阻塞异常' }),
slowestTrainingMode: z.enum(TrainingModeEnum).optional().meta({ description: '最慢训练阶段' }),
slowestTrainingStatus: z
.enum(CollectionTrainingStatusEnum)
.optional()
.default(CollectionTrainingStatusEnum.ready)
.meta({ description: '最慢训练阶段状态' })
});
export type CollectionTrainingStatusType = z.infer<typeof CollectionTrainingStatusSchema>;
/* ====== service type ===== */
/* ================= dataset ===================== */
......@@ -329,7 +348,7 @@ export const DatasetCollectionItemSchema = CollectionWithDatasetSchema.extend({
permission: PermissionSchema,
indexAmount: z.number().meta({ description: '索引数量' }),
errorCount: z.number().optional().meta({ description: '错误数量' })
});
}).merge(CollectionTrainingStatusSchema);
export type DatasetCollectionItemType = z.infer<typeof DatasetCollectionItemSchema>;
/* ================= data ===================== */
......
packages/global/openapi/core/dataset/collection/api.ts
......@@ -7,12 +7,11 @@ import { ObjectIdSchema } from '../../../../common/type/mongo';
import { OutLinkChatAuthSchema } from '../../../../support/permission/chat';
import {
DatasetCollectionSyncResultEnum,
DatasetCollectionTypeEnum,
DatasetCollectionDataProcessModeEnum,
TrainingModeEnum
} from '../../../../core/dataset/constants';
import {
ChunkSettingsSchema,
CollectionTrainingStatusSchema,
DatasetCollectionItemSchema,
DatasetCollectionSchema
} from '../../../../core/dataset/type';
......@@ -21,6 +20,9 @@ import { PaginationResponseSchema, PaginationSchema } from '../../../api';
import z from 'zod';
// ============= Scroll Collections =============
/**
* @deprecated Use ListCollectionV2BodySchema and /core/dataset/collection/listV2 instead.
*/
export const ScrollCollectionsBodySchema = z.object({
datasetId: z.string(),
parentId: z.string().nullable().optional().default(null),
......@@ -114,27 +116,27 @@ export const ListCollectionV2BodySchema = PaginationSchema.extend({
export type ListCollectionV2BodyType = z.infer<typeof ListCollectionV2BodySchema>;
// ============= List Collections V2 Response =============
export const DatasetCollectionsListItemSchema = z.object({
_id: ObjectIdSchema.meta({ description: '集合 ID' }),
parentId: DatasetCollectionSchema.shape.parentId,
tmbId: DatasetCollectionSchema.shape.tmbId,
name: DatasetCollectionSchema.shape.name,
type: DatasetCollectionSchema.shape.type,
createTime: DatasetCollectionSchema.shape.createTime,
updateTime: DatasetCollectionSchema.shape.updateTime,
forbid: DatasetCollectionSchema.shape.forbid,
trainingType: DatasetCollectionSchema.shape.trainingType,
tags: z.array(z.string()).optional().meta({ description: '标签' }),
externalFileId: z.string().optional().meta({ description: '外部文件 ID' }),
fileId: z.string().optional().meta({ description: '文件 ID' }),
rawLink: z.string().optional().meta({ description: '原始链接' }),
permission: PermissionSchema,
dataAmount: z.number().meta({ description: '数据数量' }),
trainingAmount: z.number().meta({ description: '训练数量' }),
hasError: z.boolean().optional().meta({ description: '是否错误' })
});
export const DatasetCollectionsListItemSchema = z
.object({
_id: ObjectIdSchema.meta({ description: '集合 ID' }),
parentId: DatasetCollectionSchema.shape.parentId,
tmbId: DatasetCollectionSchema.shape.tmbId,
name: DatasetCollectionSchema.shape.name,
type: DatasetCollectionSchema.shape.type,
createTime: DatasetCollectionSchema.shape.createTime,
updateTime: DatasetCollectionSchema.shape.updateTime,
forbid: DatasetCollectionSchema.shape.forbid,
trainingType: DatasetCollectionSchema.shape.trainingType,
tags: z.array(z.string()).optional().meta({ description: '标签' }),
externalFileId: z.string().optional().meta({ description: '外部文件 ID' }),
fileId: z.string().optional().meta({ description: '文件 ID' }),
rawLink: z.string().optional().meta({ description: '原始链接' }),
permission: PermissionSchema,
dataAmount: z.number().meta({ description: '数据数量' })
})
.merge(CollectionTrainingStatusSchema);
export type DatasetCollectionsListItemType = z.infer<typeof DatasetCollectionsListItemSchema>;
export const ListCollectionV2ResponseSchema = PaginationResponseSchema(
DatasetCollectionsListItemSchema
......
packages/global/openapi/core/dataset/collection/index.ts
......@@ -78,7 +78,8 @@ export const DatasetCollectionPath: OpenAPIPath = {
'/core/dataset/collection/scrollList': {
post: {
summary: '获取数据集集合列表（滚动分页）',
description: '获取数据集集合列表（滚动分页）',
description: '已废弃：获取数据集集合列表（滚动分页）。请改用 /core/dataset/collection/listV2',
deprecated: true,
tags: [TagsMap.datasetCollection],
requestBody: {
content: {
......
packages/global/openapi/core/dataset/training/api.ts
import { z } from 'zod';
import { ObjectIdSchema } from '../../../../common/type/mongo';
import { TrainingModeEnum } from '../../../../core/dataset/constants';
import { DatasetCollectionTypeEnum, TrainingModeEnum } from '../../../../core/dataset/constants';
import { DatasetTrainingSchema } from '../../../../core/dataset/type';
import { PaginationSchema, PaginationResponseSchema } from '../../../api';
......@@ -8,28 +8,50 @@ import { PaginationSchema, PaginationResponseSchema } from '../../../api';
* API: 更新训练数据（或重试所有错误数据）
* Route: PUT /api/core/dataset/training/updateTrainingData
* ============================================================================ */
export const UpdateTrainingDataBodySchema = z.object({
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
}),
dataId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a07',
description: '训练数据 ID，不传则重试集合内所有错误数据'
}),
q: z.string().optional().meta({
example: '什么是 FastGPT？',
description: '问题/主文本'
}),
a: z.string().optional().meta({
example: 'FastGPT 是一个 AI Agent 构建平台',
description: '回答/补充文本'
}),
chunkIndex: z.int().min(0).optional().meta({
example: 0,
description: '块索引'
export const UpdateTrainingDataBodySchema = z
.object({
datasetId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID，不传 dataId 时按知识库范围重试所有最终/阻塞异常'
}),
collectionId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID，不传 dataId 时按集合范围重试所有最终/阻塞异常'
}),
dataId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a07',
description: '训练数据 ID，传入则只重试或更新该训练数据'
}),
q: z.string().optional().meta({
example: '什么是 FastGPT？',
description: '问题/主文本'
}),
a: z.string().optional().meta({
example: 'FastGPT 是一个 AI Agent 构建平台',
description: '回答/补充文本'
}),
chunkIndex: z.int().min(0).optional().meta({
example: 0,
description: '块索引'
})
})
});
.superRefine((data, ctx) => {
if (!data.collectionId && !data.datasetId && !data.dataId) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
path: ['collectionId'],
message: 'collectionId, datasetId or dataId is required'
});
}
if (!data.dataId && data.collectionId && data.datasetId) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
path: ['collectionId'],
message: 'collectionId and datasetId cannot be used together without dataId'
});
}
});
export type UpdateTrainingDataBody = z.infer<typeof UpdateTrainingDataBodySchema>;
export const UpdateTrainingDataResponseSchema = z.undefined().meta({ description: '更新成功' });
......@@ -99,6 +121,10 @@ export const GetTrainingDataDetailResponseSchema = z
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
}),
mode: z.enum(TrainingModeEnum).meta({
example: TrainingModeEnum.chunk,
description: '训练模式'
......@@ -132,13 +158,135 @@ export const GetTrainingErrorBodySchema = PaginationSchema.extend({
});
export type GetTrainingErrorBody = z.infer<typeof GetTrainingErrorBodySchema>;
export const GetTrainingErrorResponseSchema = PaginationResponseSchema(
DatasetTrainingSchema.omit({ billId: true }).extend({
billId: z.string().optional()
export const TrainingErrorItemSchema = DatasetTrainingSchema.omit({ billId: true }).extend({
billId: z.string().optional()
});
export type TrainingErrorItemType = z.infer<typeof TrainingErrorItemSchema>;
export const TrainingErrorCollectionSchema = z.object({
_id: ObjectIdSchema.meta({ description: '集合 ID' }),
collectionId: ObjectIdSchema.meta({ description: '集合 ID' }),
name: z.string().meta({ description: '集合名称' }),
type: z.enum(DatasetCollectionTypeEnum).meta({ description: '集合类型' }),
sourceName: z.string().optional().meta({ description: '来源名称' }),
sourceId: z.string().optional().meta({ description: '来源 ID' })
});
export type TrainingErrorCollectionType = z.infer<typeof TrainingErrorCollectionSchema>;
export const TrainingErrorGroupSchema = z.object({
collection: TrainingErrorCollectionSchema.meta({ description: '集合信息' }),
items: z.array(TrainingErrorItemSchema).meta({ description: '异常训练记录列表' }),
errorCount: z.number().meta({
description: '该集合最终/阻塞异常总数'
}),
hasMoreItems: z.boolean().meta({
description: '该集合是否还有更多异常项未返回'
})
);
});
export type TrainingErrorGroupType = z.infer<typeof TrainingErrorGroupSchema>;
export const GetTrainingErrorResponseSchema = PaginationResponseSchema(TrainingErrorItemSchema);
export type GetTrainingErrorResponse = z.infer<typeof GetTrainingErrorResponseSchema>;
export const DatasetTrainingErrorPaginationLimits = {
maxPageSize: 20,
maxOffset: 100000,
maxPageNum: 5000,
defaultItemPageSize: 5,
maxItemOffset: 100000,
maxItemPageSize: 20
} as const;
/* ============================================================================
* API: 获取知识库训练错误列表（分页）
* Route: POST /api/core/dataset/training/getDatasetTrainingError
* ============================================================================ */
export const GetDatasetTrainingErrorBodySchema = PaginationSchema.extend({
pageSize: z.coerce
.number()
.int()
.min(1)
.max(DatasetTrainingErrorPaginationLimits.maxPageSize)
.optional()
.meta({
example: 10,
description: `每页集合数量，范围 [1, ${DatasetTrainingErrorPaginationLimits.maxPageSize}]`
}),
offset: z.coerce
.number()
.int()
.min(0)
.max(DatasetTrainingErrorPaginationLimits.maxOffset)
.optional()
.meta({
example: 0,
description: `集合分页偏移量，范围 [0, ${DatasetTrainingErrorPaginationLimits.maxOffset}]`
}),
pageNum: z.coerce
.number()
.int()
.min(1)
.max(DatasetTrainingErrorPaginationLimits.maxPageNum)
.optional()
.meta({
example: 1,
description: `集合分页页码，范围 [1, ${DatasetTrainingErrorPaginationLimits.maxPageNum}]`
}),
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
collectionId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID。传入时只分页加载该集合内的异常 chunk'
}),
itemOffset: z.coerce
.number()
.int()
.min(0)
.max(DatasetTrainingErrorPaginationLimits.maxItemOffset)
.optional()
.meta({
example: 5,
description: `集合内异常 chunk 偏移量，用于加载更多，范围 [0, ${DatasetTrainingErrorPaginationLimits.maxItemOffset}]`
}),
itemPageSize: z.coerce
.number()
.int()
.min(1)
.max(DatasetTrainingErrorPaginationLimits.maxItemPageSize)
.optional()
.meta({
example: DatasetTrainingErrorPaginationLimits.defaultItemPageSize,
description: `每个集合返回的异常 chunk 数量，范围 [1, ${DatasetTrainingErrorPaginationLimits.maxItemPageSize}]`
})
});
export type GetDatasetTrainingErrorBody = z.infer<typeof GetDatasetTrainingErrorBodySchema>;
export const GetDatasetTrainingErrorResponseSchema =
PaginationResponseSchema(TrainingErrorGroupSchema);
export type GetDatasetTrainingErrorResponse = z.infer<typeof GetDatasetTrainingErrorResponseSchema>;
/* ============================================================================
* API: 检查知识库是否存在训练错误
* Route: GET /api/core/dataset/training/hasDatasetTrainingError
* ============================================================================ */
export const HasDatasetTrainingErrorQuerySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
})
});
export type HasDatasetTrainingErrorQuery = z.infer<typeof HasDatasetTrainingErrorQuerySchema>;
export const HasDatasetTrainingErrorResponseSchema = z.object({
hasError: z.boolean().meta({
example: true,
description: '知识库内是否存在最终/阻塞异常训练记录'
})
});
export type HasDatasetTrainingErrorResponse = z.infer<typeof HasDatasetTrainingErrorResponseSchema>;
/* ============================================================================
* API: 获取数据集训练队列状态
* Route: GET /api/core/dataset/training/getDatasetTrainingQueue
......
packages/global/openapi/core/dataset/training/index.ts
......@@ -6,14 +6,17 @@ import {
DeleteTrainingDataBodySchema,
GetTrainingDataDetailBodySchema,
GetTrainingErrorBodySchema,
GetDatasetTrainingErrorBodySchema,
HasDatasetTrainingErrorQuerySchema,
GetDatasetTrainingQueueQuerySchema
} from './api';
export const DatasetTrainingPath: OpenAPIPath = {
'/core/dataset/training/updateTrainingData': {
put: {
summary: '更新训练数据',
description: '更新单条训练数据，或批量重试集合内所有错误数据（不传 dataId）',
summary: '更新训练数据或重试训练异常',
description:
'更新或重试训练异常：传 dataId 时重试或编辑后重训单个 chunk；不传 dataId 时，可按 collectionId 或 datasetId 重试全部最终/阻塞异常',
tags: [TagsMap.datasetTraining],
requestBody: {
content: {
......@@ -92,8 +95,8 @@ export const DatasetTrainingPath: OpenAPIPath = {
'/core/dataset/training/getTrainingError': {
post: {
summary: '获取训练错误列表',
description: '分页查询集合内训练失败的数据列表',
summary: '获取集合训练错误列表',
description: '分页查询集合内最终/阻塞异常训练记录',
tags: [TagsMap.datasetTraining],
requestBody: {
content: {
......@@ -110,6 +113,43 @@ export const DatasetTrainingPath: OpenAPIPath = {
}
},
'/core/dataset/training/getDatasetTrainingError': {
post: {
summary: '获取知识库训练错误列表',
description:
'分页查询知识库内存在最终/阻塞异常的集合，并返回每个集合内的部分异常 chunk；传 collectionId 时加载该集合更多异常 chunk',
tags: [TagsMap.datasetTraining],
requestBody: {
content: {
'application/json': {
schema: GetDatasetTrainingErrorBodySchema
}
}
},
responses: {
200: {
description: '成功返回知识库错误数据分页列表'
}
}
}
},
'/core/dataset/training/hasDatasetTrainingError': {
get: {
summary: '检查知识库是否存在训练错误',
description: '轻量检查知识库内是否存在最终/阻塞异常训练记录，用于列表徽章状态探测',
tags: [TagsMap.datasetTraining],
requestParams: {
query: HasDatasetTrainingErrorQuerySchema
},
responses: {
200: {
description: '成功返回是否存在训练错误'
}
}
}
},
'/core/dataset/training/getDatasetTrainingQueue': {
get: {
summary: '获取训练队列状态',
......
packages/service/core/dataset/training/constants.ts deleted 100644 → 0
export enum ImportDataSourceEnum {
fileLocal = 'fileLocal',
fileLink = 'fileLink',
fileCustom = 'fileCustom'
}
packages/service/core/dataset/training/controller.ts
......@@ -12,23 +12,54 @@ import { getLLMMaxChunkSize } from '../../../../global/core/dataset/training/uti
import { retryFn } from '@fastgpt/global/common/system/utils';
import { getLogger, LogCategories } from '../../../common/logger';
import { checkTimerLock, deleteTimerLock } from '../../../common/system/timerLock/utils';
import { BLOCKED_LOCK_TIME } from './query';
const logger = getLogger(LogCategories.MODULE.DATASET.TRAINING);
export const lockTrainingDataByTeamId = async (teamId: string): Promise<any> => {
export const lockTrainingDataByTeamId = async (
teamId: string,
currentTrainingId?: string
): Promise<any> => {
const timerId = `lock_training_data--${teamId}`;
const errorMsg = i18nT('common:code_error.team_error.ai_points_not_enough');
const lockCurrentTraining = () => {
if (!currentTrainingId) return Promise.resolve();
return MongoDatasetTraining.updateOne(
{
teamId,
_id: currentTrainingId
},
{
lockTime: BLOCKED_LOCK_TIME,
errorMsg
}
);
};
// 5 分钟闸门：并发/多节点调用时，只有首个抢到锁的会执行；TTL 作为兜底
const acquired = await checkTimerLock({ timerId, lockMinuted: 30 });
if (!acquired) return;
if (!acquired) {
// 其它 worker 已在执行团队级锁定时，当前已领取任务仍需要单独标记，避免最后一次重试被扣到 0 后不可见。
await lockCurrentTraining().catch((error) => {
logger.error('lock current training data failed', { teamId, currentTrainingId, error });
});
return;
}
try {
await MongoDatasetTraining.updateMany(
{
teamId
teamId,
$or: [
{ retryCount: { $gt: 0 } },
...(currentTrainingId ? [{ _id: currentTrainingId }] : [])
]
},
{
lockTime: new Date('2999/5/5')
lockTime: BLOCKED_LOCK_TIME,
errorMsg
}
);
} catch (error) {
......
packages/service/core/dataset/training/query.ts 0 → 100644
import {
CollectionTrainingStatusEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import type { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
type TrainingStatusCount = {
activeCount: number;
finalErrorCount: number;
};
export const BLOCKED_LOCK_TIME = new Date('2050-01-01');
export const trainingModeRankMap: Record<TrainingModeEnum, number> = {
[TrainingModeEnum.parse]: 0,
[TrainingModeEnum.imageParse]: 1,
[TrainingModeEnum.qa]: 2,
[TrainingModeEnum.image]: 3,
[TrainingModeEnum.auto]: 4,
[TrainingModeEnum.chunk]: 5
};
export const trainingModeRanks = Object.values(TrainingModeEnum).map((mode) => ({
mode,
rank: trainingModeRankMap[mode]
}));
const trimmedErrorMsgExpr = (fieldPath = '$errorMsg') => ({
$trim: {
input: {
$ifNull: [fieldPath, '']
}
}
});
/**
* 判断训练记录是否有有效错误信息。空字符串和纯空白字符串都视为无错误，
* 避免自动重试或历史脏数据被错误计入最终异常。
*/
export const hasEffectiveErrorMsg = (training?: Pick<DatasetTrainingSchemaType, 'errorMsg'>) => {
return typeof training?.errorMsg === 'string' && training.errorMsg.trim() !== '';
};
/**
* active 表示仍可能被训练队列继续处理的剩余任务，包含普通排队/训练中和自动重试中。
* 这里不判断 lockTime 是否已经到达队列可消费时间，只判断未被永久锁定。
*/
export const isActiveTraining = (
training?: Pick<DatasetTrainingSchemaType, 'retryCount' | 'lockTime'>
) => {
return (training?.retryCount ?? 0) > 0 && new Date(training?.lockTime ?? 0) < BLOCKED_LOCK_TIME;
};
export const isTemporarilyFailedTraining = (
training?: Pick<DatasetTrainingSchemaType, 'retryCount' | 'lockTime' | 'errorMsg'>
) => {
return hasEffectiveErrorMsg(training) && isActiveTraining(training);
};
export const isFinalErrorTraining = (
training?: Pick<DatasetTrainingSchemaType, 'retryCount' | 'lockTime' | 'errorMsg'>
) => {
return (
hasEffectiveErrorMsg(training) &&
((training?.retryCount ?? 0) <= 0 || new Date(training?.lockTime ?? 0) >= BLOCKED_LOCK_TIME)
);
};
export const isRemainingTraining = (
training?: Pick<DatasetTrainingSchemaType, 'retryCount' | 'lockTime' | 'errorMsg'>
) => {
return isActiveTraining(training) || isFinalErrorTraining(training);
};
export const hasEffectiveErrorMsgExpr = { $gt: [{ $strLenCP: trimmedErrorMsgExpr() }, 0] };
export const activeTrainingExpr = {
$and: [{ $gt: ['$retryCount', 0] }, { $lt: ['$lockTime', BLOCKED_LOCK_TIME] }]
};
export const finalErrorTrainingExpr = {
$and: [
hasEffectiveErrorMsgExpr,
{
$or: [{ $lte: ['$retryCount', 0] }, { $gte: ['$lockTime', BLOCKED_LOCK_TIME] }]
}
]
};
export const remainingTrainingExpr = {
$or: [activeTrainingExpr, finalErrorTrainingExpr]
};
export const hasEffectiveErrorMsgMatch = {
$expr: hasEffectiveErrorMsgExpr
};
export const activeTrainingMatch = {
retryCount: { $gt: 0 },
lockTime: { $lt: BLOCKED_LOCK_TIME }
};
export const finalErrorTrainingMatch = {
$expr: finalErrorTrainingExpr
};
export const remainingTrainingMatch = {
$or: [activeTrainingMatch, finalErrorTrainingMatch]
};
/**
* rank 越小表示流程越早；collection 的“最慢阶段”就是剩余任务里流程最早的阶段。
*/
export const getTrainingModeRank = (mode?: TrainingModeEnum) => {
if (!mode) return Number.MAX_SAFE_INTEGER;
return trainingModeRankMap[mode] ?? Number.MAX_SAFE_INTEGER;
};
/**
* 返回流程中更早的训练阶段，用于计算用户感知上的“最慢阶段”。
*/
export const compareTrainingModeBySlowest = (a?: TrainingModeEnum, b?: TrainingModeEnum) => {
return getTrainingModeRank(a) - getTrainingModeRank(b);
};
export const getSlowestTrainingMode = (modes: Array<TrainingModeEnum | undefined>) => {
return modes.filter(Boolean).sort((a, b) => compareTrainingModeBySlowest(a, b))[0] as
| TrainingModeEnum
| undefined;
};
/**
* 根据各阶段 active/final error 数量计算 collection 级最慢阶段状态。
* 最慢阶段只有最终异常时才展示 error。
*/
export const getSlowestTrainingStatus = (
modeCounts: Partial<Record<TrainingModeEnum, TrainingStatusCount>>
) => {
const slowestTrainingMode = getSlowestTrainingMode(
Object.entries(modeCounts)
.filter(([, count]) => (count?.activeCount ?? 0) + (count?.finalErrorCount ?? 0) > 0)
.map(([mode]) => mode as TrainingModeEnum)
);
if (!slowestTrainingMode) {
return {
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
}
const slowestCounts = modeCounts[slowestTrainingMode];
return {
slowestTrainingMode,
slowestTrainingStatus:
(slowestCounts?.activeCount ?? 0) > 0
? CollectionTrainingStatusEnum.running
: CollectionTrainingStatusEnum.error
};
};
packages/service/core/dataset/training/schema.ts
......@@ -11,7 +11,6 @@ import {
} from '@fastgpt/global/support/user/team/constant';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
import { DatasetDataCollectionName } from '../data/schema';
import { getLogger, LogCategories } from '../../../common/logger';
export const DatasetTrainingCollectionName = 'dataset_trainings';
......@@ -121,6 +120,12 @@ TrainingDataSchema.virtual('data', {
// lock training data(teamId); delete training data
TrainingDataSchema.index({ teamId: 1, datasetId: 1 });
// collection 级状态、错误列表、删除、详情
TrainingDataSchema.index({
teamId: 1,
datasetId: 1,
collectionId: 1
});
// get training data and sort
TrainingDataSchema.index({ mode: 1, retryCount: 1, lockTime: 1, weight: -1 });
TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 * 60 }); // 7 days
......
packages/service/support/permission/dataset/auth.ts
......@@ -138,7 +138,6 @@ export const authDataset = async ({
export async function authDatasetCollection({
collectionId,
per = NullPermissionVal,
isRoot = false,
...props
}: AuthModeType & {
collectionId: string;
......
packages/service/test/core/dataset/training/controller.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import { i18nT } from '@fastgpt/global/common/i18n/utils';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { lockTrainingDataByTeamId } from '@fastgpt/service/core/dataset/training/controller';
import {
BLOCKED_LOCK_TIME,
finalErrorTrainingMatch,
isFinalErrorTraining
} from '@fastgpt/service/core/dataset/training/query';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { getRootUser } from '@test/datas/users';
describe('dataset training controller', () => {
it('should lock retryable team trainings with AI points error message', async () => {
const root = await getRootUser();
const otherRoot = await getRootUser();
const datasetId = '507f1f77bcf86cd799439011';
const collectionId = '507f1f77bcf86cd799439012';
const billId = 'test';
const [retryable, exhausted, otherTeam] = await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 3
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 0
},
{
teamId: otherRoot.teamId,
tmbId: otherRoot.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 3
}
]);
await lockTrainingDataByTeamId(String(root.teamId));
const lockedTraining = await MongoDatasetTraining.findById(retryable._id).lean();
const exhaustedTraining = await MongoDatasetTraining.findById(exhausted._id).lean();
const otherTeamTraining = await MongoDatasetTraining.findById(otherTeam._id).lean();
const finalErrorLockedTrainingCount = await MongoDatasetTraining.countDocuments({
_id: retryable._id,
...finalErrorTrainingMatch
});
const errorMsg = i18nT('common:code_error.team_error.ai_points_not_enough');
expect(lockedTraining?.lockTime).toEqual(BLOCKED_LOCK_TIME);
expect(lockedTraining?.errorMsg).toBe(errorMsg);
expect(finalErrorLockedTrainingCount).toBe(1);
expect(
isFinalErrorTraining({
retryCount: lockedTraining?.retryCount,
lockTime: lockedTraining?.lockTime,
errorMsg: lockedTraining?.errorMsg
})
).toBe(true);
expect(exhaustedTraining?.lockTime).not.toEqual(BLOCKED_LOCK_TIME);
expect(exhaustedTraining?.errorMsg).toBeUndefined();
expect(otherTeamTraining?.lockTime).not.toEqual(BLOCKED_LOCK_TIME);
expect(otherTeamTraining?.errorMsg).toBeUndefined();
});
it('should lock the current picked training even when retry count is exhausted', async () => {
const root = await getRootUser();
const datasetId = '507f1f77bcf86cd799439021';
const collectionId = '507f1f77bcf86cd799439022';
const billId = 'test';
const [pickedTraining, exhaustedHistory] = await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 0
}
]);
await lockTrainingDataByTeamId(String(root.teamId), String(pickedTraining._id));
const lockedTraining = await MongoDatasetTraining.findById(pickedTraining._id).lean();
const untouchedTraining = await MongoDatasetTraining.findById(exhaustedHistory._id).lean();
const errorMsg = i18nT('common:code_error.team_error.ai_points_not_enough');
expect(lockedTraining?.lockTime).toEqual(BLOCKED_LOCK_TIME);
expect(lockedTraining?.errorMsg).toBe(errorMsg);
expect(untouchedTraining?.lockTime).not.toEqual(BLOCKED_LOCK_TIME);
expect(untouchedTraining?.errorMsg).toBeUndefined();
});
});
packages/service/test/core/dataset/training/query.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import {
CollectionTrainingStatusEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import {
BLOCKED_LOCK_TIME,
compareTrainingModeBySlowest,
getSlowestTrainingStatus,
getTrainingModeRank,
hasEffectiveErrorMsg,
isActiveTraining,
isFinalErrorTraining,
isRemainingTraining,
isTemporarilyFailedTraining
} from '@fastgpt/service/core/dataset/training/query';
describe('dataset training query helpers', () => {
it('treats empty or whitespace errorMsg as ineffective', () => {
expect(hasEffectiveErrorMsg({ errorMsg: undefined })).toBe(false);
expect(hasEffectiveErrorMsg({ errorMsg: '' })).toBe(false);
expect(hasEffectiveErrorMsg({ errorMsg: ' \n\t' })).toBe(false);
expect(hasEffectiveErrorMsg({ errorMsg: 'failed' })).toBe(true);
});
it('separates active, temporary failed and final error records', () => {
const active = {
retryCount: 1,
lockTime: new Date('2049-12-31'),
errorMsg: ''
};
const temporaryFailed = {
retryCount: 1,
lockTime: new Date('2049-12-31'),
errorMsg: ' failed '
};
const retryExhausted = {
retryCount: 0,
lockTime: new Date('2049-12-31'),
errorMsg: 'failed'
};
const permanentlyLocked = {
retryCount: 3,
lockTime: BLOCKED_LOCK_TIME,
errorMsg: 'failed'
};
expect(isActiveTraining(active)).toBe(true);
expect(isRemainingTraining(active)).toBe(true);
expect(isActiveTraining(temporaryFailed)).toBe(true);
expect(isTemporarilyFailedTraining(temporaryFailed)).toBe(true);
expect(isFinalErrorTraining(temporaryFailed)).toBe(false);
expect(isActiveTraining(retryExhausted)).toBe(false);
expect(isFinalErrorTraining(retryExhausted)).toBe(true);
expect(isRemainingTraining(retryExhausted)).toBe(true);
expect(isActiveTraining(permanentlyLocked)).toBe(false);
expect(isFinalErrorTraining(permanentlyLocked)).toBe(true);
});
it('orders modes by the earliest remaining stage as slowest', () => {
expect(getTrainingModeRank(TrainingModeEnum.parse)).toBeLessThan(
getTrainingModeRank(TrainingModeEnum.imageParse)
);
expect(getTrainingModeRank(TrainingModeEnum.imageParse)).toBeLessThan(
getTrainingModeRank(TrainingModeEnum.qa)
);
expect(
compareTrainingModeBySlowest(TrainingModeEnum.image, TrainingModeEnum.chunk)
).toBeLessThan(0);
});
it('returns running when the slowest stage still has active records', () => {
const status = getSlowestTrainingStatus({
[TrainingModeEnum.parse]: { activeCount: 1, finalErrorCount: 0 },
[TrainingModeEnum.chunk]: { activeCount: 0, finalErrorCount: 2 }
});
expect(status).toEqual({
slowestTrainingMode: TrainingModeEnum.parse,
slowestTrainingStatus: CollectionTrainingStatusEnum.running
});
});
it('returns error only when the slowest stage has final errors and no active records', () => {
const status = getSlowestTrainingStatus({
[TrainingModeEnum.image]: { activeCount: 0, finalErrorCount: 1 },
[TrainingModeEnum.chunk]: { activeCount: 3, finalErrorCount: 0 }
});
expect(status).toEqual({
slowestTrainingMode: TrainingModeEnum.image,
slowestTrainingStatus: CollectionTrainingStatusEnum.error
});
});
it('returns ready when there are no remaining records', () => {
expect(getSlowestTrainingStatus({})).toEqual({
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
});
});
});
packages/web/components/common/Tag/index.tsx
......@@ -4,10 +4,13 @@ import { Box, type BoxProps, Flex, type FlexProps } from '@chakra-ui/react';
export type ColorSchemaType =
| 'white'
| 'blue'
| 'cyan'
| 'green'
| 'red'
| 'lightRed'
| 'yellow'
| 'gray'
| 'lightGray'
| 'purple'
| 'adora';
......@@ -57,6 +60,21 @@ const colorMap: Record<
bg: 'primary.50',
color: 'primary.600'
},
cyan: {
borderColor: '#BCE7FF',
bg: '#F0FBFF',
color: '#0884DD'
},
lightRed: {
borderColor: 'red.200',
bg: 'red.50',
color: 'red.500'
},
lightGray: {
borderColor: 'myGray.200',
bg: 'myGray.100',
color: 'myGray.500'
},
purple: {
borderColor: '#ECF',
bg: '#F6EEFA',
......
packages/web/hooks/useScrollPagination.tsx
import React, { type ReactNode, type RefObject, useMemo, useRef, useState } from 'react';
import React, { type ReactNode, type RefObject, useRef, useState } from 'react';
import { Box, type BoxProps } from '@chakra-ui/react';
import { useToast } from './useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
......@@ -190,6 +190,7 @@ export function useScrollPagination<
EmptyTip,
showErrorToast = true,
disabled = false,
showNoMoreTip = true,
...props
}: {
......@@ -200,6 +201,7 @@ export function useScrollPagination<
EmptyTip?: React.JSX.Element;
showErrorToast?: boolean;
disabled?: boolean;
showNoMoreTip?: boolean;
} & Parameters<typeof useRequest>[1]
) {
const { t } = useTranslation();
......@@ -286,7 +288,7 @@ export function useScrollPagination<
}
);
let ScrollRef = useRef<HTMLDivElement>(null);
const ScrollRef = useRef<HTMLDivElement>(null);
const ScrollData = useMemoizedFn(
({
children,
......@@ -299,11 +301,11 @@ export function useScrollPagination<
ScrollContainerRef?: RefObject<HTMLDivElement>;
} & BoxProps) => {
const ref = ScrollContainerRef || ScrollRef;
const loadText = useMemo(() => {
const loadText = (() => {
if (isLoading || isLoadingProp) return t('common:is_requesting');
if (noMore) return t('common:request_end');
return t('common:request_more');
}, [isLoading, noMore]);
})();
const scroll = useScroll(ref);
......@@ -339,7 +341,7 @@ export function useScrollPagination<
</Box>
)}
{children}
{scrollLoadType === 'bottom' && !isEmpty && (
{scrollLoadType === 'bottom' && !isEmpty && (showNoMoreTip || !noMore) && (
<Box
mt={2}
fontSize={'xs'}
......
packages/web/i18n/en/dataset.json
......@@ -201,5 +201,9 @@
"website_info": "Website Information",
"yuque_dataset": "Yuque Knowledge Base",
"yuque_dataset_config": "Configure Yuque Knowledge Base",
"yuque_dataset_desc": "Build knowledge base using Yuque documents by configuring document permissions, documents will not be stored twice"
"yuque_dataset_desc": "Build knowledge base using Yuque documents by configuring document permissions, documents will not be stored twice",
"training.stage_error": "{{stage}} error",
"training.status_error": "Training error",
"training_error_list": "Error list",
"training_error_empty": "No training errors"
}
packages/web/i18n/zh-CN/dataset.json
......@@ -201,5 +201,9 @@
"website_info": "网站信息",
"yuque_dataset": "语雀知识库",
"yuque_dataset_config": "配置语雀知识库",
"yuque_dataset_desc": "可通过配置语雀文档权限，使用语雀文档构建知识库，文档不会进行二次存储"
"yuque_dataset_desc": "可通过配置语雀文档权限，使用语雀文档构建知识库，文档不会进行二次存储",
"training.stage_error": "{{stage}}异常",
"training.status_error": "训练异常",
"training_error_list": "异常列表",
"training_error_empty": "暂无训练异常"
}
packages/web/i18n/zh-Hant/dataset.json
......@@ -201,5 +201,9 @@
"website_info": "網站資訊",
"yuque_dataset": "語雀知識庫",
"yuque_dataset_config": "設定語雀知識庫",
"yuque_dataset_desc": "可透過設定語雀文件權限，使用語雀文件建構知識庫，文件不會進行二次儲存"
"yuque_dataset_desc": "可透過設定語雀文件權限，使用語雀文件建構知識庫，文件不會進行二次儲存",
"training.stage_error": "{{stage}}異常",
"training.status_error": "訓練異常",
"training_error_list": "異常列表",
"training_error_empty": "暫無訓練異常"
}
projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx
......@@ -40,7 +40,15 @@ const FileSourceSelector = dynamic(() => import('../Import/components/FileSource
const BackupImportModal = dynamic(() => import('./BackupImportModal'));
const TemplateImportModal = dynamic(() => import('./TemplateImportModal'));
const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
const Header = ({
hasTrainingData,
hasTrainingError,
onOpenTrainingErrorModal
}: {
hasTrainingData: boolean;
hasTrainingError: boolean;
onOpenTrainingErrorModal: () => void;
}) => {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
const { isPc } = useSystem();
......@@ -196,6 +204,23 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
{datasetDetail.type !== DatasetTypeEnum.websiteDataset &&
datasetDetail.permission.hasWritePer &&
feConfigs?.isPlus && <HeaderTagPopOver />}
{hasTrainingError && (
<Button
variant={'whiteBase'}
h={'36px'}
px={'14px'}
color={'#F97066'}
borderColor={'#F97066'}
_hover={{
color: '#F97066',
borderColor: '#F97066'
}}
onClick={onOpenTrainingErrorModal}
>
{t('dataset:training_error_list')}
</Button>
)}
</HStack>
{/* diff collection button */}
......
projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingErrorEditView.tsx 0 → 100644
import { Box, Button, Flex } from '@chakra-ui/react';
import type { GetTrainingDataDetailResponse } from '@fastgpt/global/openapi/core/dataset/training/api';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import { useTranslation } from 'next-i18next';
import MyTextarea from '@/components/common/Textarea/MyTextarea';
import MyImage from '@/components/MyImage';
import { useForm } from 'react-hook-form';
const TrainingErrorEditView = ({
loading,
editChunk,
onCancel,
onSave
}: {
loading: boolean;
editChunk: GetTrainingDataDetailResponse;
onCancel: () => void;
onSave: (data: { q: string; a?: string; chunkIndex?: number }) => void;
}) => {
const { t } = useTranslation();
const { register, handleSubmit } = useForm({
defaultValues: {
q: editChunk?.q || '',
a: editChunk?.a || ''
}
});
return (
<Flex flexDirection={'column'} gap={4}>
{editChunk?.imagePreviewUrl && (
<Box>
<FormLabel>{t('file:image')}</FormLabel>
<Box w={'100%'} h={'200px'} border={'base'} borderRadius={'md'}>
<MyImage src={editChunk.imagePreviewUrl} alt="image" w={'100%'} h={'100%'} />
</Box>
</Box>
)}
<Box>
{(editChunk?.a || editChunk?.imagePreviewUrl) && (
<FormLabel>
{editChunk?.a
? t('common:dataset_data_input_chunk_content')
: t('common:dataset_data_input_q')}
</FormLabel>
)}
<MyTextarea
{...register('q', { required: true })}
minH={editChunk?.a || editChunk?.imagePreviewUrl ? 200 : 400}
/>
</Box>
{editChunk?.a && (
<Box>
<Box>{t('common:dataset_data_input_a')}</Box>
<MyTextarea {...register('a')} minH={200} />
</Box>
)}
<Flex justifyContent={'flex-end'} gap={4}>
<Button variant={'outline'} onClick={onCancel}>
{t('common:Cancel')}
</Button>
<Button isLoading={loading} variant={'primary'} onClick={handleSubmit(onSave)}>
{t('common:Confirm')}
</Button>
</Flex>
</Flex>
);
};
export default TrainingErrorEditView;
projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingErrorList.tsx 0 → 100644
import {
Box,
Button,
Flex,
Table,
TableContainer,
Tbody,
Td,
Th,
Thead,
Tr
} from '@chakra-ui/react';
import type { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import type {
GetDatasetTrainingErrorBody,
GetTrainingDataDetailResponse,
GetTrainingErrorBody,
TrainingErrorGroupType,
TrainingErrorItemType,
UpdateTrainingDataBody
} from '@fastgpt/global/openapi/core/dataset/training/api';
import type { Permission } from '@fastgpt/global/support/permission/controller';
import type { PaginationResponseType } from '@fastgpt/global/openapi/api';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyBox from '@fastgpt/web/components/common/MyBox';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { useTranslation } from 'next-i18next';
import { useMemoizedFn, useScroll, useThrottleEffect } from 'ahooks';
import { forwardRef, type MouseEvent, useEffect, useMemo, useRef, useState } from 'react';
import RawSourceBox from '@/components/core/dataset/RawSourceBox';
import {
deleteTrainingData,
getDatasetTrainingError,
getTrainingDataDetail,
getTrainingError,
updateTrainingData
} from '@/web/core/dataset/api/training';
import { formatTrainingStageText } from '@/web/core/dataset/trainingStatus';
import TrainingErrorEditView from './TrainingErrorEditView';
export type TrainingErrorScope =
| { type: 'collection'; collectionId: string }
| { type: 'dataset'; datasetId: string };
type TrainingErrorListItem = TrainingErrorItemType | TrainingErrorGroupType;
type TrainingErrorListParams = GetTrainingErrorBody | GetDatasetTrainingErrorBody;
type TrainingErrorListResponse = PaginationResponseType<TrainingErrorListItem>;
type TrainingErrorRequestParams =
| Omit<GetTrainingErrorBody, 'pageNum' | 'offset' | 'pageSize'>
| Omit<GetDatasetTrainingErrorBody, 'pageNum' | 'offset' | 'pageSize'>;
const datasetGroupItemPageSize = 5;
const ErrorActionButton = forwardRef<
HTMLButtonElement,
{
icon: string;
children: React.ReactNode;
onClick?: () => void;
isLoading?: boolean;
}
>(({ icon, children, onClick, isLoading }, ref) => (
<Button
ref={ref}
variant={'unstyled'}
color={'myGray.600'}
display={'inline-flex'}
alignItems={'center'}
justifyContent={'center'}
gap={'6px'}
px={2}
py={'6px'}
h={'28px'}
minW={0}
borderRadius={'6px'}
fontSize={'12px'}
lineHeight={'16px'}
fontWeight={500}
letterSpacing={'0.5px'}
bg={'transparent'}
_hover={{ bg: 'transparent', color: 'myGray.600' }}
_active={{ bg: 'transparent' }}
_disabled={{ opacity: 0.6, cursor: 'not-allowed' }}
isLoading={isLoading}
onClick={onClick}
>
<MyIcon name={icon as any} w={4} flexShrink={0} />
{children}
</Button>
));
ErrorActionButton.displayName = 'ErrorActionButton';
const ErrorMessage = ({ errorMsg }: { errorMsg?: string }) => {
const { t } = useTranslation();
const text = errorMsg ? t(errorMsg as any) : '-';
return (
<MyTooltip shouldWrapChildren={false} placement={'bottom-start'} offset={[0, 10]} label={text}>
<Box
className="textEllipsis"
display={'inline-block'}
maxW={'100%'}
verticalAlign={'middle'}
color={'myGray.600'}
fontSize={'12px'}
lineHeight={'16px'}
>
{text}
</Box>
</MyTooltip>
);
};
const TrainingStageText = ({ item }: { item: TrainingErrorItemType }) => {
const { t } = useTranslation();
const text = formatTrainingStageText(item.mode, t);
return (
<Box
className="textEllipsis"
color={'myGray.900'}
fontSize={'12px'}
lineHeight={'16px'}
whiteSpace={'nowrap'}
maxW={'165px'}
>
{text}
</Box>
);
};
const ActionButtons = ({
item,
isRetryLoading,
isEditLoading,
isDeleteLoading,
onRetry,
onEdit,
onDelete
}: {
item: TrainingErrorItemType;
isRetryLoading: boolean;
isEditLoading: boolean;
isDeleteLoading: boolean;
onRetry: (item: TrainingErrorItemType) => void;
onEdit: (item: TrainingErrorItemType) => void;
onDelete: (item: TrainingErrorItemType) => void;
}) => {
const { t } = useTranslation();
return (
<Flex alignItems={'center'} justifyContent={'flex-end'}>
<ErrorActionButton
icon={'common/confirm/restoreTip'}
isLoading={isRetryLoading}
onClick={() => onRetry(item)}
>
{t('dataset:dataset.ReTrain')}
</ErrorActionButton>
<Box w={'1px'} height={'16px'} bg={'#E8EBF0'} />
<ErrorActionButton icon={'edit'} isLoading={isEditLoading} onClick={() => onEdit(item)}>
{t('dataset:dataset.Edit_Chunk')}
</ErrorActionButton>
<Box w={'1px'} height={'16px'} bg={'#E8EBF0'} />
<ErrorActionButton icon={'delete'} isLoading={isDeleteLoading} onClick={() => onDelete(item)}>
{t('dataset:dataset.Delete_Chunk')}
</ErrorActionButton>
</Flex>
);
};
const TrainingErrorList = ({
scope,
permission,
onRefresh,
onClose,
showFooter = false
}: {
scope: TrainingErrorScope;
permission: Permission;
onRefresh?: () => void;
onClose?: () => void;
showFooter?: boolean;
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const [editChunk, setEditChunk] = useState<GetTrainingDataDetailResponse>();
const collectionScrollRef = useRef<HTMLDivElement>(null);
const collectionAutoFillOffsetRef = useRef<number>();
const datasetScrollRef = useRef<HTMLDivElement>(null);
const datasetAutoFillOffsetRef = useRef<number>();
const pendingDatasetScrollTopRef = useRef<number>();
const trainingErrorDataRef = useRef<TrainingErrorListItem[]>([]);
const getErrorData = (data: TrainingErrorListParams): Promise<TrainingErrorListResponse> => {
if (scope.type === 'collection') return getTrainingError(data as GetTrainingErrorBody);
return getDatasetTrainingError(data as GetDatasetTrainingErrorBody);
};
const requestParams = useMemo(() => {
if (scope.type === 'collection') return { collectionId: scope.collectionId };
return { datasetId: scope.datasetId };
}, [scope]);
const { data, setData, setTotal, ScrollData, isLoading, total, fetchData } = useScrollPagination<
TrainingErrorListParams,
TrainingErrorListResponse
>(getErrorData, {
pageSize: scope.type === 'collection' ? 15 : 10,
params: (scope.type === 'dataset'
? { ...requestParams, itemPageSize: datasetGroupItemPageSize }
: requestParams) as TrainingErrorRequestParams,
refreshDeps: [
scope.type,
scope.type === 'dataset' ? scope.datasetId : '',
scope.type === 'collection' ? scope.collectionId : ''
],
showNoMoreTip: scope.type !== 'dataset',
EmptyTip: <EmptyTip text={t('dataset:training_error_empty')} />
});
useEffect(() => {
trainingErrorDataRef.current = data;
}, [data]);
/**
* 单条异常被删除、重试或编辑后都会离开异常列表，这里只修补当前列表。
* dataset 视图保留每个 collection 已加载的 items 数量，避免“加载更多”后的内容缩回首屏数量。
*/
const removeHandledItemFromLocalData = useMemoizedFn(
({ collectionId, dataId }: { collectionId?: string; dataId: string }) => {
const targetCollectionId = collectionId ? String(collectionId) : undefined;
const targetDataId = String(dataId);
const currentData = trainingErrorDataRef.current;
if (scope.type === 'collection') {
const nextData = (currentData as TrainingErrorItemType[]).filter(
(item) => String(item._id) !== targetDataId
);
const removedCount = currentData.length - nextData.length;
if (removedCount === 0) return;
trainingErrorDataRef.current = nextData;
setData(nextData);
setTotal((prev) => Math.max(prev - removedCount, 0));
return;
}
let removedGroupCount = 0;
let removedItemCount = 0;
const nextData = (currentData as TrainingErrorGroupType[]).reduce<TrainingErrorGroupType[]>(
(groups, group) => {
if (
targetCollectionId !== undefined &&
String(group.collection.collectionId) !== targetCollectionId
) {
groups.push(group);
return groups;
}
const nextItems = group.items.filter((item) => String(item._id) !== targetDataId);
const currentRemovedItemCount = group.items.length - nextItems.length;
if (currentRemovedItemCount === 0) {
groups.push(group);
return groups;
}
removedItemCount += currentRemovedItemCount;
const nextErrorCount = Math.max(group.errorCount - currentRemovedItemCount, 0);
if (nextErrorCount === 0) {
removedGroupCount += 1;
return groups;
}
groups.push({
...group,
items: nextItems,
errorCount: nextErrorCount,
hasMoreItems: nextItems.length < nextErrorCount
});
return groups;
},
[]
);
if (removedItemCount === 0) return;
trainingErrorDataRef.current = nextData;
setData(nextData);
if (removedGroupCount > 0) {
setTotal((prev) => Math.max(prev - removedGroupCount, 0));
}
}
);
const clearLocalErrorData = useMemoizedFn(() => {
trainingErrorDataRef.current = [];
setData([]);
setTotal(0);
});
const { runAsync: getData, loading: getDataLoading } = useRequest(
(data: { collectionId: string; dataId: string }) => {
return getTrainingDataDetail(data);
},
{
manual: true,
onSuccess: (data) => {
setEditChunk(data);
}
}
);
const { runAsync: deleteData, loading: deleteLoading } = useRequest(
(data: { collectionId: string; dataId: string }) => {
return deleteTrainingData(data);
},
{
manual: true,
onSuccess: (_, [deletedItem]) => {
removeHandledItemFromLocalData(deletedItem);
onRefresh?.();
}
}
);
const { runAsync: updateData, loading: updateLoading } = useRequest(
(data: UpdateTrainingDataBody) => {
return updateTrainingData(data);
},
{
manual: true,
onSuccess: (_, [updatedData]) => {
if (updatedData.dataId) {
removeHandledItemFromLocalData({
collectionId: updatedData.collectionId,
dataId: updatedData.dataId
});
} else {
clearLocalErrorData();
}
onRefresh?.();
setEditChunk(undefined);
}
}
);
const { runAsync: loadMoreGroupItems, loading: loadMoreGroupItemsLoading } = useRequest(
(group: TrainingErrorGroupType) => {
if (scope.type !== 'dataset') return Promise.reject('scope error');
return getDatasetTrainingError({
datasetId: scope.datasetId,
collectionId: String(group.collection.collectionId),
offset: 0,
pageSize: 1,
itemOffset: group.items.length,
itemPageSize: datasetGroupItemPageSize
});
},
{
manual: true,
onSuccess: (res, [group]) => {
const nextGroup = res.list[0] as TrainingErrorGroupType | undefined;
if (!nextGroup || nextGroup.items.length === 0) {
pendingDatasetScrollTopRef.current = undefined;
return;
}
const targetCollectionId = String(group.collection.collectionId);
setData((prev) => {
const nextData = prev.map((item) => {
if (!('items' in item)) return item;
if (String(item.collection.collectionId) !== targetCollectionId) return item;
const loadedItemIds = new Set(item.items.map((chunk) => String(chunk._id)));
const appendItems = nextGroup.items.filter(
(chunk) => !loadedItemIds.has(String(chunk._id))
);
return {
...item,
items: [...item.items, ...appendItems],
errorCount: nextGroup.errorCount,
hasMoreItems: nextGroup.hasMoreItems
};
});
trainingErrorDataRef.current = nextData;
return nextData;
});
const scrollTop = pendingDatasetScrollTopRef.current;
pendingDatasetScrollTopRef.current = undefined;
if (scrollTop !== undefined) {
requestAnimationFrame(() => {
if (datasetScrollRef.current) {
datasetScrollRef.current.scrollTop = scrollTop;
}
});
}
},
onError: () => {
pendingDatasetScrollTopRef.current = undefined;
}
}
);
const getItemCollectionId = (item: TrainingErrorItemType) => String(item.collectionId);
const checkPermission = (hasPermission: boolean) => {
if (hasPermission) return true;
toast({
title: t('common:error_un_permission'),
status: 'warning'
});
return false;
};
const handleRetryItem = (item: TrainingErrorItemType) => {
if (!checkPermission(permission.hasWritePer)) return;
return updateData({
collectionId: getItemCollectionId(item),
dataId: String(item._id)
});
};
const handleEditItem = (item: TrainingErrorItemType) => {
if (!checkPermission(permission.hasWritePer)) return;
const collectionId = getItemCollectionId(item);
return getData({ collectionId, dataId: String(item._id) });
};
const handleDeleteItem = (item: TrainingErrorItemType) => {
if (!checkPermission(permission.hasManagePer)) return;
const collectionId = getItemCollectionId(item);
return deleteData({ collectionId, dataId: String(item._id) });
};
const handleRetryAll = async () => {
if (!checkPermission(permission.hasWritePer)) return;
if (scope.type === 'collection') {
await updateData({
collectionId: scope.collectionId
});
return;
}
await updateData({
datasetId: scope.datasetId
});
onClose?.();
};
const captureDatasetScrollTop = () => {
if (scope.type !== 'dataset') return;
pendingDatasetScrollTopRef.current = datasetScrollRef.current?.scrollTop;
};
const handleLoadMoreMouseDown = (e: MouseEvent<HTMLButtonElement>) => {
/*
Chakra Button 在 isLoading 时会 disabled，若按钮先获得焦点，Modal 的 FocusLock
会在失焦后把焦点恢复到弹窗内其它按钮并带动内部滚动。鼠标点击加载更多不需要抢焦点。
*/
e.preventDefault();
captureDatasetScrollTop();
};
const handleLoadMoreGroupItems = (group: TrainingErrorGroupType) => {
captureDatasetScrollTop();
loadMoreGroupItems(group);
};
const datasetGroups = useMemo(() => {
if (scope.type !== 'dataset') return [] as TrainingErrorGroupType[];
const groupMap = new Map<string, TrainingErrorGroupType>();
for (const group of data as TrainingErrorGroupType[]) {
const collectionId = String(group.collection.collectionId);
const existGroup = groupMap.get(collectionId);
if (existGroup) {
existGroup.items.push(...group.items);
} else {
groupMap.set(collectionId, {
...group,
items: [...group.items]
});
}
}
return Array.from(groupMap.values());
}, [data, scope.type]);
const collectionItems = data as TrainingErrorItemType[];
const listLoading = isLoading;
// 操作按钮保持独立 loading，不遮罩列表；但操作进行中锁住滚动分页，避免 offset 和后端变更并发错位。
const listRequestLocked = isLoading || updateLoading || getDataLoading || deleteLoading;
const pageNoMore = data.length >= total;
const collectionScroll = useScroll(collectionScrollRef);
const loadMoreCollectionItems = useMemoizedFn(() => {
if (scope.type !== 'collection' || listRequestLocked || pageNoMore) return;
fetchData({ init: false, ScrollContainerRef: collectionScrollRef });
});
const loadMoreDatasetGroups = useMemoizedFn(() => {
if (scope.type !== 'dataset' || listRequestLocked || pageNoMore) return;
fetchData({ init: false, ScrollContainerRef: datasetScrollRef });
});
useThrottleEffect(
() => {
if (scope.type !== 'collection' || listRequestLocked || pageNoMore) return;
const scrollContainer = collectionScrollRef.current;
if (!scrollContainer || collectionItems.length === 0) return;
const { scrollTop, scrollHeight, clientHeight } = scrollContainer;
if (scrollTop + clientHeight >= scrollHeight - 100) {
loadMoreCollectionItems();
}
},
[collectionScroll],
{ wait: 50 }
);
useEffect(() => {
if (scope.type !== 'collection') return;
if (collectionItems.length === 0) {
collectionAutoFillOffsetRef.current = undefined;
return;
}
if (listRequestLocked || pageNoMore) return;
const scrollContainer = collectionScrollRef.current;
if (!scrollContainer) return;
const { scrollHeight, clientHeight } = scrollContainer;
if (scrollHeight > clientHeight) return;
if (collectionAutoFillOffsetRef.current === collectionItems.length) return;
collectionAutoFillOffsetRef.current = collectionItems.length;
loadMoreCollectionItems();
}, [collectionItems.length, listRequestLocked, pageNoMore, loadMoreCollectionItems, scope.type]);
useEffect(() => {
if (scope.type !== 'dataset') return;
if (data.length === 0) {
datasetAutoFillOffsetRef.current = undefined;
return;
}
if (listRequestLocked || pageNoMore) return;
const scrollContainer = datasetScrollRef.current;
if (!scrollContainer) return;
const { scrollHeight, clientHeight } = scrollContainer;
if (scrollHeight > clientHeight) return;
if (datasetAutoFillOffsetRef.current === data.length) return;
datasetAutoFillOffsetRef.current = data.length;
loadMoreDatasetGroups();
}, [listRequestLocked, data.length, loadMoreDatasetGroups, pageNoMore, scope.type]);
if (editChunk) {
return (
<TrainingErrorEditView
loading={updateLoading}
editChunk={editChunk}
onCancel={() => setEditChunk(undefined)}
onSave={(formData) => {
if (!checkPermission(permission.hasWritePer)) return;
updateData({
collectionId: String(editChunk.collectionId),
dataId: String(editChunk._id),
...formData
});
}}
/>
);
}
return (
<>
{scope.type === 'collection' ? (
<MyBox ref={collectionScrollRef} h={'400px'} overflowY={'auto'} isLoading={listLoading}>
<TableContainer fontSize={'12px'}>
<Table variant={'simple'}>
<Thead>
<Tr>
<Th pr={0}>{t('dataset:dataset.Chunk_Number')}</Th>
<Th pr={0}>{t('dataset:dataset.Training_Status')}</Th>
<Th>{t('dataset:dataset.Error_Message')}</Th>
<Th w={'220px'}>{t('dataset:dataset.Operation')}</Th>
</Tr>
</Thead>
<Tbody>
{collectionItems.map((item) => (
<Tr key={String(item._id)}>
<Td>{item.chunkIndex + 1}</Td>
<Td>
<TrainingStageText item={item} />
</Td>
<Td maxW={50}>
<ErrorMessage errorMsg={item.errorMsg} />
</Td>
<Td w={'220px'} px={3}>
<ActionButtons
item={item}
isRetryLoading={updateLoading}
isEditLoading={getDataLoading}
isDeleteLoading={deleteLoading}
onRetry={handleRetryItem}
onEdit={handleEditItem}
onDelete={handleDeleteItem}
/>
</Td>
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
{total === 0 && !listLoading && <EmptyTip text={t('dataset:training_error_empty')} />}
</MyBox>
) : (
<ScrollData
maxH={'60vh'}
pr={1}
ScrollContainerRef={datasetScrollRef}
isLoading={listLoading}
>
<Flex flexDir={'column'} gap={4}>
{datasetGroups.map((group, index) => (
<Box key={String(group.collection.collectionId)}>
<Flex alignItems={'center'} gap={2} minW={0}>
<Box
color={'myGray.900'}
fontWeight={500}
fontSize={'12px'}
lineHeight={'16px'}
whiteSpace={'pre'}
>
{`${index + 1} `}
</Box>
<RawSourceBox
collectionId={group.collection.collectionId}
collectionType={group.collection.type as DatasetCollectionTypeEnum}
sourceName={group.collection.sourceName || group.collection.name}
sourceId={group.collection.sourceId}
canView={false}
alignItems={'center'}
fontSize={'12px'}
lineHeight={'16px'}
color={'myGray.900'}
minW={0}
/>
</Flex>
<Box pl={6} py={2.5}>
<Box bg={'#FBFBFC'} borderRadius={'8px'} px={4} py={4}>
<Flex flexDir={'column'} gap={2.5}>
{group.items.map((item) => (
<Flex key={String(item._id)} alignItems={'center'} h={'28px'} gap={'32px'}>
<Box w={'180px'} flexShrink={0} minW={0}>
<TrainingStageText item={item} />
</Box>
<Box flex={'1 1 auto'} minW={0}>
<ErrorMessage errorMsg={item.errorMsg} />
</Box>
<Box flexShrink={0}>
<ActionButtons
item={item}
isRetryLoading={updateLoading}
isEditLoading={getDataLoading}
isDeleteLoading={deleteLoading}
onRetry={handleRetryItem}
onEdit={handleEditItem}
onDelete={handleDeleteItem}
/>
</Box>
</Flex>
))}
</Flex>
{group.hasMoreItems && (
<Button
variant={'ghost'}
display={'flex'}
mx={'auto'}
mt={2.5}
h={'20px'}
minH={'20px'}
px={1.5}
minW={0}
fontSize={'11px'}
lineHeight={'14px'}
color={'myGray.400'}
_hover={{ bg: 'transparent', color: 'myGray.500' }}
_active={{ bg: 'transparent' }}
isLoading={loadMoreGroupItemsLoading}
onMouseDown={handleLoadMoreMouseDown}
onClick={() => handleLoadMoreGroupItems(group)}
>
{t('common:request_more')}
</Button>
)}
</Box>
</Box>
</Box>
))}
</Flex>
</ScrollData>
)}
{showFooter && (
<Flex justifyContent={'flex-end'} gap={3} mt={6}>
<Button variant={'whiteBase'} onClick={onClose}>
{t('common:Cancel')}
</Button>
{total > 0 && (
<Button
variant={'primary'}
color={'white'}
isLoading={updateLoading}
onClick={handleRetryAll}
>
{t('dataset:retry_all')}
</Button>
)}
</Flex>
)}
</>
);
};
export default TrainingErrorList;
projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingErrorModal.tsx 0 → 100644
import MyModal from '@fastgpt/web/components/v2/common/MyModal';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type';
import { useTranslation } from 'next-i18next';
import TrainingErrorList from './TrainingErrorList';
const TrainingErrorModal = ({
datasetId,
permission,
onClose,
onRefresh
}: {
datasetId: string;
permission: DatasetItemType['permission'];
onClose: () => void;
onRefresh?: () => void;
}) => {
const { t } = useTranslation();
return (
<MyModal
isOpen
onClose={onClose}
title={t('dataset:training_error_list')}
size={'lg'}
isCentered
borderRadius={'10px'}
sx={{
'.chakra-modal__close-btn': {
top: '8px',
right: '8px',
w: '36px',
h: '36px'
}
}}
>
<TrainingErrorList
scope={{ type: 'dataset', datasetId }}
permission={permission}
onClose={onClose}
onRefresh={onRefresh}
showFooter
/>
</MyModal>
);
};
export default TrainingErrorModal;
projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingStates.tsx
import {
Box,
Button,
Flex,
ModalBody,
Table,
TableContainer,
Tbody,
Td,
Th,
Thead,
Tr
} from '@chakra-ui/react';
import { Box, Flex, ModalBody } from '@chakra-ui/react';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useTranslation } from 'next-i18next';
import MyTag from '@fastgpt/web/components/common/Tag/index';
......@@ -18,26 +6,14 @@ import FillRowTabs from '@fastgpt/web/components/common/Tabs/FillRowTabs';
import { useMemo, useState } from 'react';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { getDatasetCollectionTrainingDetail } from '@/web/core/dataset/api/collection';
import {
deleteTrainingData,
getTrainingDataDetail,
getTrainingError,
updateTrainingData
} from '@/web/core/dataset/api/training';
import { DatasetCollectionDataProcessModeEnum } from '@fastgpt/global/core/dataset/constants';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { type GetTrainingDataDetailResponse } from '@fastgpt/global/openapi/core/dataset/training/api';
import MyTextarea from '@/components/common/Textarea/MyTextarea';
import { TrainingProcess } from '@/web/core/dataset/constants';
import { useForm } from 'react-hook-form';
import type { GetCollectionTrainingDetailResponseType } from '@fastgpt/global/openapi/core/dataset/collection/api';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import MyImage from '@/components/MyImage';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import type { Permission } from '@fastgpt/global/support/permission/controller';
import React from 'react';
import TrainingErrorList from './TrainingErrorList';
enum TrainingStatus {
NotStart = 'NotStart',
......@@ -57,6 +33,8 @@ const ProgressView = ({
const isQA = trainingDetail?.trainingType === DatasetCollectionDataProcessModeEnum.qa;
const isImageParse =
trainingDetail?.trainingType === DatasetCollectionDataProcessModeEnum.imageParse;
const isImageIndex = trainingDetail.advancedTraining.imageIndex;
const isAutoIndexes = trainingDetail.advancedTraining.autoIndexes;
/*
状态计算
......@@ -136,7 +114,7 @@ const ProgressView = ({
}
]
: []),
...(trainingDetail?.advancedTraining.imageIndex
...(isImageIndex
? [
{
errorCount: trainingDetail.errorCounts.image,
......@@ -148,7 +126,7 @@ const ProgressView = ({
}
]
: []),
...(trainingDetail?.advancedTraining.autoIndexes
...(isAutoIndexes
? [
{
errorCount: trainingDetail.errorCounts.auto,
......@@ -185,8 +163,8 @@ const ProgressView = ({
trainingDetail.queuedCounts,
trainingDetail.trainingCounts,
trainingDetail.errorCounts,
trainingDetail?.advancedTraining.imageIndex,
trainingDetail?.advancedTraining.autoIndexes,
isImageIndex,
isAutoIndexes,
trainingDetail.trainedCount,
t,
isImageParse,
......@@ -286,231 +264,14 @@ const ProgressView = ({
);
};
const ErrorView = ({
collectionId,
refreshTrainingDetail
}: {
collectionId: string;
refreshTrainingDetail: () => void;
}) => {
const { t } = useTranslation();
const TrainingText = {
[TrainingModeEnum.parse]: t('dataset:process.Parsing'),
[TrainingModeEnum.chunk]: t('dataset:process.Vectorizing'),
[TrainingModeEnum.qa]: t('dataset:process.Get QA'),
[TrainingModeEnum.imageParse]: t('dataset:process.Image_Index'),
[TrainingModeEnum.image]: t('dataset:process.Image_Index'),
[TrainingModeEnum.auto]: t('dataset:process.Auto_Index')
};
const [editChunk, setEditChunk] = useState<GetTrainingDataDetailResponse>();
const {
data: errorList,
ScrollData,
isLoading,
refreshList
} = useScrollPagination(getTrainingError, {
pageSize: 15,
params: {
collectionId
},
EmptyTip: <EmptyTip />
});
const { runAsync: getData, loading: getDataLoading } = useRequest(
(data: { collectionId: string; dataId: string }) => {
return getTrainingDataDetail(data);
},
{
manual: true,
onSuccess: (data) => {
setEditChunk(data);
}
}
);
const { runAsync: deleteData, loading: deleteLoading } = useRequest(
(data: { collectionId: string; dataId: string }) => {
return deleteTrainingData(data);
},
{
manual: true,
onSuccess: () => {
refreshList();
}
}
);
const { runAsync: updateData, loading: updateLoading } = useRequest(
(data: { collectionId: string; dataId: string; q?: string; a?: string }) => {
return updateTrainingData(data);
},
{
manual: true,
onSuccess: () => {
refreshList();
refreshTrainingDetail();
setEditChunk(undefined);
}
}
);
if (editChunk) {
return (
<EditView
loading={updateLoading}
editChunk={editChunk}
onCancel={() => setEditChunk(undefined)}
onSave={(data) => {
updateData({
collectionId,
dataId: editChunk._id,
...data
});
}}
/>
);
}
return (
<ScrollData
h={'400px'}
isLoading={isLoading || updateLoading || getDataLoading || deleteLoading}
>
<TableContainer overflowY={'auto'} fontSize={'12px'}>
<Table variant={'simple'}>
<Thead>
<Tr>
<Th pr={0}>{t('dataset:dataset.Chunk_Number')}</Th>
<Th pr={0}>{t('dataset:dataset.Training_Status')}</Th>
<Th>{t('dataset:dataset.Error_Message')}</Th>
<Th w={'220px'}>{t('dataset:dataset.Operation')}</Th>
</Tr>
</Thead>
<Tbody>
{errorList.map((item, index) => (
<Tr key={index}>
<Td>{item.chunkIndex + 1}</Td>
<Td>{TrainingText[item.mode]}</Td>
<Td maxW={50}>
<MyTooltip shouldWrapChildren={false} placement={'auto'} label={t(item.errorMsg)}>
{t(item.errorMsg)}
</MyTooltip>
</Td>
<Td w={'220px'} px={3}>
<Flex alignItems={'center'}>
<Button
variant={'ghost'}
size={'sm'}
color={'myGray.600'}
leftIcon={<MyIcon name={'common/confirm/restoreTip'} w={4} />}
fontSize={'mini'}
onClick={() => updateData({ collectionId, dataId: item._id })}
>
{t('dataset:dataset.ReTrain')}
</Button>
<Box w={'1px'} height={'16px'} bg={'myGray.200'} />
<Button
variant={'ghost'}
size={'sm'}
color={'myGray.600'}
leftIcon={<MyIcon name={'edit'} w={4} />}
fontSize={'mini'}
onClick={() => getData({ collectionId, dataId: item._id })}
>
{t('dataset:dataset.Edit_Chunk')}
</Button>
<Box w={'1px'} height={'16px'} bg={'myGray.200'} />
<Button
variant={'ghost'}
size={'sm'}
color={'myGray.600'}
leftIcon={<MyIcon name={'delete'} w={4} />}
fontSize={'mini'}
onClick={() => {
deleteData({ collectionId, dataId: item._id });
}}
>
{t('dataset:dataset.Delete_Chunk')}
</Button>
</Flex>
</Td>
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
</ScrollData>
);
};
const EditView = ({
loading,
editChunk,
onCancel,
onSave
}: {
loading: boolean;
editChunk: GetTrainingDataDetailResponse;
onCancel: () => void;
onSave: (data: { q: string; a?: string }) => void;
}) => {
const { t } = useTranslation();
const { register, handleSubmit } = useForm({
defaultValues: {
q: editChunk?.q || '',
a: editChunk?.a || ''
}
});
return (
<Flex flexDirection={'column'} gap={4}>
{editChunk?.imagePreviewUrl && (
<Box>
<FormLabel>{t('file:image')}</FormLabel>
<Box w={'100%'} h={'200px'} border={'base'} borderRadius={'md'}>
<MyImage src={editChunk.imagePreviewUrl} alt="image" w={'100%'} h={'100%'} />
</Box>
</Box>
)}
<Box>
{(editChunk?.a || editChunk?.imagePreviewUrl) && (
<FormLabel>
{editChunk?.a
? t('common:dataset_data_input_chunk_content')
: t('common:dataset_data_input_q')}
</FormLabel>
)}
<MyTextarea
{...register('q', { required: true })}
minH={editChunk?.a || editChunk?.imagePreviewUrl ? 200 : 400}
/>
</Box>
{editChunk?.a && (
<Box>
<Box>{t('common:dataset_data_input_a')}</Box>
<MyTextarea {...register('a')} minH={200} />
</Box>
)}
<Flex justifyContent={'flex-end'} gap={4}>
<Button variant={'outline'} onClick={onCancel}>
{t('common:Cancel')}
</Button>
<Button isLoading={loading} variant={'primary'} onClick={handleSubmit(onSave)}>
{t('common:Confirm')}
</Button>
</Flex>
</Flex>
);
};
const TrainingStates = ({
collectionId,
permission,
defaultTab = 'states',
onClose
}: {
collectionId: string;
permission: Permission;
defaultTab?: 'states' | 'errors';
onClose: () => void;
}) => {
......@@ -527,18 +288,6 @@ const TrainingStates = ({
manual: false
});
// All retry logic
const { runAsync: handleRetryAll, loading: retrying } = useRequest(
() => updateTrainingData({ collectionId }),
{
manual: true,
onSuccess: () => {
refreshTrainingDetail();
},
errorToast: t('dataset:retry_failed')
}
);
const errorCounts = Object.values(trainingDetail?.errorCounts || {}).reduce(
(acc, count) => acc + count,
0
......@@ -567,17 +316,15 @@ const TrainingStates = ({
}
]}
/>
{tab === 'errors' && errorCounts > 0 && (
<Button variant={'whiteBase'} size="sm" isLoading={retrying} onClick={handleRetryAll}>
{t('dataset:retry_all')}
</Button>
)}
</Flex>
{tab === 'states' && trainingDetail && <ProgressView trainingDetail={trainingDetail} />}
{tab === 'errors' && (
<ErrorView
collectionId={collectionId}
refreshTrainingDetail={refreshTrainingDetail}
<TrainingErrorList
scope={{ type: 'collection', collectionId }}
permission={permission}
onRefresh={refreshTrainingDetail}
onClose={onClose}
showFooter={errorCounts > 0}
/>
)}
</ModalBody>
......
projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx
......@@ -52,6 +52,13 @@ import TagsPopOver from './TagsPopOver';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import TrainingStates from './TrainingStates';
import { useTableMultipleSelect } from '@fastgpt/web/hooks/useTableMultipleSelect';
import {
getCollectionTrainingStatusColorSchema,
getCollectionTrainingStatusText
} from '@/web/core/dataset/trainingStatus';
import TrainingErrorModal from './TrainingErrorModal';
import type { DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api';
import { hasDatasetTrainingError as checkDatasetTrainingError } from '@/web/core/dataset/api/training';
const Header = dynamic(() => import('./Header'));
const EmptyCollectionTip = dynamic(() => import('./EmptyCollectionTip'));
......@@ -66,7 +73,10 @@ const CollectionCard = () => {
const [trainingStatesCollection, setTrainingStatesCollection] = useState<{
collectionId: string;
permission: DatasetCollectionsListItemType['permission'];
}>();
const [isTrainingErrorModalOpen, setIsTrainingErrorModalOpen] = useState(false);
const [hasDatasetTrainingError, setHasDatasetTrainingError] = useState(false);
const { collections, Pagination, total, getData, isGetting, pageNum, pageSize } =
useContextSelector(CollectionPageContext, (v) => v);
......@@ -76,34 +86,17 @@ const CollectionCard = () => {
() =>
collections.map((collection) => {
const icon = getCollectionIcon({ type: collection.type, name: collection.name });
const status = (() => {
if (collection.hasError) {
return {
statusText: t('common:core.dataset.collection.status.error'),
colorSchema: 'red'
};
}
if (collection.trainingAmount > 0) {
return {
statusText: t('common:dataset.collections.Collection Embedding', {
total: collection.trainingAmount
}),
colorSchema: 'gray'
};
}
return {
statusText: t('common:core.dataset.collection.status.active'),
colorSchema: 'green'
};
})();
const statusColorSchema = getCollectionTrainingStatusColorSchema(collection);
const statusText = getCollectionTrainingStatusText(collection);
return {
...collection,
icon,
...status
statusText,
statusColorSchema
};
}),
[collections, t]
[collections]
);
const {
......@@ -124,6 +117,22 @@ const CollectionCard = () => {
const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
title: t('common:Rename')
});
const { runAsync: refreshDatasetTrainingError } = useRequest(
async () => {
const res = await checkDatasetTrainingError(datasetDetail._id);
return res.hasError;
},
{
manual: false,
refreshDeps: [datasetDetail._id],
errorToast: '',
onSuccess(hasError) {
setHasDatasetTrainingError(hasError);
}
}
);
const { runAsync: onUpdateCollection, loading: isUpdating } = useRequest(
putDatasetCollectionById,
{
......@@ -147,6 +156,7 @@ const CollectionCard = () => {
{
onSuccess() {
getData(pageNum);
refreshDatasetTrainingError().catch(() => undefined);
},
successToast: t('common:delete_success'),
errorToast: t('common:delete_failed')
......@@ -174,12 +184,18 @@ const CollectionCard = () => {
useRequest(
async () => {
const shouldRefreshTrainingError =
hasTrainingData || datasetDetail.status !== DatasetStatusEnum.active;
if (datasetDetail.status !== DatasetStatusEnum.active) {
loadDatasetDetail(datasetDetail._id);
}
if (hasTrainingData) {
getData(pageNum);
}
if (shouldRefreshTrainingError) {
await refreshDatasetTrainingError().catch(() => undefined);
}
},
{
pollingInterval: 6000,
......@@ -198,7 +214,9 @@ const CollectionCard = () => {
parentId: targetId
});
getData(pageNum);
} catch {}
} catch {
// Drag failures are handled by the request layer toast; keep the list state unchanged here.
}
}
});
......@@ -208,7 +226,11 @@ const CollectionCard = () => {
<MyBox isLoading={isLoading} h={'100%'} py={[2, 4]} overflow={'hidden'}>
<Flex ref={BoxRef} flexDirection={'column'} py={[1, 0]} h={'100%'} px={[2, 6]}>
{/* header */}
<Header hasTrainingData={hasTrainingData} />
<Header
hasTrainingData={hasTrainingData}
hasTrainingError={hasDatasetTrainingError}
onOpenTrainingErrorModal={() => setIsTrainingErrorModalOpen(true)}
/>
{/* collection table */}
<TableContainer mt={3} overflowY={'auto'} fontSize={'sm'} flex={'1 0 0'} h={0}>
......@@ -297,9 +319,9 @@ const CollectionCard = () => {
<Td py={2}>
{collection.trainingType
? t(
(DatasetCollectionDataProcessModeMap[collection.trainingType]?.label ||
'-') as any
)
(DatasetCollectionDataProcessModeMap[collection.trainingType]?.label ||
'-') as any
)
: '-'}
</Td>
<Td py={2}>{collection.dataAmount || '-'}</Td>
......@@ -311,16 +333,21 @@ const CollectionCard = () => {
<MyTooltip label={t('common:Click_to_expand')}>
<MyTag
showDot
colorSchema={collection.colorSchema as any}
colorSchema={collection.statusColorSchema}
type={'fill'}
fontSize={'mini'}
letterSpacing={'0.5px'}
onClick={(e) => {
e.stopPropagation();
setTrainingStatesCollection({ collectionId: collection._id });
setTrainingStatesCollection({
collectionId: collection._id,
permission: collection.permission
});
}}
>
<Flex fontWeight={'medium'} alignItems={'center'} gap={1}>
{t(collection.statusText as any)}
<MyIcon name={'common/maximize'} w={'11px'} />
<MyIcon name={'common/maximize'} w={'10px'} h={'10px'} />
</Flex>
</MyTag>
</MyTooltip>
......@@ -371,25 +398,25 @@ const CollectionCard = () => {
children: [
...(collectionCanSync(collection.type)
? [
{
label: (
<Flex alignItems={'center'}>
<MyIcon
name={'common/refreshLight'}
w={'0.9rem'}
mr={2}
/>
{t('dataset:collection_sync')}
</Flex>
),
onClick: () =>
openSyncConfirm({
onConfirm: () => {
onclickStartSync(collection._id);
}
})()
}
]
{
label: (
<Flex alignItems={'center'}>
<MyIcon
name={'common/refreshLight'}
w={'0.9rem'}
mr={2}
/>
{t('dataset:collection_sync')}
</Flex>
),
onClick: () =>
openSyncConfirm({
onConfirm: () => {
onclickStartSync(collection._id);
}
})()
}
]
: []),
{
label: (
......@@ -441,8 +468,8 @@ const CollectionCard = () => {
customContent:
collection.type === DatasetCollectionTypeEnum.folder
? t(
'common:dataset.collections.Confirm to delete the folder'
)
'common:dataset.collections.Confirm to delete the folder'
)
: t('common:dataset.Confirm to delete the file')
})()
}
......@@ -497,10 +524,23 @@ const CollectionCard = () => {
{!!trainingStatesCollection && (
<TrainingStates
collectionId={trainingStatesCollection.collectionId}
permission={trainingStatesCollection.permission}
onClose={() => setTrainingStatesCollection(undefined)}
/>
)}
{isTrainingErrorModalOpen && (
<TrainingErrorModal
datasetId={datasetDetail._id}
permission={datasetDetail.permission}
onClose={() => setIsTrainingErrorModalOpen(false)}
onRefresh={() => {
getData(pageNum);
refreshDatasetTrainingError().catch(() => undefined);
}}
/>
)}
{!!moveCollectionData && (
<SelectCollections
datasetId={datasetDetail._id}
......@@ -526,4 +566,4 @@ const CollectionCard = () => {
);
};
export default React.memo(CollectionCard);
export default React.memo(CollectionCard);
\ No newline at end of file
projects/app/src/pageComponents/dataset/detail/DataCard.tsx
......@@ -36,6 +36,10 @@ import { formatFileSize } from '@fastgpt/global/common/file/tools';
import MyImage from '@fastgpt/web/components/common/Image/MyImage';
import dynamic from 'next/dynamic';
import { downloadFetch } from '@/web/common/system/utils';
import {
getCollectionTrainingStatusColorSchema,
getCollectionTrainingStatusText
} from '@/web/core/dataset/trainingStatus';
const InsertImagesModal = dynamic(() => import('./data/InsertImageModal'), {
ssr: false
......@@ -101,6 +105,13 @@ const DataCard = () => {
);
const canWrite = useMemo(() => datasetDetail.permission.hasWritePer, [datasetDetail]);
const collectionTrainingStatus = useMemo(() => {
if (!collection) return;
return {
text: getCollectionTrainingStatusText(collection),
colorSchema: getCollectionTrainingStatusColorSchema(collection)
};
}, [collection]);
const [
isInsertImagesModalOpen,
......@@ -240,21 +251,19 @@ const DataCard = () => {
indexAmount: collection?.indexAmount ?? '-'
})}
</Box>
{!!collection?.errorCount && (
{!!collectionTrainingStatus && (
<MyTag
colorSchema={'red'}
type={'fill'}
cursor={'pointer'}
rounded={'full'}
ml={2}
colorSchema={collectionTrainingStatus.colorSchema}
onClick={() => {
setErrorModalId(collection._id);
setErrorModalId(collection?._id || '');
}}
>
<Flex fontWeight={'medium'} alignItems={'center'} gap={1}>
{t('dataset:data_error_amount', {
errorAmount: collection?.errorCount
})}
{t(collectionTrainingStatus.text as any)}
<MyIcon name={'common/maximize'} w={'11px'} />
</Flex>
</MyTag>
......@@ -462,10 +471,11 @@ const DataCard = () => {
}}
/>
)}
{errorModalId && (
{errorModalId && collection && (
<TrainingStates
defaultTab={'errors'}
defaultTab={collection?.hasError ? 'errors' : 'states'}
collectionId={errorModalId}
permission={collection.permission}
onClose={() => {
setErrorModalId('');
refreshList();
......
projects/app/src/pages/api/core/dataset/collection/detail.ts
......@@ -9,6 +9,7 @@ import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collecti
import { getVectorCount } from '@fastgpt/service/common/vectorDB/controller';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { Types } from '@fastgpt/service/common/mongo';
import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset';
import { isS3ObjectKey } from '@fastgpt/service/common/s3/utils';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
......@@ -18,6 +19,124 @@ import {
GetCollectionDetailQuerySchema,
GetCollectionDetailResponseSchema
} from '@fastgpt/global/openapi/core/dataset/collection/api';
import {
activeTrainingExpr,
finalErrorTrainingExpr,
getSlowestTrainingStatus,
remainingTrainingMatch,
trainingModeRanks
} from '@fastgpt/service/core/dataset/training/query';
import { CollectionTrainingStatusEnum } from '@fastgpt/global/core/dataset/constants';
const defaultCollectionTrainingStatus = {
trainingAmount: 0,
activeTrainingAmount: 0,
finalErrorAmount: 0,
hasError: false,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
/**
* 获取数据集集合的训练状态统计信息
* @param teamId - 团队ID
* @param datasetId - 数据集ID
* @param collectionId - 集合ID
* @returns 包含训练数量、活跃训练数、错误数及最慢训练状态的统计对象
*/
const getCollectionTrainingStatus = async ({
teamId,
datasetId,
collectionId
}: {
teamId: Types.ObjectId;
datasetId: Types.ObjectId;
collectionId: Types.ObjectId;
}) => {
const [trainingStatus] = await MongoDatasetTraining.aggregate(
[
{
$match: {
teamId,
datasetId,
collectionId,
...remainingTrainingMatch
}
},
{
$addFields: {
modeRank: {
$switch: {
branches: trainingModeRanks.map(({ mode, rank }) => ({
case: { $eq: ['$mode', mode] },
then: rank
})),
default: 999
}
},
isActiveTraining: activeTrainingExpr,
isFinalErrorTraining: finalErrorTrainingExpr
}
},
{
$group: {
_id: '$mode',
modeRank: { $first: '$modeRank' },
activeCount: { $sum: { $cond: ['$isActiveTraining', 1, 0] } },
finalErrorCount: { $sum: { $cond: ['$isFinalErrorTraining', 1, 0] } },
trainingAmount: { $sum: 1 }
}
},
{
$sort: {
modeRank: 1
}
},
{
$group: {
_id: null,
trainingAmount: { $sum: '$trainingAmount' },
activeTrainingAmount: { $sum: '$activeCount' },
finalErrorAmount: { $sum: '$finalErrorCount' },
modeCounts: {
$push: {
mode: '$_id',
activeCount: '$activeCount',
finalErrorCount: '$finalErrorCount'
}
}
}
}
],
readFromSecondary
);
if (!trainingStatus) return defaultCollectionTrainingStatus;
const { slowestTrainingMode, slowestTrainingStatus } = getSlowestTrainingStatus(
Object.fromEntries(
trainingStatus.modeCounts.map(
({
mode,
activeCount,
finalErrorCount
}: {
mode: any;
activeCount: number;
finalErrorCount: number;
}) => [mode, { activeCount, finalErrorCount }]
)
)
);
return {
trainingAmount: trainingStatus.trainingAmount,
activeTrainingAmount: trainingStatus.activeTrainingAmount,
finalErrorAmount: trainingStatus.finalErrorAmount,
hasError: trainingStatus.finalErrorAmount > 0,
slowestTrainingMode,
slowestTrainingStatus
};
};
async function handler(req: ApiRequestProps): Promise<GetCollectionDetailResponseType> {
const { id } = parseApiInput({ req, querySchema: GetCollectionDetailQuerySchema }).query;
......@@ -36,23 +155,18 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionDetailRespons
return Promise.reject('Invalid dataset file key');
}
const [file, indexAmount, errorCount] = await Promise.all([
const [file, indexAmount, trainingStatus] = await Promise.all([
fileId ? getS3DatasetSource().getFileMetadata(fileId) : undefined,
getVectorCount({
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id
}),
MongoDatasetTraining.countDocuments(
{
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: id,
errorMsg: { $exists: true },
retryCount: { $lte: 0 }
},
readFromSecondary
)
getCollectionTrainingStatus({
teamId: new Types.ObjectId(collection.teamId),
datasetId: new Types.ObjectId(collection.datasetId),
collectionId: new Types.ObjectId(collection._id)
})
]);
return GetCollectionDetailResponseSchema.parse({
......@@ -65,7 +179,8 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionDetailRespons
}),
permission,
file,
errorCount
...trainingStatus,
errorCount: trainingStatus.finalErrorAmount
});
}
......
projects/app/src/pages/api/core/dataset/collection/listV2.ts
......@@ -13,11 +13,64 @@ import { replaceRegChars } from '@fastgpt/global/common/string/tools';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import {
activeTrainingExpr,
finalErrorTrainingExpr,
getSlowestTrainingStatus,
remainingTrainingMatch,
trainingModeRanks
} from '@fastgpt/service/core/dataset/training/query';
import {
CollectionTrainingStatusEnum,
type TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import {
ListCollectionV2BodySchema,
ListCollectionV2ResponseSchema,
type ListCollectionV2ResponseType
} from '@fastgpt/global/openapi/core/dataset/collection/api';
const defaultCollectionTrainingStatus = {
trainingAmount: 0,
activeTrainingAmount: 0,
finalErrorAmount: 0,
hasError: false,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
type TrainingAmountAggregateItem = {
_id: string;
trainingAmount: number;
activeTrainingAmount: number;
finalErrorAmount: number;
modeCounts: {
mode: TrainingModeEnum;
activeCount: number;
finalErrorCount: number;
}[];
};
const formatTrainingStatus = (item?: TrainingAmountAggregateItem) => {
if (!item) return defaultCollectionTrainingStatus;
const { slowestTrainingMode, slowestTrainingStatus } = getSlowestTrainingStatus(
Object.fromEntries(
item.modeCounts.map(({ mode, activeCount, finalErrorCount }) => [
mode,
{ activeCount, finalErrorCount }
])
)
);
return {
trainingAmount: item.trainingAmount,
activeTrainingAmount: item.activeTrainingAmount,
finalErrorAmount: item.finalErrorAmount,
hasError: item.finalErrorAmount > 0,
slowestTrainingMode,
slowestTrainingStatus
};
};
async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseType> {
const {
datasetId,
......@@ -96,8 +149,7 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy
tags: item.tags
}),
dataAmount: 0,
trainingAmount: 0,
hasError: false,
...defaultCollectionTrainingStatus,
permission
}))
),
......@@ -118,7 +170,7 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy
// Compute data amount
const [trainingAmount, dataAmount]: [
{ _id: string; count: number; hasError: boolean }[],
TrainingAmountAggregateItem[],
{ _id: string; count: number }[]
] = await Promise.all([
MongoDatasetTraining.aggregate(
......@@ -127,14 +179,75 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy
$match: {
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(datasetId),
collectionId: { $in: collectionIds }
collectionId: { $in: collectionIds },
...remainingTrainingMatch
}
},
{
$addFields: {
modeRank: {
$switch: {
branches: trainingModeRanks.map(({ mode, rank }) => ({
case: { $eq: ['$mode', mode] },
then: rank
})),
default: 999
}
},
isActiveTraining: activeTrainingExpr,
isFinalErrorTraining: finalErrorTrainingExpr
}
},
{
$group: {
_id: '$collectionId',
count: { $sum: 1 },
hasError: { $max: { $cond: [{ $ifNull: ['$errorMsg', false] }, true, false] } }
trainingAmount: { $sum: 1 },
activeTrainingAmount: { $sum: { $cond: ['$isActiveTraining', 1, 0] } },
finalErrorAmount: { $sum: { $cond: ['$isFinalErrorTraining', 1, 0] } },
modeCounts: {
$push: {
mode: '$mode',
modeRank: '$modeRank',
activeCount: { $cond: ['$isActiveTraining', 1, 0] },
finalErrorCount: { $cond: ['$isFinalErrorTraining', 1, 0] }
}
}
}
},
{ $unwind: '$modeCounts' },
{
$group: {
_id: {
collectionId: '$_id',
mode: '$modeCounts.mode',
modeRank: '$modeCounts.modeRank'
},
trainingAmount: { $first: '$trainingAmount' },
activeTrainingAmount: { $first: '$activeTrainingAmount' },
finalErrorAmount: { $first: '$finalErrorAmount' },
activeCount: { $sum: '$modeCounts.activeCount' },
finalErrorCount: { $sum: '$modeCounts.finalErrorCount' }
}
},
{
$sort: {
'_id.collectionId': 1,
'_id.modeRank': 1
}
},
{
$group: {
_id: '$_id.collectionId',
trainingAmount: { $first: '$trainingAmount' },
activeTrainingAmount: { $first: '$activeTrainingAmount' },
finalErrorAmount: { $first: '$finalErrorAmount' },
modeCounts: {
$push: {
mode: '$_id.mode',
activeCount: '$activeCount',
finalErrorCount: '$finalErrorCount'
}
}
}
}
],
......@@ -171,10 +284,10 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy
datasetId,
tags: item.tags
}),
trainingAmount:
trainingAmount.find((amount) => String(amount._id) === String(item._id))?.count || 0,
dataAmount: dataAmount.find((amount) => String(amount._id) === String(item._id))?.count || 0,
hasError: trainingAmount.find((amount) => String(amount._id) === String(item._id))?.hasError,
...formatTrainingStatus(
trainingAmount.find((amount) => String(amount._id) === String(item._id))
),
permission
}))
);
......
projects/app/src/pages/api/core/dataset/collection/scrollList.ts
/**
* @deprecated Use /core/dataset/collection/listV2 instead.
*/
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { NextAPI } from '@/service/middleware/entry';
......@@ -5,7 +8,10 @@ import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/tra
import { Types } from '@fastgpt/service/common/mongo';
import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import {
CollectionTrainingStatusEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { type PaginationResponse } from '@fastgpt/global/openapi/api';
import type { DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api';
......@@ -14,6 +20,13 @@ import { replaceRegChars } from '@fastgpt/global/common/string/tools';
import { ScrollCollectionsBodySchema } from '@fastgpt/global/openapi/core/dataset/collection/api';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
const defaultCollectionTrainingStatus = {
activeTrainingAmount: 0,
finalErrorAmount: 0,
hasError: false,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
async function handler(
req: ApiRequestProps
): Promise<PaginationResponse<DatasetCollectionsListItemType>> {
......@@ -80,6 +93,7 @@ async function handler(
...item,
dataAmount: 0,
trainingAmount: 0,
...defaultCollectionTrainingStatus,
indexAmount: 0,
permission
}))
......@@ -150,7 +164,11 @@ async function handler(
},
trainingAmount: {
$ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0]
}
},
activeTrainingAmount: { $literal: 0 },
finalErrorAmount: { $literal: 0 },
hasError: { $literal: false },
slowestTrainingStatus: { $literal: CollectionTrainingStatusEnum.ready }
}
}
]),
......
projects/app/src/pages/api/core/dataset/collection/trainingDetail.ts
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import {
DatasetCollectionDataProcessModeEnum,
type TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { type TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
......@@ -15,6 +12,11 @@ import {
GetCollectionTrainingDetailResponseSchema,
type GetCollectionTrainingDetailResponseType
} from '@fastgpt/global/openapi/core/dataset/collection/api';
import {
BLOCKED_LOCK_TIME,
activeTrainingMatch,
finalErrorTrainingMatch
} from '@fastgpt/service/core/dataset/training/query';
const defaultCounts: Record<TrainingModeEnum, number> = {
parse: 0,
......@@ -57,7 +59,7 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai
$match: {
_id: { $lt: new Types.ObjectId(minId) },
retryCount: { $gt: 0 },
lockTime: { $lt: new Date('2050/1/1') }
lockTime: { $lt: BLOCKED_LOCK_TIME }
}
},
{
......@@ -72,8 +74,7 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai
{
$match: {
...match,
retryCount: { $gt: 0 },
lockTime: { $lt: new Date('2050/1/1') }
...activeTrainingMatch
}
},
{
......@@ -87,8 +88,7 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai
{
$match: {
...match,
// retryCount: { $lte: 0 },
errorMsg: { $exists: true }
...finalErrorTrainingMatch
}
},
{
......
projects/app/src/pages/api/core/dataset/training/deleteTrainingData.ts
......@@ -35,3 +35,6 @@ async function handler(req: ApiRequestProps): Promise<DeleteTrainingDataResponse
}
export default NextAPI(handler);
export type deleteTrainingDataBody =
import('@fastgpt/global/openapi/core/dataset/training/api').DeleteTrainingDataBody;
export type deleteTrainingDataResponse = DeleteTrainingDataResponse;
projects/app/src/pages/api/core/dataset/training/getDatasetTrainingError.ts 0 → 100644
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
import { Types, type PipelineStage } from '@fastgpt/service/common/mongo';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import {
DatasetTrainingErrorPaginationLimits,
GetDatasetTrainingErrorBodySchema,
GetDatasetTrainingErrorResponseSchema,
type GetDatasetTrainingErrorBody,
type GetDatasetTrainingErrorResponse,
type TrainingErrorGroupType,
type TrainingErrorItemType
} from '@fastgpt/global/openapi/core/dataset/training/api';
import {
finalErrorTrainingMatch,
trainingModeRanks
} from '@fastgpt/service/core/dataset/training/query';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
async function handler(req: ApiRequestProps): Promise<GetDatasetTrainingErrorResponse> {
const {
datasetId,
collectionId,
itemOffset = 0,
itemPageSize = DatasetTrainingErrorPaginationLimits.defaultItemPageSize
} = parseApiInput({
req,
bodySchema: GetDatasetTrainingErrorBodySchema
}).body;
const { offset, pageSize: rawPageSize } = parsePaginationRequest(req);
const pageSize = Math.min(rawPageSize, DatasetTrainingErrorPaginationLimits.maxPageSize);
const itemLimit = Math.min(itemPageSize, DatasetTrainingErrorPaginationLimits.maxItemPageSize);
const itemSkip = Math.min(itemOffset, DatasetTrainingErrorPaginationLimits.maxItemOffset);
const { teamId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: ReadPermissionVal
});
const baseTrainingMatch = {
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(dataset._id),
...finalErrorTrainingMatch
};
/**
* 复用集合内异常 chunk 查询流水线，确保首屏和“加载更多”的排序完全一致。
*/
const buildItemPipeline = ({
match,
skip,
limit
}: {
match: Record<string, any>;
skip: number;
limit: number;
}): PipelineStage[] => [
{ $match: match },
{
$addFields: {
modeRank: {
$switch: {
branches: trainingModeRanks.map(({ mode, rank }) => ({
case: { $eq: ['$mode', mode] },
then: rank
})),
default: 999
}
}
}
},
{ $sort: { modeRank: 1, chunkIndex: 1, _id: 1 } },
{ $skip: skip },
{ $limit: limit },
{ $project: { modeRank: 0 } }
];
if (collectionId) {
const collection = await MongoDatasetCollection.findOne(
{
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(dataset._id),
_id: new Types.ObjectId(collectionId),
type: { $ne: DatasetCollectionTypeEnum.folder }
},
undefined,
readFromSecondary
).lean();
if (!collection) {
return GetDatasetTrainingErrorResponseSchema.parse({
total: 0,
list: []
});
}
const collectionMatch = {
...baseTrainingMatch,
collectionId: new Types.ObjectId(collection._id)
};
const [items, errorCount] = await Promise.all([
MongoDatasetTraining.aggregate(
buildItemPipeline({
match: collectionMatch,
skip: itemSkip,
limit: itemLimit
}),
readFromSecondary
),
MongoDatasetTraining.countDocuments(collectionMatch, { ...readFromSecondary })
]);
const { sourceName, sourceId } = getCollectionSourceData(collection);
return GetDatasetTrainingErrorResponseSchema.parse({
total: errorCount > 0 ? 1 : 0,
list:
errorCount > 0
? [
{
collection: {
_id: collection._id,
collectionId: collection._id,
name: collection.name,
type: collection.type,
sourceName,
sourceId
},
items,
errorCount,
hasMoreItems: itemSkip + items.length < errorCount
}
]
: []
});
}
const [collectionStats, totalResult] = await Promise.all([
MongoDatasetTraining.aggregate(
[
{ $match: baseTrainingMatch },
{
$group: {
_id: '$collectionId',
errorCount: { $sum: 1 }
}
},
{ $sort: { _id: 1 } },
{ $skip: offset },
{ $limit: pageSize }
],
readFromSecondary
),
MongoDatasetTraining.aggregate(
[
{ $match: baseTrainingMatch },
{
$group: {
_id: '$collectionId'
}
},
{ $count: 'total' }
],
readFromSecondary
)
]);
const total = totalResult[0]?.total ?? 0;
const collectionIds = collectionStats.map((item: { _id: any }) => item._id);
const collections = collectionIds.length
? await MongoDatasetCollection.find(
{
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(dataset._id),
_id: { $in: collectionIds },
type: { $ne: DatasetCollectionTypeEnum.folder }
},
undefined,
readFromSecondary
).lean()
: [];
const collectionMap = new Map(
collections.map((collection) => [String(collection._id), collection])
);
const statsMap = new Map(
collectionStats.map((item: { _id: any; errorCount: number }) => [
String(item._id),
item.errorCount
])
);
const itemsList = collectionIds.length
? await Promise.all(
collectionIds.map((id: any) =>
MongoDatasetTraining.aggregate(
buildItemPipeline({
match: {
...baseTrainingMatch,
collectionId: id
},
skip: 0,
limit: itemLimit
}),
readFromSecondary
)
)
)
: [];
const list = collectionIds.reduce<TrainingErrorGroupType[]>((groups, id: any, index: number) => {
const collection = collectionMap.get(String(id));
const errorCount = statsMap.get(String(id)) ?? 0;
const items = (itemsList[index] ?? []) as TrainingErrorItemType[];
if (!collection || errorCount === 0 || items.length === 0) return groups;
const { sourceName, sourceId } = getCollectionSourceData(collection);
groups.push({
collection: {
_id: collection._id,
collectionId: collection._id,
name: collection.name,
type: collection.type,
sourceName,
sourceId
},
items,
errorCount,
hasMoreItems: items.length < errorCount
});
return groups;
}, []);
return GetDatasetTrainingErrorResponseSchema.parse({
total,
list
});
}
export default NextAPI(handler);
export type getDatasetTrainingErrorBody = GetDatasetTrainingErrorBody;
export type getDatasetTrainingErrorResponse = GetDatasetTrainingErrorResponse;
projects/app/src/pages/api/core/dataset/training/getTrainingDataDetail.ts
......@@ -41,6 +41,7 @@ async function handler(req: ApiRequestProps): Promise<GetTrainingDataDetailRespo
return GetTrainingDataDetailResponseSchema.parse({
_id: data._id,
datasetId: data.datasetId,
collectionId: data.collectionId,
mode: data.mode,
imagePreviewUrl:
data.imageId && isS3ObjectKey(data.imageId, 'dataset')
......
projects/app/src/pages/api/core/dataset/training/getTrainingError.ts
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
import { Types, type PipelineStage } from '@fastgpt/service/common/mongo';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
......@@ -9,11 +10,19 @@ import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import {
GetTrainingErrorBodySchema,
GetTrainingErrorResponseSchema,
type GetTrainingErrorBody,
type GetTrainingErrorResponse
} from '@fastgpt/global/openapi/core/dataset/training/api';
import {
finalErrorTrainingMatch,
trainingModeRanks
} from '@fastgpt/service/core/dataset/training/query';
async function handler(req: ApiRequestProps): Promise<GetTrainingErrorResponse> {
const { collectionId } = parseApiInput({ req, bodySchema: GetTrainingErrorBodySchema }).body;
const { collectionId } = parseApiInput({
req,
bodySchema: GetTrainingErrorBodySchema
}).body;
const { offset, pageSize } = parsePaginationRequest(req);
const { collection } = await authDatasetCollection({
......@@ -25,26 +34,43 @@ async function handler(req: ApiRequestProps): Promise<GetTrainingErrorResponse>
});
const match = {
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id,
errorMsg: { $exists: true }
teamId: new Types.ObjectId(collection.teamId),
datasetId: new Types.ObjectId(collection.datasetId),
collectionId: new Types.ObjectId(collection._id),
...finalErrorTrainingMatch
};
const pipeline: PipelineStage[] = [
{ $match: match },
{
$addFields: {
modeRank: {
$switch: {
branches: trainingModeRanks.map(({ mode, rank }) => ({
case: { $eq: ['$mode', mode] },
then: rank
})),
default: 999
}
}
}
},
{ $sort: { modeRank: 1, chunkIndex: 1, _id: 1 } },
{ $skip: offset },
{ $limit: pageSize },
{ $project: { modeRank: 0 } }
];
const [errorList, total] = await Promise.all([
MongoDatasetTraining.find(match, undefined, {
...readFromSecondary
})
.skip(offset)
.limit(pageSize)
.lean(),
MongoDatasetTraining.aggregate(pipeline, readFromSecondary),
MongoDatasetTraining.countDocuments(match, { ...readFromSecondary })
]);
return GetTrainingErrorResponseSchema.parse({
list: errorList,
total
total,
list: errorList
});
}
export default NextAPI(handler);
export type getTrainingErrorBody = GetTrainingErrorBody;
export type getTrainingErrorResponse = GetTrainingErrorResponse;
projects/app/src/pages/api/core/dataset/training/hasDatasetTrainingError.ts 0 → 100644
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { Types } from '@fastgpt/service/common/mongo';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { finalErrorTrainingMatch } from '@fastgpt/service/core/dataset/training/query';
import {
HasDatasetTrainingErrorQuerySchema,
HasDatasetTrainingErrorResponseSchema,
type HasDatasetTrainingErrorQuery,
type HasDatasetTrainingErrorResponse
} from '@fastgpt/global/openapi/core/dataset/training/api';
async function handler(req: ApiRequestProps): Promise<HasDatasetTrainingErrorResponse> {
const { datasetId } = parseApiInput({
req,
querySchema: HasDatasetTrainingErrorQuerySchema
}).query;
const { teamId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: ReadPermissionVal
});
const errorRecord = await MongoDatasetTraining.findOne(
{
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(dataset._id),
...finalErrorTrainingMatch
},
{ _id: 1 },
readFromSecondary
).lean();
return HasDatasetTrainingErrorResponseSchema.parse({
hasError: !!errorRecord
});
}
export default NextAPI(handler);
export type hasDatasetTrainingErrorQuery = HasDatasetTrainingErrorQuery;
export type hasDatasetTrainingErrorResponse = HasDatasetTrainingErrorResponse;
projects/app/src/pages/api/core/dataset/training/updateTrainingData.ts
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import {
authDataset,
authDatasetCollection
} from '@fastgpt/service/support/permission/dataset/auth';
import { NextAPI } from '@/service/middleware/entry';
import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
......@@ -10,33 +13,48 @@ import {
type UpdateTrainingDataResponse
} from '@fastgpt/global/openapi/core/dataset/training/api';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { finalErrorTrainingMatch } from '@fastgpt/service/core/dataset/training/query';
async function handler(req: ApiRequestProps): Promise<UpdateTrainingDataResponse> {
const { collectionId, dataId, q, a, chunkIndex } = parseApiInput({
req,
bodySchema: UpdateTrainingDataBodySchema
}).body;
const body = parseApiInput({ req, bodySchema: UpdateTrainingDataBodySchema }).body;
const { collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: WritePermissionVal
});
// 不传 dataId 时是批量重试：collectionId 和 datasetId 分别限定不同的重试范围。
if (!body.dataId) {
const retryMatch = await (async () => {
if (body.collectionId) {
const { collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId: body.collectionId,
per: WritePermissionVal
});
const trainingMatch = {
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id
};
return {
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id
};
}
const { teamId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId: body.datasetId!,
per: WritePermissionVal
});
return {
teamId,
datasetId: dataset._id
};
})();
// If dataId is not passed, all error data in this collection will be retried.
if (!dataId) {
await MongoDatasetTraining.updateMany(
{
...trainingMatch,
errorMsg: { $exists: true, $ne: null }
...retryMatch,
...finalErrorTrainingMatch
},
{
$unset: { errorMsg: '' },
......@@ -47,45 +65,57 @@ async function handler(req: ApiRequestProps): Promise<UpdateTrainingDataResponse
return UpdateTrainingDataResponseSchema.parse(undefined);
}
// Single data retry logic
const data = await MongoDatasetTraining.findOne({ ...trainingMatch, _id: dataId });
const { q, a, chunkIndex } = body;
// 单条重试只信任 dataId 找到的训练记录，再用记录所属 collection 做权限校验。
const data = await MongoDatasetTraining.findById(body.dataId);
if (!data) {
return Promise.reject('data not found');
}
const { collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId: data.collectionId,
per: WritePermissionVal
});
if (
String(collection.teamId) !== String(data.teamId) ||
String(collection.datasetId) !== String(data.datasetId) ||
String(collection._id) !== String(data.collectionId)
) {
return Promise.reject('data not found');
}
const trainingMatch = {
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id,
_id: data._id
};
// Add to chunk
if (data.imageId && q) {
await MongoDatasetTraining.updateOne(
{
...trainingMatch,
_id: dataId
},
{
$unset: { errorMsg: '' },
retryCount: 3,
mode: TrainingModeEnum.chunk,
...(q !== undefined && { q }),
...(a !== undefined && { a }),
...(chunkIndex !== undefined && { chunkIndex }),
lockTime: new Date('2000')
}
);
await MongoDatasetTraining.updateOne(trainingMatch, {
$unset: { errorMsg: '' },
retryCount: 3,
mode: TrainingModeEnum.chunk,
...(q !== undefined && { q }),
...(a !== undefined && { a }),
...(chunkIndex !== undefined && { chunkIndex }),
lockTime: new Date('2000')
});
} else {
await MongoDatasetTraining.updateOne(
{
...trainingMatch,
_id: dataId
},
{
$unset: { errorMsg: '' },
retryCount: 3,
...(q !== undefined && { q }),
...(a !== undefined && { a }),
...(chunkIndex !== undefined && { chunkIndex }),
lockTime: new Date('2000')
}
);
await MongoDatasetTraining.updateOne(trainingMatch, {
$unset: { errorMsg: '' },
retryCount: 3,
...(q !== undefined && { q }),
...(a !== undefined && { a }),
...(chunkIndex !== undefined && { chunkIndex }),
lockTime: new Date('2000')
});
}
return UpdateTrainingDataResponseSchema.parse(undefined);
......
projects/app/src/service/core/dataset/queues/datasetParse.ts
......@@ -167,7 +167,7 @@ export const datasetParseQueue = async (): Promise<any> => {
continue;
}
// Check team points and lock(No mistakes will be thrown here)
if (!(await checkTeamAiPointsAndLock(data.teamId))) {
if (!(await checkTeamAiPointsAndLock(data.teamId, String(data._id)))) {
continue;
}
......
projects/app/src/service/core/dataset/queues/generateQA.ts
......@@ -112,7 +112,7 @@ export async function generateQA(): Promise<any> {
continue;
}
// auth balance
if (!(await checkTeamAiPointsAndLock(data.teamId))) {
if (!(await checkTeamAiPointsAndLock(data.teamId, String(data._id)))) {
continue;
}
......
projects/app/src/service/core/dataset/queues/generateVector.ts
......@@ -160,7 +160,7 @@ export async function generateVector(): Promise<any> {
}
// auth balance
if (!(await checkTeamAiPointsAndLock(data.teamId))) {
if (!(await checkTeamAiPointsAndLock(data.teamId, String(data._id)))) {
continue;
}
......
projects/app/src/service/core/dataset/queues/utils.ts
......@@ -6,7 +6,7 @@ import { InformLevelEnum } from '@fastgpt/global/support/user/inform/constants';
import { getLogger, LogCategories } from '@fastgpt/service/common/logger';
const logger = getLogger(LogCategories.MODULE.DATASET.QUEUES);
export const checkTeamAiPointsAndLock = async (teamId: string) => {
export const checkTeamAiPointsAndLock = async (teamId: string, currentTrainingId?: string) => {
try {
await checkTeamAIPoints(teamId);
return true;
......@@ -21,7 +21,7 @@ export const checkTeamAiPointsAndLock = async (teamId: string) => {
teamId
});
logger.info('余额不足，暂停知识库处理');
await lockTrainingDataByTeamId(teamId);
await lockTrainingDataByTeamId(teamId, currentTrainingId);
} catch (error) {}
}
return false;
......
projects/app/src/web/core/dataset/api/training.ts
......@@ -7,7 +7,10 @@ import type {
GetTrainingDataDetailBody,
GetTrainingDataDetailResponse,
GetTrainingErrorBody,
GetTrainingErrorResponse
GetTrainingErrorResponse,
GetDatasetTrainingErrorBody,
GetDatasetTrainingErrorResponse,
HasDatasetTrainingErrorResponse
} from '@fastgpt/global/openapi/core/dataset/training/api';
export const postRebuildEmbedding = (data: RebuildEmbeddingBody) =>
......@@ -29,3 +32,11 @@ export const getTrainingDataDetail = (data: GetTrainingDataDetailBody) =>
export const getTrainingError = (data: GetTrainingErrorBody) =>
POST<GetTrainingErrorResponse>(`/core/dataset/training/getTrainingError`, data);
export const getDatasetTrainingError = (data: GetDatasetTrainingErrorBody) =>
POST<GetDatasetTrainingErrorResponse>(`/core/dataset/training/getDatasetTrainingError`, data);
export const hasDatasetTrainingError = (datasetId: string) =>
GET<HasDatasetTrainingErrorResponse>(`/core/dataset/training/hasDatasetTrainingError`, {
datasetId
});
projects/app/src/web/core/dataset/constants.ts
import { defaultQAModels, defaultVectorModels } from '@fastgpt/global/core/ai/constants';
import {
CollectionTrainingStatusEnum,
DatasetCollectionDataProcessModeEnum,
DatasetCollectionTypeEnum,
DatasetTypeEnum,
......@@ -60,7 +61,12 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
chunkSize: 0,
indexSize: 512,
permission: new DatasetPermission(),
indexAmount: 0
indexAmount: 0,
trainingAmount: 0,
activeTrainingAmount: 0,
finalErrorAmount: 0,
hasError: false,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
export const TrainingProcess = {
......
projects/app/src/web/core/dataset/trainingStatus.ts 0 → 100644
import {
CollectionTrainingStatusEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { i18nT } from '@fastgpt/global/common/i18n/utils';
import type { TFunction } from 'next-i18next';
import type { ColorSchemaType } from '@fastgpt/web/components/common/Tag';
/**
* 根据训练模式获取对应的阶段文本描述
* @param mode - 训练模式枚举，若未提供则返回“等待中”
* @returns 国际化后的阶段文本字符串
*/
export const getTrainingStageText = (mode?: TrainingModeEnum) => {
const textMap = {
[TrainingModeEnum.parse]: i18nT('dataset:process.Parsing'),
[TrainingModeEnum.imageParse]: i18nT('dataset:process.Parse_Image'),
[TrainingModeEnum.qa]: i18nT('dataset:process.Get QA'),
[TrainingModeEnum.image]: i18nT('dataset:process.Image_Index'),
[TrainingModeEnum.auto]: i18nT('dataset:process.Auto_Index'),
[TrainingModeEnum.chunk]: i18nT('dataset:process.Vectorizing')
};
return mode ? textMap[mode] : i18nT('dataset:process.Waiting');
};
/**
* 根据集合中最慢的训练任务和状态，获取整体训练状态文本
* @param params.slowestTrainingMode - 最慢任务的训练模式
* @param params.slowestTrainingStatus - 最慢任务的训练状态
* @returns 国际化后的状态文本（错误/进行中具体阶段/等待中/已就绪）
*/
export const getCollectionTrainingStatusText = ({
slowestTrainingMode,
slowestTrainingStatus
}: {
slowestTrainingMode?: TrainingModeEnum;
slowestTrainingStatus?: CollectionTrainingStatusEnum;
}) => {
if (slowestTrainingStatus === CollectionTrainingStatusEnum.error) {
return i18nT('dataset:training.status_error');
}
if (slowestTrainingStatus === CollectionTrainingStatusEnum.running && slowestTrainingMode) {
return getTrainingStageText(slowestTrainingMode);
}
if (slowestTrainingStatus === CollectionTrainingStatusEnum.running) {
return i18nT('dataset:process.Waiting');
}
return i18nT('dataset:process.Is_Ready');
};
/**
* 使用传入的翻译函数格式化训练阶段文本
* @param mode - 训练模式枚举
* @param t - i18next 翻译函数实例
* @returns 国际化后的阶段文本字符串
*/
export const formatTrainingStageText = (mode: TrainingModeEnum | undefined, t: TFunction) => {
return t(getTrainingStageText(mode) as any);
};
/**
* 根据集合中最慢的训练任务和状态，获取状态标签的颜色主题
* @param params.slowestTrainingMode - 最慢任务的训练模式
* @param params.slowestTrainingStatus - 最慢任务的训练状态
* @returns MyTag 可消费的颜色主题名
*/
export const getCollectionTrainingStatusColorSchema = ({
slowestTrainingMode,
slowestTrainingStatus
}: {
slowestTrainingMode?: TrainingModeEnum;
slowestTrainingStatus?: CollectionTrainingStatusEnum;
}): ColorSchemaType => {
if (slowestTrainingStatus === CollectionTrainingStatusEnum.error) {
return 'lightRed';
}
if (slowestTrainingStatus !== CollectionTrainingStatusEnum.running) {
return 'green';
}
switch (slowestTrainingMode) {
case TrainingModeEnum.parse:
case TrainingModeEnum.imageParse:
return 'blue';
case TrainingModeEnum.qa:
case TrainingModeEnum.image:
case TrainingModeEnum.auto:
return 'cyan';
case TrainingModeEnum.chunk:
return 'adora';
default:
return 'lightGray';
}
};
projects/app/test/api/core/dataset/collection/trainingStatus.test.ts 0 → 100644
import listHandler from '@/pages/api/core/dataset/collection/listV2';
import scrollListHandler from '@/pages/api/core/dataset/collection/scrollList';
import detailHandler from '@/pages/api/core/dataset/collection/detail';
import trainingDetailHandler from '@/pages/api/core/dataset/collection/trainingDetail';
import {
CollectionTrainingStatusEnum,
DatasetCollectionTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { DatasetCollectionsListItemSchema } from '@fastgpt/global/openapi/core/dataset/collection/api';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { getRootUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest';
describe('collection training status api', () => {
it('should expose unified active/final error/slowest status in list and detail', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 3
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'final error'
}
]);
const listRes = await Call(listHandler, {
auth: root,
body: {
datasetId: dataset._id,
pageSize: 10,
offset: 0,
filterTags: []
}
});
expect(listRes.code).toBe(200);
expect(listRes.data.list[0]).toMatchObject({
trainingAmount: 2,
activeTrainingAmount: 1,
finalErrorAmount: 1,
hasError: true,
slowestTrainingMode: TrainingModeEnum.parse,
slowestTrainingStatus: CollectionTrainingStatusEnum.running
});
const detailRes = await Call(detailHandler, {
auth: root,
query: {
id: collection._id
}
});
expect(detailRes.code).toBe(200);
expect(detailRes.data).toMatchObject({
trainingAmount: 2,
activeTrainingAmount: 1,
finalErrorAmount: 1,
errorCount: 1,
hasError: true,
slowestTrainingMode: TrainingModeEnum.parse,
slowestTrainingStatus: CollectionTrainingStatusEnum.running
});
});
it('should use active counts for progress and final errors for error tab', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 3,
errorMsg: 'temporary failed'
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'final failed'
}
]);
const res = await Call(trainingDetailHandler, {
auth: root,
query: {
collectionId: collection._id
}
});
expect(res.code).toBe(200);
expect(res.data.trainingCounts.qa).toBe(1);
expect(res.data.errorCounts.qa).toBe(0);
expect(res.data.errorCounts.chunk).toBe(1);
});
it('should keep deprecated scrollList compatible with the collection list item schema', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'final failed'
});
const res = await Call(scrollListHandler, {
auth: root,
body: {
datasetId: dataset._id,
pageSize: 10,
offset: 0,
filterTags: []
}
});
expect(res.code).toBe(200);
expect(() => DatasetCollectionsListItemSchema.parse(res.data.list[0])).not.toThrow();
expect(res.data.list[0]).toMatchObject({
trainingAmount: 1,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
});
});
});
projects/app/test/api/core/dataset/training/deleteTrainingData.test.ts
......@@ -39,7 +39,11 @@ describe('delete training data test', () => {
mode: TrainingModeEnum.chunk
});
const res = await Call<deleteTrainingDataBody, {}, deleteTrainingDataResponse>(handler, {
const res = await Call<
deleteTrainingDataBody,
Record<string, never>,
deleteTrainingDataResponse
>(handler, {
auth: root,
body: {
collectionId: collection._id,
......@@ -100,7 +104,11 @@ describe('delete training data test', () => {
mode: TrainingModeEnum.chunk
});
const res = await Call<deleteTrainingDataBody, {}, deleteTrainingDataResponse>(handler, {
const res = await Call<
deleteTrainingDataBody,
Record<string, never>,
deleteTrainingDataResponse
>(handler, {
auth: root,
body: {
datasetId: foreignDataset._id,
......@@ -114,4 +122,4 @@ describe('delete training data test', () => {
expect(res.code).toBe(200);
expect(existingTrainingData).toBeTruthy();
});
});
});
\ No newline at end of file
projects/app/test/api/core/dataset/training/getDatasetTrainingError.test.ts 0 → 100644
import handler, {
type getDatasetTrainingErrorBody,
type getDatasetTrainingErrorResponse
} from '@/pages/api/core/dataset/training/getDatasetTrainingError';
import {
DatasetCollectionTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { DatasetTrainingErrorPaginationLimits } from '@fastgpt/global/openapi/core/dataset/training/api';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { ApiRequestInputParseError } from '@fastgpt/service/common/zod/requestParseError';
import { getRootUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest';
type EmptyQuery = Record<string, never>;
describe('dataset training error list test', () => {
it('should paginate error collections and load more chunks inside one collection', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const [fileCollection, linkCollection] = await MongoDatasetCollection.create([
{
name: 'file',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
updateTime: new Date('2025-01-01')
},
{
name: 'link',
type: DatasetCollectionTypeEnum.link,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
rawLink: 'https://example.com',
updateTime: new Date('2025-01-02')
}
]);
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: fileCollection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 3,
errorMsg: 'temporary failed',
chunkIndex: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: fileCollection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'file error',
chunkIndex: 2
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: fileCollection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 0,
errorMsg: 'file parse error',
chunkIndex: 1
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: fileCollection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: ' ',
chunkIndex: 3
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: linkCollection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 0,
errorMsg: 'link error',
chunkIndex: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: linkCollection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 3,
errorMsg: 'link chunk error',
chunkIndex: 1
}
]);
const firstPageRes = await Call<
getDatasetTrainingErrorBody,
EmptyQuery,
getDatasetTrainingErrorResponse
>(handler, {
auth: root,
body: {
datasetId: dataset._id,
pageSize: 1,
offset: 0,
itemPageSize: 1
}
});
expect(firstPageRes.code).toBe(200);
expect(firstPageRes.data.total).toBe(2);
expect(firstPageRes.data.list).toHaveLength(1);
expect(firstPageRes.data.list[0].collection.name).toBe('file');
expect(firstPageRes.data.list[0].errorCount).toBe(2);
expect(firstPageRes.data.list[0].hasMoreItems).toBe(true);
expect(firstPageRes.data.list[0].items).toHaveLength(1);
expect(firstPageRes.data.list[0].items[0].mode).toBe(TrainingModeEnum.parse);
const secondPageRes = await Call<
getDatasetTrainingErrorBody,
EmptyQuery,
getDatasetTrainingErrorResponse
>(handler, {
auth: root,
body: {
datasetId: dataset._id,
pageSize: 1,
offset: 1,
itemPageSize: 1
}
});
expect(secondPageRes.code).toBe(200);
expect(secondPageRes.data.total).toBe(2);
expect(secondPageRes.data.list).toHaveLength(1);
expect(secondPageRes.data.list[0].collection.name).toBe('link');
expect(secondPageRes.data.list[0].errorCount).toBe(1);
expect(secondPageRes.data.list[0].hasMoreItems).toBe(false);
expect(secondPageRes.data.list[0].items[0].mode).toBe(TrainingModeEnum.qa);
const loadMoreRes = await Call<
getDatasetTrainingErrorBody,
EmptyQuery,
getDatasetTrainingErrorResponse
>(handler, {
auth: root,
body: {
datasetId: dataset._id,
collectionId: fileCollection._id,
pageSize: 1,
offset: 0,
itemOffset: 1,
itemPageSize: 1
}
});
expect(loadMoreRes.code).toBe(200);
expect(loadMoreRes.data.total).toBe(1);
expect(loadMoreRes.data.list).toHaveLength(1);
expect(loadMoreRes.data.list[0].collection.name).toBe('file');
expect(loadMoreRes.data.list[0].errorCount).toBe(2);
expect(loadMoreRes.data.list[0].hasMoreItems).toBe(false);
expect(loadMoreRes.data.list[0].items).toHaveLength(1);
expect(loadMoreRes.data.list[0].items[0].mode).toBe(TrainingModeEnum.chunk);
});
it.each([
{
name: 'invalid itemOffset',
body: {
datasetId: '507f1f77bcf86cd799439011',
pageSize: 1,
offset: 0,
itemOffset: 'not-a-number',
itemPageSize: 1
}
},
{
name: 'oversized itemPageSize',
body: {
datasetId: '507f1f77bcf86cd799439011',
pageSize: 1,
offset: 0,
itemPageSize: DatasetTrainingErrorPaginationLimits.maxItemPageSize + 1
}
},
{
name: 'invalid pageSize',
body: {
datasetId: '507f1f77bcf86cd799439011',
pageSize: 'not-a-number',
offset: 0,
itemPageSize: 1
}
},
{
name: 'oversized pageSize',
body: {
datasetId: '507f1f77bcf86cd799439011',
pageSize: DatasetTrainingErrorPaginationLimits.maxPageSize + 1,
offset: 0,
itemPageSize: 1
}
}
])('should reject $name before querying Mongo', async ({ body }) => {
const res = await Call<any, EmptyQuery, getDatasetTrainingErrorResponse>(handler, {
body
});
expect(res.code).not.toBe(200);
expect(res.error).toBeInstanceOf(ApiRequestInputParseError);
});
});
projects/app/test/api/core/dataset/training/getTrainingDataDetail.test.ts
import handler, {
type getTrainingDataDetailBody,
type getTrainingDataDetailResponse
} from '@/pages/api/core/dataset/training/getTrainingDataDetail';
import handler from '@/pages/api/core/dataset/training/getTrainingDataDetail';
import {
DatasetCollectionTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import type {
GetTrainingDataDetailBody,
GetTrainingDataDetailResponse
} from '@fastgpt/global/openapi/core/dataset/training/api';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
......@@ -41,7 +42,11 @@ describe('get training data detail test', () => {
a: 'test'
});
const res = await Call<getTrainingDataDetailBody, {}, getTrainingDataDetailResponse>(handler, {
const res = await Call<
GetTrainingDataDetailBody,
Record<string, never>,
GetTrainingDataDetailResponse
>(handler, {
auth: root,
body: {
collectionId: collection._id,
......@@ -103,7 +108,11 @@ describe('get training data detail test', () => {
a: 'foreign'
});
const res = await Call<getTrainingDataDetailBody, {}, getTrainingDataDetailResponse>(handler, {
const res = await Call<
GetTrainingDataDetailBody,
Record<string, never>,
GetTrainingDataDetailResponse
>(handler, {
auth: root,
body: {
datasetId: foreignDataset._id,
......
projects/app/test/api/core/dataset/training/getTrainingError.test.ts
......@@ -13,8 +13,10 @@ import { getRootUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest';
type EmptyQuery = Record<string, never>;
describe('training error list test', () => {
it('should return training error list', async () => {
it('should return final error list in collection scope', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
......@@ -30,19 +32,63 @@ describe('training error list test', () => {
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create(
[...Array(10).keys()].map((i) => ({
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'chunk first in insert order',
chunkIndex: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 0,
errorMsg: 'parse should sort before chunk',
chunkIndex: 9
},
...[...Array(10).keys()].map((i) => ({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
errorMsg: 'test'
}))
);
retryCount: 0,
errorMsg: 'test',
chunkIndex: i
})),
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 3,
errorMsg: 'temporary failed'
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 0,
errorMsg: ' '
}
]);
const res = await Call<getTrainingErrorBody, {}, getTrainingErrorResponse>(handler, {
const res = await Call<getTrainingErrorBody, EmptyQuery, getTrainingErrorResponse>(handler, {
auth: root,
body: {
collectionId: collection._id,
......@@ -52,7 +98,25 @@ describe('training error list test', () => {
});
expect(res.code).toBe(200);
expect(res.data.total).toBe(10);
expect(res.data.total).toBe(12);
expect(res.data.list.length).toBe(10);
expect(
res.data.list.map((item) => ('mode' in item ? item.mode : undefined)).slice(0, 2)
).toEqual([TrainingModeEnum.parse, TrainingModeEnum.chunk]);
expect(res.data.list.every((item) => 'mode' in item)).toBe(true);
});
it('should reject dataset scope request', async () => {
const root = await getRootUser();
const res = await Call<getTrainingErrorBody, EmptyQuery, getTrainingErrorResponse>(handler, {
auth: root,
body: {
datasetId: '507f1f77bcf86cd799439012',
pageSize: 10,
offset: 0
} as any
});
expect(res.code).not.toBe(200);
});
});
projects/app/test/api/core/dataset/training/hasDatasetTrainingError.test.ts 0 → 100644
import handler, {
type hasDatasetTrainingErrorQuery,
type hasDatasetTrainingErrorResponse
} from '@/pages/api/core/dataset/training/hasDatasetTrainingError';
import {
DatasetCollectionTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { getRootUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest';
type EmptyBody = Record<string, never>;
describe('dataset training error existence test', () => {
it('should only report final or blocked training errors', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 3,
errorMsg: 'temporary failed',
chunkIndex: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 0,
errorMsg: ' ',
chunkIndex: 1
}
]);
const noFinalErrorRes = await Call<
EmptyBody,
hasDatasetTrainingErrorQuery,
hasDatasetTrainingErrorResponse
>(handler, {
auth: root,
query: {
datasetId: dataset._id
}
});
expect(noFinalErrorRes.code).toBe(200);
expect(noFinalErrorRes.data.hasError).toBe(false);
await MongoDatasetTraining.create({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 0,
errorMsg: 'parse failed',
chunkIndex: 2
});
const finalErrorRes = await Call<
EmptyBody,
hasDatasetTrainingErrorQuery,
hasDatasetTrainingErrorResponse
>(handler, {
auth: root,
query: {
datasetId: dataset._id
}
});
expect(finalErrorRes.code).toBe(200);
expect(finalErrorRes.data.hasError).toBe(true);
});
});
projects/app/test/api/core/dataset/training/updateTrainingData.test.ts
import handler, {
type updateTrainingDataBody,
type updateTrainingDataResponse
} from '@/pages/api/core/dataset/training/updateTrainingData';
import handler from '@/pages/api/core/dataset/training/updateTrainingData';
import {
DatasetCollectionTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import type {
UpdateTrainingDataBody,
UpdateTrainingDataResponse
} from '@fastgpt/global/openapi/core/dataset/training/api';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { getRootUser } from '@test/datas/users';
import { getRootUser, getUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest';
type EmptyQuery = Record<string, never>;
describe('update training data test', () => {
it('should update training data', async () => {
const root = await getRootUser();
......@@ -39,16 +42,18 @@ describe('update training data test', () => {
mode: TrainingModeEnum.chunk
});
const res = await Call<updateTrainingDataBody, {}, updateTrainingDataResponse>(handler, {
auth: root,
body: {
collectionId: collection._id,
dataId: trainingData._id,
q: 'test',
a: 'test',
chunkIndex: 1
const res = await Call<UpdateTrainingDataBody, EmptyQuery, UpdateTrainingDataResponse>(
handler,
{
auth: root,
body: {
dataId: trainingData._id,
q: 'test',
a: 'test',
chunkIndex: 1
}
}
});
);
const updatedTrainingData = await MongoDatasetTraining.findOne({
teamId: root.teamId,
......@@ -62,43 +67,255 @@ describe('update training data test', () => {
expect(updatedTrainingData?.chunkIndex).toBe(1);
});
it('should ignore legacy datasetId and only update data from the authorized collection', async () => {
it('should retry single training data by dataId', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
const trainingData = await MongoDatasetTraining.create({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
errorMsg: 'failed',
retryCount: 0
});
const res = await Call<UpdateTrainingDataBody, EmptyQuery, UpdateTrainingDataResponse>(
handler,
{
auth: root,
body: {
dataId: trainingData._id,
q: 'retry'
}
}
);
const updatedTrainingData = await MongoDatasetTraining.findById(trainingData._id).lean();
expect(res.code).toBe(200);
expect(updatedTrainingData?.q).toBe('retry');
expect(updatedTrainingData?.errorMsg).toBeUndefined();
expect(updatedTrainingData?.retryCount).toBe(3);
});
it('should retry only final errors in collection scope', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
const [finalError, activeRetry] = await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'final error'
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 3,
errorMsg: 'temporary error'
}
]);
const res = await Call<UpdateTrainingDataBody, EmptyQuery, UpdateTrainingDataResponse>(
handler,
{
auth: root,
body: {
collectionId: collection._id
}
}
);
const updatedFinalError = await MongoDatasetTraining.findById(finalError._id).lean();
const updatedActiveRetry = await MongoDatasetTraining.findById(activeRetry._id).lean();
expect(res.code).toBe(200);
expect(updatedFinalError?.errorMsg).toBeUndefined();
expect(updatedFinalError?.retryCount).toBe(3);
expect(updatedActiveRetry?.errorMsg).toBe('temporary error');
});
it('should retry only final errors in dataset scope', async () => {
const root = await getRootUser();
const [dataset, foreignDataset] = await Promise.all([
MongoDataset.create({
const [dataset, foreignDataset] = await MongoDataset.create([
{
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
}),
MongoDataset.create({
},
{
name: 'foreign',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
})
}
]);
const [collection, foreignCollection] = await Promise.all([
MongoDatasetCollection.create({
const [collection, anotherCollection, foreignCollection] = await MongoDatasetCollection.create([
{
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
}),
MongoDatasetCollection.create({
},
{
name: 'another',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
},
{
name: 'foreign',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: foreignDataset._id
})
}
]);
const [datasetFinalError, datasetBlockedError, datasetActiveRetry, foreignFinalError] =
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'final error'
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: anotherCollection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 1,
lockTime: new Date('2999'),
errorMsg: 'blocked error'
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 3,
errorMsg: 'temporary error'
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: foreignDataset._id,
collectionId: foreignCollection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'foreign final error'
}
]);
const res = await Call<UpdateTrainingDataBody, EmptyQuery, UpdateTrainingDataResponse>(
handler,
{
auth: root,
body: {
datasetId: dataset._id
}
}
);
const [updatedFinalError, updatedBlockedError, updatedActiveRetry, updatedForeignFinalError] =
await Promise.all([
MongoDatasetTraining.findById(datasetFinalError._id).lean(),
MongoDatasetTraining.findById(datasetBlockedError._id).lean(),
MongoDatasetTraining.findById(datasetActiveRetry._id).lean(),
MongoDatasetTraining.findById(foreignFinalError._id).lean()
]);
expect(res.code).toBe(200);
expect(updatedFinalError?.errorMsg).toBeUndefined();
expect(updatedFinalError?.retryCount).toBe(3);
expect(updatedBlockedError?.errorMsg).toBeUndefined();
expect(updatedBlockedError?.retryCount).toBe(3);
expect(updatedBlockedError?.lockTime?.getTime()).toBe(new Date('2000').getTime());
expect(updatedActiveRetry?.errorMsg).toBe('temporary error');
expect(updatedForeignFinalError?.errorMsg).toBe('foreign final error');
});
it('should not let request datasetId bypass item collection authorization', async () => {
const [user, foreignUser] = await Promise.all([
getUser('normal-user'),
getUser('foreign-user')
]);
const [dataset, foreignDataset] = await MongoDataset.create([
{
name: 'test',
teamId: user.teamId,
tmbId: user.tmbId,
vectorModel: 'test',
agentModel: 'test'
},
{
name: 'foreign',
teamId: foreignUser.teamId,
tmbId: foreignUser.tmbId,
vectorModel: 'test',
agentModel: 'test'
}
]);
const foreignCollection = await MongoDatasetCollection.create({
name: 'foreign',
type: DatasetCollectionTypeEnum.file,
teamId: foreignUser.teamId,
tmbId: foreignUser.tmbId,
datasetId: foreignDataset._id
});
const foreignTrainingData = await MongoDatasetTraining.create({
teamId: root.teamId,
tmbId: root.tmbId,
teamId: foreignUser.teamId,
tmbId: foreignUser.tmbId,
datasetId: foreignDataset._id,
collectionId: foreignCollection._id,
billId: 'test',
......@@ -107,16 +324,18 @@ describe('update training data test', () => {
a: 'origin'
});
const res = await Call<updateTrainingDataBody, {}, updateTrainingDataResponse>(handler, {
auth: root,
body: {
datasetId: foreignDataset._id,
collectionId: collection._id,
dataId: foreignTrainingData._id,
q: 'changed',
a: 'changed'
} as any
});
const res = await Call<UpdateTrainingDataBody, EmptyQuery, UpdateTrainingDataResponse>(
handler,
{
auth: user,
body: {
datasetId: dataset._id,
dataId: foreignTrainingData._id,
q: 'changed',
a: 'changed'
} as any
}
);
const existingTrainingData = await MongoDatasetTraining.findById(foreignTrainingData._id);
......
projects/app/test/pages/api/core/dataset/training/updateTrainingData.test.ts
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { describe, expect, it, vi, beforeEach } from 'vitest';
import { handler } from '@/pages/api/core/dataset/training/updateTrainingData';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import {
authDataset,
authDatasetCollection
} from '@fastgpt/service/support/permission/dataset/auth';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
const datasetId = '507f1f77bcf86cd799439011';
......@@ -11,13 +14,14 @@ const foreignDatasetId = '507f1f77bcf86cd799439014';
vi.mock('@fastgpt/service/core/dataset/training/schema', () => ({
MongoDatasetTraining: {
findOne: vi.fn(),
findById: vi.fn(),
updateOne: vi.fn(),
updateMany: vi.fn()
}
}));
vi.mock('@fastgpt/service/support/permission/dataset/auth', () => ({
authDataset: vi.fn(),
authDatasetCollection: vi.fn()
}));
......@@ -30,25 +34,34 @@ describe('updateTrainingData', () => {
teamId: 'team1',
datasetId
}
});
} as any);
vi.mocked(authDataset).mockResolvedValue({
teamId: 'team1',
dataset: {
_id: datasetId
}
} as any);
});
it('should retry all error data when dataId is not provided', async () => {
const req = {
it('should retry only final errors in collection scope', async () => {
await handler({
body: {
collectionId
}
};
await handler(req as any);
} as any);
expect(authDatasetCollection).toHaveBeenCalledWith(
expect.objectContaining({
collectionId
})
);
expect(MongoDatasetTraining.updateMany).toHaveBeenCalledWith(
{
expect.objectContaining({
teamId: 'team1',
datasetId,
collectionId,
errorMsg: { $exists: true, $ne: null }
},
$expr: expect.any(Object)
}),
{
$unset: { errorMsg: '' },
retryCount: 3,
......@@ -57,118 +70,134 @@ describe('updateTrainingData', () => {
);
});
it('should update single training data with image', async () => {
vi.mocked(MongoDatasetTraining.findOne).mockResolvedValue({
imageId: 'image1'
});
const req = {
it('should retry only final errors in dataset scope', async () => {
await handler({
body: {
collectionId,
dataId,
q: 'question',
a: 'answer',
chunkIndex: 1
datasetId
}
};
await handler(req as any);
} as any);
expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith(
{
expect(authDataset).toHaveBeenCalledWith(
expect.objectContaining({
datasetId
})
);
expect(MongoDatasetTraining.updateMany).toHaveBeenCalledWith(
expect.objectContaining({
teamId: 'team1',
datasetId,
collectionId,
_id: dataId
},
$expr: expect.any(Object)
}),
{
$unset: { errorMsg: '' },
retryCount: 3,
mode: TrainingModeEnum.chunk,
q: 'question',
a: 'answer',
chunkIndex: 1,
lockTime: new Date('2000')
}
);
});
it('should update single training data without image', async () => {
vi.mocked(MongoDatasetTraining.findOne).mockResolvedValue({});
it('should update single training data with collection boundary', async () => {
vi.mocked(MongoDatasetTraining.findById).mockResolvedValue({
_id: dataId,
imageId: 'image1',
teamId: 'team1',
datasetId,
collectionId
});
const req = {
await handler({
body: {
collectionId,
dataId,
q: 'question',
a: 'answer',
chunkIndex: 1
}
};
} as any);
await handler(req as any);
const match = {
teamId: 'team1',
datasetId,
collectionId,
_id: dataId
};
expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith(
{
teamId: 'team1',
datasetId,
collectionId,
_id: dataId
},
{
$unset: { errorMsg: '' },
retryCount: 3,
q: 'question',
a: 'answer',
chunkIndex: 1,
lockTime: new Date('2000')
}
expect(MongoDatasetTraining.findById).toHaveBeenCalledWith(dataId);
expect(authDatasetCollection).toHaveBeenCalledWith(
expect.objectContaining({
collectionId
})
);
expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith(match, {
$unset: { errorMsg: '' },
retryCount: 3,
mode: TrainingModeEnum.chunk,
q: 'question',
a: 'answer',
chunkIndex: 1,
lockTime: new Date('2000')
});
});
it('should reject when data not found', async () => {
vi.mocked(MongoDatasetTraining.findOne).mockResolvedValue(null);
it('should reject when single training data is not found', async () => {
vi.mocked(MongoDatasetTraining.findById).mockResolvedValue(null);
const req = {
body: {
collectionId,
dataId
}
};
await expect(handler(req as any)).rejects.toBe('data not found');
await expect(
handler({
body: {
dataId
}
} as any)
).rejects.toBe('data not found');
});
it('should ignore legacy request datasetId and use the authorized collection datasetId', async () => {
vi.mocked(MongoDatasetTraining.findOne).mockResolvedValue({});
it('should ignore legacy request datasetId and use the item collection datasetId', async () => {
vi.mocked(MongoDatasetTraining.findById).mockResolvedValue({
_id: dataId,
teamId: 'team1',
datasetId,
collectionId
});
const req = {
await handler({
body: {
datasetId: foreignDatasetId,
collectionId,
dataId,
q: 'question'
}
};
await handler(req as any);
} as any);
expect(MongoDatasetTraining.findOne).toHaveBeenCalledWith({
const match = {
teamId: 'team1',
datasetId,
collectionId,
_id: dataId
});
};
expect(MongoDatasetTraining.findById).toHaveBeenCalledWith(dataId);
expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith(
{
teamId: 'team1',
datasetId,
collectionId,
_id: dataId
},
match,
expect.objectContaining({
q: 'question'
})
);
});
});
it('should reject when the item collection boundary is inconsistent', async () => {
vi.mocked(MongoDatasetTraining.findById).mockResolvedValue({
_id: dataId,
teamId: 'team1',
datasetId: foreignDatasetId,
collectionId
});
await expect(
handler({
body: {
dataId
}
} as any)
).rejects.toBe('data not found');
expect(MongoDatasetTraining.updateOne).not.toHaveBeenCalled();
});
});
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment