Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit b3ba5de1 by light5980 Committed by GitHub
Options

feat: dataset error list (#7084)

parent 86a0ffe4
Showing with 2547 additions and 338 deletions
packages/global/core/dataset/constants.ts
......@@ -245,6 +245,12 @@ export enum TrainingModeEnum {
imageParse = 'imageParse'
}
export enum CollectionTrainingStatusEnum {
running = 'running',
error = 'error',
ready = 'ready'
}
/* ------------ search -------------- */
export enum DatasetSearchModeEnum {
embedding = 'embedding',
......
packages/global/core/dataset/type.ts
......@@ -7,6 +7,7 @@ import {
DatasetTypeEnum,
SearchScoreTypeEnum,
TrainingModeEnum,
CollectionTrainingStatusEnum,
ChunkSettingModeEnum,
ChunkTriggerConfigTypeEnum,
ParagraphChunkAIModeEnum
......@@ -262,6 +263,24 @@ export const CollectionWithDatasetSchema = DatasetCollectionSchema.extend({
});
export type CollectionWithDatasetType = z.infer<typeof CollectionWithDatasetSchema>;
export const CollectionTrainingStatusSchema = z.object({
trainingAmount: z.number().optional().default(0).meta({ description: '剩余训练数量' }),
activeTrainingAmount: z
.number()
.optional()
.default(0)
.meta({ description: '仍会继续处理的训练数量' }),
finalErrorAmount: z.number().optional().default(0).meta({ description: '最终/阻塞异常训练数量' }),
hasError: z.boolean().optional().default(false).meta({ description: '是否存在最终/阻塞异常' }),
slowestTrainingMode: z.enum(TrainingModeEnum).optional().meta({ description: '最慢训练阶段' }),
slowestTrainingStatus: z
.enum(CollectionTrainingStatusEnum)
.optional()
.default(CollectionTrainingStatusEnum.ready)
.meta({ description: '最慢训练阶段状态' })
});
export type CollectionTrainingStatusType = z.infer<typeof CollectionTrainingStatusSchema>;
/* ====== service type ===== */
/* ================= dataset ===================== */
......@@ -329,7 +348,7 @@ export const DatasetCollectionItemSchema = CollectionWithDatasetSchema.extend({
permission: PermissionSchema,
indexAmount: z.number().meta({ description: '索引数量' }),
errorCount: z.number().optional().meta({ description: '错误数量' })
});
}).merge(CollectionTrainingStatusSchema);
export type DatasetCollectionItemType = z.infer<typeof DatasetCollectionItemSchema>;
/* ================= data ===================== */
......
packages/global/openapi/core/dataset/collection/api.ts
......@@ -7,12 +7,11 @@ import { ObjectIdSchema } from '../../../../common/type/mongo';
import { OutLinkChatAuthSchema } from '../../../../support/permission/chat';
import {
DatasetCollectionSyncResultEnum,
DatasetCollectionTypeEnum,
DatasetCollectionDataProcessModeEnum,
TrainingModeEnum
} from '../../../../core/dataset/constants';
import {
ChunkSettingsSchema,
CollectionTrainingStatusSchema,
DatasetCollectionItemSchema,
DatasetCollectionSchema
} from '../../../../core/dataset/type';
......@@ -21,6 +20,9 @@ import { PaginationResponseSchema, PaginationSchema } from '../../../api';
import z from 'zod';
// ============= Scroll Collections =============
/**
* @deprecated Use ListCollectionV2BodySchema and /core/dataset/collection/listV2 instead.
*/
export const ScrollCollectionsBodySchema = z.object({
datasetId: z.string(),
parentId: z.string().nullable().optional().default(null),
......@@ -114,27 +116,27 @@ export const ListCollectionV2BodySchema = PaginationSchema.extend({
export type ListCollectionV2BodyType = z.infer<typeof ListCollectionV2BodySchema>;
// ============= List Collections V2 Response =============
export const DatasetCollectionsListItemSchema = z.object({
_id: ObjectIdSchema.meta({ description: '集合 ID' }),
parentId: DatasetCollectionSchema.shape.parentId,
tmbId: DatasetCollectionSchema.shape.tmbId,
name: DatasetCollectionSchema.shape.name,
type: DatasetCollectionSchema.shape.type,
createTime: DatasetCollectionSchema.shape.createTime,
updateTime: DatasetCollectionSchema.shape.updateTime,
forbid: DatasetCollectionSchema.shape.forbid,
trainingType: DatasetCollectionSchema.shape.trainingType,
tags: z.array(z.string()).optional().meta({ description: '标签' }),
externalFileId: z.string().optional().meta({ description: '外部文件 ID' }),
fileId: z.string().optional().meta({ description: '文件 ID' }),
rawLink: z.string().optional().meta({ description: '原始链接' }),
permission: PermissionSchema,
dataAmount: z.number().meta({ description: '数据数量' }),
trainingAmount: z.number().meta({ description: '训练数量' }),
hasError: z.boolean().optional().meta({ description: '是否错误' })
});
export const DatasetCollectionsListItemSchema = z
.object({
_id: ObjectIdSchema.meta({ description: '集合 ID' }),
parentId: DatasetCollectionSchema.shape.parentId,
tmbId: DatasetCollectionSchema.shape.tmbId,
name: DatasetCollectionSchema.shape.name,
type: DatasetCollectionSchema.shape.type,
createTime: DatasetCollectionSchema.shape.createTime,
updateTime: DatasetCollectionSchema.shape.updateTime,
forbid: DatasetCollectionSchema.shape.forbid,
trainingType: DatasetCollectionSchema.shape.trainingType,
tags: z.array(z.string()).optional().meta({ description: '标签' }),
externalFileId: z.string().optional().meta({ description: '外部文件 ID' }),
fileId: z.string().optional().meta({ description: '文件 ID' }),
rawLink: z.string().optional().meta({ description: '原始链接' }),
permission: PermissionSchema,
dataAmount: z.number().meta({ description: '数据数量' })
})
.merge(CollectionTrainingStatusSchema);
export type DatasetCollectionsListItemType = z.infer<typeof DatasetCollectionsListItemSchema>;
export const ListCollectionV2ResponseSchema = PaginationResponseSchema(
DatasetCollectionsListItemSchema
......
packages/global/openapi/core/dataset/collection/index.ts
......@@ -78,7 +78,8 @@ export const DatasetCollectionPath: OpenAPIPath = {
'/core/dataset/collection/scrollList': {
post: {
summary: '获取数据集集合列表（滚动分页）',
description: '获取数据集集合列表（滚动分页）',
description: '已废弃：获取数据集集合列表（滚动分页）。请改用 /core/dataset/collection/listV2',
deprecated: true,
tags: [TagsMap.datasetCollection],
requestBody: {
content: {
......
packages/global/openapi/core/dataset/training/api.ts
import { z } from 'zod';
import { ObjectIdSchema } from '../../../../common/type/mongo';
import { TrainingModeEnum } from '../../../../core/dataset/constants';
import { DatasetCollectionTypeEnum, TrainingModeEnum } from '../../../../core/dataset/constants';
import { DatasetTrainingSchema } from '../../../../core/dataset/type';
import { PaginationSchema, PaginationResponseSchema } from '../../../api';
......@@ -8,28 +8,50 @@ import { PaginationSchema, PaginationResponseSchema } from '../../../api';
* API: 更新训练数据（或重试所有错误数据）
* Route: PUT /api/core/dataset/training/updateTrainingData
* ============================================================================ */
export const UpdateTrainingDataBodySchema = z.object({
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
}),
dataId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a07',
description: '训练数据 ID，不传则重试集合内所有错误数据'
}),
q: z.string().optional().meta({
example: '什么是 FastGPT？',
description: '问题/主文本'
}),
a: z.string().optional().meta({
example: 'FastGPT 是一个 AI Agent 构建平台',
description: '回答/补充文本'
}),
chunkIndex: z.int().min(0).optional().meta({
example: 0,
description: '块索引'
export const UpdateTrainingDataBodySchema = z
.object({
datasetId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID，不传 dataId 时按知识库范围重试所有最终/阻塞异常'
}),
collectionId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID，不传 dataId 时按集合范围重试所有最终/阻塞异常'
}),
dataId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a07',
description: '训练数据 ID，传入则只重试或更新该训练数据'
}),
q: z.string().optional().meta({
example: '什么是 FastGPT？',
description: '问题/主文本'
}),
a: z.string().optional().meta({
example: 'FastGPT 是一个 AI Agent 构建平台',
description: '回答/补充文本'
}),
chunkIndex: z.int().min(0).optional().meta({
example: 0,
description: '块索引'
})
})
});
.superRefine((data, ctx) => {
if (!data.collectionId && !data.datasetId && !data.dataId) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
path: ['collectionId'],
message: 'collectionId, datasetId or dataId is required'
});
}
if (!data.dataId && data.collectionId && data.datasetId) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
path: ['collectionId'],
message: 'collectionId and datasetId cannot be used together without dataId'
});
}
});
export type UpdateTrainingDataBody = z.infer<typeof UpdateTrainingDataBodySchema>;
export const UpdateTrainingDataResponseSchema = z.undefined().meta({ description: '更新成功' });
......@@ -99,6 +121,10 @@ export const GetTrainingDataDetailResponseSchema = z
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
}),
mode: z.enum(TrainingModeEnum).meta({
example: TrainingModeEnum.chunk,
description: '训练模式'
......@@ -132,13 +158,135 @@ export const GetTrainingErrorBodySchema = PaginationSchema.extend({
});
export type GetTrainingErrorBody = z.infer<typeof GetTrainingErrorBodySchema>;
export const GetTrainingErrorResponseSchema = PaginationResponseSchema(
DatasetTrainingSchema.omit({ billId: true }).extend({
billId: z.string().optional()
export const TrainingErrorItemSchema = DatasetTrainingSchema.omit({ billId: true }).extend({
billId: z.string().optional()
});
export type TrainingErrorItemType = z.infer<typeof TrainingErrorItemSchema>;
export const TrainingErrorCollectionSchema = z.object({
_id: ObjectIdSchema.meta({ description: '集合 ID' }),
collectionId: ObjectIdSchema.meta({ description: '集合 ID' }),
name: z.string().meta({ description: '集合名称' }),
type: z.enum(DatasetCollectionTypeEnum).meta({ description: '集合类型' }),
sourceName: z.string().optional().meta({ description: '来源名称' }),
sourceId: z.string().optional().meta({ description: '来源 ID' })
});
export type TrainingErrorCollectionType = z.infer<typeof TrainingErrorCollectionSchema>;
export const TrainingErrorGroupSchema = z.object({
collection: TrainingErrorCollectionSchema.meta({ description: '集合信息' }),
items: z.array(TrainingErrorItemSchema).meta({ description: '异常训练记录列表' }),
errorCount: z.number().meta({
description: '该集合最终/阻塞异常总数'
}),
hasMoreItems: z.boolean().meta({
description: '该集合是否还有更多异常项未返回'
})
);
});
export type TrainingErrorGroupType = z.infer<typeof TrainingErrorGroupSchema>;
export const GetTrainingErrorResponseSchema = PaginationResponseSchema(TrainingErrorItemSchema);
export type GetTrainingErrorResponse = z.infer<typeof GetTrainingErrorResponseSchema>;
export const DatasetTrainingErrorPaginationLimits = {
maxPageSize: 20,
maxOffset: 100000,
maxPageNum: 5000,
defaultItemPageSize: 5,
maxItemOffset: 100000,
maxItemPageSize: 20
} as const;
/* ============================================================================
* API: 获取知识库训练错误列表（分页）
* Route: POST /api/core/dataset/training/getDatasetTrainingError
* ============================================================================ */
export const GetDatasetTrainingErrorBodySchema = PaginationSchema.extend({
pageSize: z.coerce
.number()
.int()
.min(1)
.max(DatasetTrainingErrorPaginationLimits.maxPageSize)
.optional()
.meta({
example: 10,
description: `每页集合数量，范围 [1, ${DatasetTrainingErrorPaginationLimits.maxPageSize}]`
}),
offset: z.coerce
.number()
.int()
.min(0)
.max(DatasetTrainingErrorPaginationLimits.maxOffset)
.optional()
.meta({
example: 0,
description: `集合分页偏移量，范围 [0, ${DatasetTrainingErrorPaginationLimits.maxOffset}]`
}),
pageNum: z.coerce
.number()
.int()
.min(1)
.max(DatasetTrainingErrorPaginationLimits.maxPageNum)
.optional()
.meta({
example: 1,
description: `集合分页页码，范围 [1, ${DatasetTrainingErrorPaginationLimits.maxPageNum}]`
}),
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
collectionId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID。传入时只分页加载该集合内的异常 chunk'
}),
itemOffset: z.coerce
.number()
.int()
.min(0)
.max(DatasetTrainingErrorPaginationLimits.maxItemOffset)
.optional()
.meta({
example: 5,
description: `集合内异常 chunk 偏移量，用于加载更多，范围 [0, ${DatasetTrainingErrorPaginationLimits.maxItemOffset}]`
}),
itemPageSize: z.coerce
.number()
.int()
.min(1)
.max(DatasetTrainingErrorPaginationLimits.maxItemPageSize)
.optional()
.meta({
example: DatasetTrainingErrorPaginationLimits.defaultItemPageSize,
description: `每个集合返回的异常 chunk 数量，范围 [1, ${DatasetTrainingErrorPaginationLimits.maxItemPageSize}]`
})
});
export type GetDatasetTrainingErrorBody = z.infer<typeof GetDatasetTrainingErrorBodySchema>;
export const GetDatasetTrainingErrorResponseSchema =
PaginationResponseSchema(TrainingErrorGroupSchema);
export type GetDatasetTrainingErrorResponse = z.infer<typeof GetDatasetTrainingErrorResponseSchema>;
/* ============================================================================
* API: 检查知识库是否存在训练错误
* Route: GET /api/core/dataset/training/hasDatasetTrainingError
* ============================================================================ */
export const HasDatasetTrainingErrorQuerySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
})
});
export type HasDatasetTrainingErrorQuery = z.infer<typeof HasDatasetTrainingErrorQuerySchema>;
export const HasDatasetTrainingErrorResponseSchema = z.object({
hasError: z.boolean().meta({
example: true,
description: '知识库内是否存在最终/阻塞异常训练记录'
})
});
export type HasDatasetTrainingErrorResponse = z.infer<typeof HasDatasetTrainingErrorResponseSchema>;
/* ============================================================================
* API: 获取数据集训练队列状态
* Route: GET /api/core/dataset/training/getDatasetTrainingQueue
......
packages/global/openapi/core/dataset/training/index.ts
......@@ -6,14 +6,17 @@ import {
DeleteTrainingDataBodySchema,
GetTrainingDataDetailBodySchema,
GetTrainingErrorBodySchema,
GetDatasetTrainingErrorBodySchema,
HasDatasetTrainingErrorQuerySchema,
GetDatasetTrainingQueueQuerySchema
} from './api';
export const DatasetTrainingPath: OpenAPIPath = {
'/core/dataset/training/updateTrainingData': {
put: {
summary: '更新训练数据',
description: '更新单条训练数据，或批量重试集合内所有错误数据（不传 dataId）',
summary: '更新训练数据或重试训练异常',
description:
'更新或重试训练异常：传 dataId 时重试或编辑后重训单个 chunk；不传 dataId 时，可按 collectionId 或 datasetId 重试全部最终/阻塞异常',
tags: [TagsMap.datasetTraining],
requestBody: {
content: {
......@@ -92,8 +95,8 @@ export const DatasetTrainingPath: OpenAPIPath = {
'/core/dataset/training/getTrainingError': {
post: {
summary: '获取训练错误列表',
description: '分页查询集合内训练失败的数据列表',
summary: '获取集合训练错误列表',
description: '分页查询集合内最终/阻塞异常训练记录',
tags: [TagsMap.datasetTraining],
requestBody: {
content: {
......@@ -110,6 +113,43 @@ export const DatasetTrainingPath: OpenAPIPath = {
}
},
'/core/dataset/training/getDatasetTrainingError': {
post: {
summary: '获取知识库训练错误列表',
description:
'分页查询知识库内存在最终/阻塞异常的集合，并返回每个集合内的部分异常 chunk；传 collectionId 时加载该集合更多异常 chunk',
tags: [TagsMap.datasetTraining],
requestBody: {
content: {
'application/json': {
schema: GetDatasetTrainingErrorBodySchema
}
}
},
responses: {
200: {
description: '成功返回知识库错误数据分页列表'
}
}
}
},
'/core/dataset/training/hasDatasetTrainingError': {
get: {
summary: '检查知识库是否存在训练错误',
description: '轻量检查知识库内是否存在最终/阻塞异常训练记录，用于列表徽章状态探测',
tags: [TagsMap.datasetTraining],
requestParams: {
query: HasDatasetTrainingErrorQuerySchema
},
responses: {
200: {
description: '成功返回是否存在训练错误'
}
}
}
},
'/core/dataset/training/getDatasetTrainingQueue': {
get: {
summary: '获取训练队列状态',
......
packages/service/core/dataset/training/constants.ts deleted 100644 → 0
export enum ImportDataSourceEnum {
fileLocal = 'fileLocal',
fileLink = 'fileLink',
fileCustom = 'fileCustom'
}
packages/service/core/dataset/training/controller.ts
......@@ -12,23 +12,54 @@ import { getLLMMaxChunkSize } from '../../../../global/core/dataset/training/uti
import { retryFn } from '@fastgpt/global/common/system/utils';
import { getLogger, LogCategories } from '../../../common/logger';
import { checkTimerLock, deleteTimerLock } from '../../../common/system/timerLock/utils';
import { BLOCKED_LOCK_TIME } from './query';
const logger = getLogger(LogCategories.MODULE.DATASET.TRAINING);
export const lockTrainingDataByTeamId = async (teamId: string): Promise<any> => {
export const lockTrainingDataByTeamId = async (
teamId: string,
currentTrainingId?: string
): Promise<any> => {
const timerId = `lock_training_data--${teamId}`;
const errorMsg = i18nT('common:code_error.team_error.ai_points_not_enough');
const lockCurrentTraining = () => {
if (!currentTrainingId) return Promise.resolve();
return MongoDatasetTraining.updateOne(
{
teamId,
_id: currentTrainingId
},
{
lockTime: BLOCKED_LOCK_TIME,
errorMsg
}
);
};
// 5 分钟闸门：并发/多节点调用时，只有首个抢到锁的会执行；TTL 作为兜底
const acquired = await checkTimerLock({ timerId, lockMinuted: 30 });
if (!acquired) return;
if (!acquired) {
// 其它 worker 已在执行团队级锁定时，当前已领取任务仍需要单独标记，避免最后一次重试被扣到 0 后不可见。
await lockCurrentTraining().catch((error) => {
logger.error('lock current training data failed', { teamId, currentTrainingId, error });
});
return;
}
try {
await MongoDatasetTraining.updateMany(
{
teamId
teamId,
$or: [
{ retryCount: { $gt: 0 } },
...(currentTrainingId ? [{ _id: currentTrainingId }] : [])
]
},
{
lockTime: new Date('2999/5/5')
lockTime: BLOCKED_LOCK_TIME,
errorMsg
}
);
} catch (error) {
......
packages/service/core/dataset/training/query.ts 0 → 100644
import {
CollectionTrainingStatusEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import type { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
type TrainingStatusCount = {
activeCount: number;
finalErrorCount: number;
};
export const BLOCKED_LOCK_TIME = new Date('2050-01-01');
export const trainingModeRankMap: Record<TrainingModeEnum, number> = {
[TrainingModeEnum.parse]: 0,
[TrainingModeEnum.imageParse]: 1,
[TrainingModeEnum.qa]: 2,
[TrainingModeEnum.image]: 3,
[TrainingModeEnum.auto]: 4,
[TrainingModeEnum.chunk]: 5
};
export const trainingModeRanks = Object.values(TrainingModeEnum).map((mode) => ({
mode,
rank: trainingModeRankMap[mode]
}));
const trimmedErrorMsgExpr = (fieldPath = '$errorMsg') => ({
$trim: {
input: {
$ifNull: [fieldPath, '']
}
}
});
/**
* 判断训练记录是否有有效错误信息。空字符串和纯空白字符串都视为无错误，
* 避免自动重试或历史脏数据被错误计入最终异常。
*/
export const hasEffectiveErrorMsg = (training?: Pick<DatasetTrainingSchemaType, 'errorMsg'>) => {
return typeof training?.errorMsg === 'string' && training.errorMsg.trim() !== '';
};
/**
* active 表示仍可能被训练队列继续处理的剩余任务，包含普通排队/训练中和自动重试中。
* 这里不判断 lockTime 是否已经到达队列可消费时间，只判断未被永久锁定。
*/
export const isActiveTraining = (
training?: Pick<DatasetTrainingSchemaType, 'retryCount' | 'lockTime'>
) => {
return (training?.retryCount ?? 0) > 0 && new Date(training?.lockTime ?? 0) < BLOCKED_LOCK_TIME;
};
export const isTemporarilyFailedTraining = (
training?: Pick<DatasetTrainingSchemaType, 'retryCount' | 'lockTime' | 'errorMsg'>
) => {
return hasEffectiveErrorMsg(training) && isActiveTraining(training);
};
export const isFinalErrorTraining = (
training?: Pick<DatasetTrainingSchemaType, 'retryCount' | 'lockTime' | 'errorMsg'>
) => {
return (
hasEffectiveErrorMsg(training) &&
((training?.retryCount ?? 0) <= 0 || new Date(training?.lockTime ?? 0) >= BLOCKED_LOCK_TIME)
);
};
export const isRemainingTraining = (
training?: Pick<DatasetTrainingSchemaType, 'retryCount' | 'lockTime' | 'errorMsg'>
) => {
return isActiveTraining(training) || isFinalErrorTraining(training);
};
export const hasEffectiveErrorMsgExpr = { $gt: [{ $strLenCP: trimmedErrorMsgExpr() }, 0] };
export const activeTrainingExpr = {
$and: [{ $gt: ['$retryCount', 0] }, { $lt: ['$lockTime', BLOCKED_LOCK_TIME] }]
};
export const finalErrorTrainingExpr = {
$and: [
hasEffectiveErrorMsgExpr,
{
$or: [{ $lte: ['$retryCount', 0] }, { $gte: ['$lockTime', BLOCKED_LOCK_TIME] }]
}
]
};
export const remainingTrainingExpr = {
$or: [activeTrainingExpr, finalErrorTrainingExpr]
};
export const hasEffectiveErrorMsgMatch = {
$expr: hasEffectiveErrorMsgExpr
};
export const activeTrainingMatch = {
retryCount: { $gt: 0 },
lockTime: { $lt: BLOCKED_LOCK_TIME }
};
export const finalErrorTrainingMatch = {
$expr: finalErrorTrainingExpr
};
export const remainingTrainingMatch = {
$or: [activeTrainingMatch, finalErrorTrainingMatch]
};
/**
* rank 越小表示流程越早；collection 的“最慢阶段”就是剩余任务里流程最早的阶段。
*/
export const getTrainingModeRank = (mode?: TrainingModeEnum) => {
if (!mode) return Number.MAX_SAFE_INTEGER;
return trainingModeRankMap[mode] ?? Number.MAX_SAFE_INTEGER;
};
/**
* 返回流程中更早的训练阶段，用于计算用户感知上的“最慢阶段”。
*/
export const compareTrainingModeBySlowest = (a?: TrainingModeEnum, b?: TrainingModeEnum) => {
return getTrainingModeRank(a) - getTrainingModeRank(b);
};
export const getSlowestTrainingMode = (modes: Array<TrainingModeEnum | undefined>) => {
return modes.filter(Boolean).sort((a, b) => compareTrainingModeBySlowest(a, b))[0] as
| TrainingModeEnum
| undefined;
};
/**
* 根据各阶段 active/final error 数量计算 collection 级最慢阶段状态。
* 最慢阶段只有最终异常时才展示 error。
*/
export const getSlowestTrainingStatus = (
modeCounts: Partial<Record<TrainingModeEnum, TrainingStatusCount>>
) => {
const slowestTrainingMode = getSlowestTrainingMode(
Object.entries(modeCounts)
.filter(([, count]) => (count?.activeCount ?? 0) + (count?.finalErrorCount ?? 0) > 0)
.map(([mode]) => mode as TrainingModeEnum)
);
if (!slowestTrainingMode) {
return {
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
}
const slowestCounts = modeCounts[slowestTrainingMode];
return {
slowestTrainingMode,
slowestTrainingStatus:
(slowestCounts?.activeCount ?? 0) > 0
? CollectionTrainingStatusEnum.running
: CollectionTrainingStatusEnum.error
};
};
packages/service/core/dataset/training/schema.ts
......@@ -11,7 +11,6 @@ import {
} from '@fastgpt/global/support/user/team/constant';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
import { DatasetDataCollectionName } from '../data/schema';
import { getLogger, LogCategories } from '../../../common/logger';
export const DatasetTrainingCollectionName = 'dataset_trainings';
......@@ -121,6 +120,12 @@ TrainingDataSchema.virtual('data', {
// lock training data(teamId); delete training data
TrainingDataSchema.index({ teamId: 1, datasetId: 1 });
// collection 级状态、错误列表、删除、详情
TrainingDataSchema.index({
teamId: 1,
datasetId: 1,
collectionId: 1
});
// get training data and sort
TrainingDataSchema.index({ mode: 1, retryCount: 1, lockTime: 1, weight: -1 });
TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 * 60 }); // 7 days
......
packages/service/support/permission/dataset/auth.ts
......@@ -138,7 +138,6 @@ export const authDataset = async ({
export async function authDatasetCollection({
collectionId,
per = NullPermissionVal,
isRoot = false,
...props
}: AuthModeType & {
collectionId: string;
......
packages/service/test/core/dataset/training/controller.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import { i18nT } from '@fastgpt/global/common/i18n/utils';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { lockTrainingDataByTeamId } from '@fastgpt/service/core/dataset/training/controller';
import {
BLOCKED_LOCK_TIME,
finalErrorTrainingMatch,
isFinalErrorTraining
} from '@fastgpt/service/core/dataset/training/query';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { getRootUser } from '@test/datas/users';
describe('dataset training controller', () => {
it('should lock retryable team trainings with AI points error message', async () => {
const root = await getRootUser();
const otherRoot = await getRootUser();
const datasetId = '507f1f77bcf86cd799439011';
const collectionId = '507f1f77bcf86cd799439012';
const billId = 'test';
const [retryable, exhausted, otherTeam] = await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 3
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 0
},
{
teamId: otherRoot.teamId,
tmbId: otherRoot.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 3
}
]);
await lockTrainingDataByTeamId(String(root.teamId));
const lockedTraining = await MongoDatasetTraining.findById(retryable._id).lean();
const exhaustedTraining = await MongoDatasetTraining.findById(exhausted._id).lean();
const otherTeamTraining = await MongoDatasetTraining.findById(otherTeam._id).lean();
const finalErrorLockedTrainingCount = await MongoDatasetTraining.countDocuments({
_id: retryable._id,
...finalErrorTrainingMatch
});
const errorMsg = i18nT('common:code_error.team_error.ai_points_not_enough');
expect(lockedTraining?.lockTime).toEqual(BLOCKED_LOCK_TIME);
expect(lockedTraining?.errorMsg).toBe(errorMsg);
expect(finalErrorLockedTrainingCount).toBe(1);
expect(
isFinalErrorTraining({
retryCount: lockedTraining?.retryCount,
lockTime: lockedTraining?.lockTime,
errorMsg: lockedTraining?.errorMsg
})
).toBe(true);
expect(exhaustedTraining?.lockTime).not.toEqual(BLOCKED_LOCK_TIME);
expect(exhaustedTraining?.errorMsg).toBeUndefined();
expect(otherTeamTraining?.lockTime).not.toEqual(BLOCKED_LOCK_TIME);
expect(otherTeamTraining?.errorMsg).toBeUndefined();
});
it('should lock the current picked training even when retry count is exhausted', async () => {
const root = await getRootUser();
const datasetId = '507f1f77bcf86cd799439021';
const collectionId = '507f1f77bcf86cd799439022';
const billId = 'test';
const [pickedTraining, exhaustedHistory] = await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 0
}
]);
await lockTrainingDataByTeamId(String(root.teamId), String(pickedTraining._id));
const lockedTraining = await MongoDatasetTraining.findById(pickedTraining._id).lean();
const untouchedTraining = await MongoDatasetTraining.findById(exhaustedHistory._id).lean();
const errorMsg = i18nT('common:code_error.team_error.ai_points_not_enough');
expect(lockedTraining?.lockTime).toEqual(BLOCKED_LOCK_TIME);
expect(lockedTraining?.errorMsg).toBe(errorMsg);
expect(untouchedTraining?.lockTime).not.toEqual(BLOCKED_LOCK_TIME);
expect(untouchedTraining?.errorMsg).toBeUndefined();
});
});
packages/service/test/core/dataset/training/query.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import {
CollectionTrainingStatusEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import {
BLOCKED_LOCK_TIME,
compareTrainingModeBySlowest,
getSlowestTrainingStatus,
getTrainingModeRank,
hasEffectiveErrorMsg,
isActiveTraining,
isFinalErrorTraining,
isRemainingTraining,
isTemporarilyFailedTraining
} from '@fastgpt/service/core/dataset/training/query';
describe('dataset training query helpers', () => {
it('treats empty or whitespace errorMsg as ineffective', () => {
expect(hasEffectiveErrorMsg({ errorMsg: undefined })).toBe(false);
expect(hasEffectiveErrorMsg({ errorMsg: '' })).toBe(false);
expect(hasEffectiveErrorMsg({ errorMsg: ' \n\t' })).toBe(false);
expect(hasEffectiveErrorMsg({ errorMsg: 'failed' })).toBe(true);
});
it('separates active, temporary failed and final error records', () => {
const active = {
retryCount: 1,
lockTime: new Date('2049-12-31'),
errorMsg: ''
};
const temporaryFailed = {
retryCount: 1,
lockTime: new Date('2049-12-31'),
errorMsg: ' failed '
};
const retryExhausted = {
retryCount: 0,
lockTime: new Date('2049-12-31'),
errorMsg: 'failed'
};
const permanentlyLocked = {
retryCount: 3,
lockTime: BLOCKED_LOCK_TIME,
errorMsg: 'failed'
};
expect(isActiveTraining(active)).toBe(true);
expect(isRemainingTraining(active)).toBe(true);
expect(isActiveTraining(temporaryFailed)).toBe(true);
expect(isTemporarilyFailedTraining(temporaryFailed)).toBe(true);
expect(isFinalErrorTraining(temporaryFailed)).toBe(false);
expect(isActiveTraining(retryExhausted)).toBe(false);
expect(isFinalErrorTraining(retryExhausted)).toBe(true);
expect(isRemainingTraining(retryExhausted)).toBe(true);
expect(isActiveTraining(permanentlyLocked)).toBe(false);
expect(isFinalErrorTraining(permanentlyLocked)).toBe(true);
});
it('orders modes by the earliest remaining stage as slowest', () => {
expect(getTrainingModeRank(TrainingModeEnum.parse)).toBeLessThan(
getTrainingModeRank(TrainingModeEnum.imageParse)
);
expect(getTrainingModeRank(TrainingModeEnum.imageParse)).toBeLessThan(
getTrainingModeRank(TrainingModeEnum.qa)
);
expect(
compareTrainingModeBySlowest(TrainingModeEnum.image, TrainingModeEnum.chunk)
).toBeLessThan(0);
});
it('returns running when the slowest stage still has active records', () => {
const status = getSlowestTrainingStatus({
[TrainingModeEnum.parse]: { activeCount: 1, finalErrorCount: 0 },
[TrainingModeEnum.chunk]: { activeCount: 0, finalErrorCount: 2 }
});
expect(status).toEqual({
slowestTrainingMode: TrainingModeEnum.parse,
slowestTrainingStatus: CollectionTrainingStatusEnum.running
});
});
it('returns error only when the slowest stage has final errors and no active records', () => {
const status = getSlowestTrainingStatus({
[TrainingModeEnum.image]: { activeCount: 0, finalErrorCount: 1 },
[TrainingModeEnum.chunk]: { activeCount: 3, finalErrorCount: 0 }
});
expect(status).toEqual({
slowestTrainingMode: TrainingModeEnum.image,
slowestTrainingStatus: CollectionTrainingStatusEnum.error
});
});
it('returns ready when there are no remaining records', () => {
expect(getSlowestTrainingStatus({})).toEqual({
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
});
});
});
packages/web/components/common/Tag/index.tsx
......@@ -4,10 +4,13 @@ import { Box, type BoxProps, Flex, type FlexProps } from '@chakra-ui/react';
export type ColorSchemaType =
| 'white'
| 'blue'
| 'cyan'
| 'green'
| 'red'
| 'lightRed'
| 'yellow'
| 'gray'
| 'lightGray'
| 'purple'
| 'adora';
......@@ -57,6 +60,21 @@ const colorMap: Record<
bg: 'primary.50',
color: 'primary.600'
},
cyan: {
borderColor: '#BCE7FF',
bg: '#F0FBFF',
color: '#0884DD'
},
lightRed: {
borderColor: 'red.200',
bg: 'red.50',
color: 'red.500'
},
lightGray: {
borderColor: 'myGray.200',
bg: 'myGray.100',
color: 'myGray.500'
},
purple: {
borderColor: '#ECF',
bg: '#F6EEFA',
......
packages/web/hooks/useScrollPagination.tsx
import React, { type ReactNode, type RefObject, useMemo, useRef, useState } from 'react';
import React, { type ReactNode, type RefObject, useRef, useState } from 'react';
import { Box, type BoxProps } from '@chakra-ui/react';
import { useToast } from './useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
......@@ -190,6 +190,7 @@ export function useScrollPagination<
EmptyTip,
showErrorToast = true,
disabled = false,
showNoMoreTip = true,
...props
}: {
......@@ -200,6 +201,7 @@ export function useScrollPagination<
EmptyTip?: React.JSX.Element;
showErrorToast?: boolean;
disabled?: boolean;
showNoMoreTip?: boolean;
} & Parameters<typeof useRequest>[1]
) {
const { t } = useTranslation();
......@@ -286,7 +288,7 @@ export function useScrollPagination<
}
);
let ScrollRef = useRef<HTMLDivElement>(null);
const ScrollRef = useRef<HTMLDivElement>(null);
const ScrollData = useMemoizedFn(
({
children,
......@@ -299,11 +301,11 @@ export function useScrollPagination<
ScrollContainerRef?: RefObject<HTMLDivElement>;
} & BoxProps) => {
const ref = ScrollContainerRef || ScrollRef;
const loadText = useMemo(() => {
const loadText = (() => {
if (isLoading || isLoadingProp) return t('common:is_requesting');
if (noMore) return t('common:request_end');
return t('common:request_more');
}, [isLoading, noMore]);
})();
const scroll = useScroll(ref);
......@@ -339,7 +341,7 @@ export function useScrollPagination<
</Box>
)}
{children}
{scrollLoadType === 'bottom' && !isEmpty && (
{scrollLoadType === 'bottom' && !isEmpty && (showNoMoreTip || !noMore) && (
<Box
mt={2}
fontSize={'xs'}
......
packages/web/i18n/en/dataset.json
......@@ -201,5 +201,9 @@
"website_info": "Website Information",
"yuque_dataset": "Yuque Knowledge Base",
"yuque_dataset_config": "Configure Yuque Knowledge Base",
"yuque_dataset_desc": "Build knowledge base using Yuque documents by configuring document permissions, documents will not be stored twice"
"yuque_dataset_desc": "Build knowledge base using Yuque documents by configuring document permissions, documents will not be stored twice",
"training.stage_error": "{{stage}} error",
"training.status_error": "Training error",
"training_error_list": "Error list",
"training_error_empty": "No training errors"
}
packages/web/i18n/zh-CN/dataset.json
......@@ -201,5 +201,9 @@
"website_info": "网站信息",
"yuque_dataset": "语雀知识库",
"yuque_dataset_config": "配置语雀知识库",
"yuque_dataset_desc": "可通过配置语雀文档权限，使用语雀文档构建知识库，文档不会进行二次存储"
"yuque_dataset_desc": "可通过配置语雀文档权限，使用语雀文档构建知识库，文档不会进行二次存储",
"training.stage_error": "{{stage}}异常",
"training.status_error": "训练异常",
"training_error_list": "异常列表",
"training_error_empty": "暂无训练异常"
}
packages/web/i18n/zh-Hant/dataset.json
......@@ -201,5 +201,9 @@
"website_info": "網站資訊",
"yuque_dataset": "語雀知識庫",
"yuque_dataset_config": "設定語雀知識庫",
"yuque_dataset_desc": "可透過設定語雀文件權限，使用語雀文件建構知識庫，文件不會進行二次儲存"
"yuque_dataset_desc": "可透過設定語雀文件權限，使用語雀文件建構知識庫，文件不會進行二次儲存",
"training.stage_error": "{{stage}}異常",
"training.status_error": "訓練異常",
"training_error_list": "異常列表",
"training_error_empty": "暫無訓練異常"
}
projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx
......@@ -40,7 +40,15 @@ const FileSourceSelector = dynamic(() => import('../Import/components/FileSource
const BackupImportModal = dynamic(() => import('./BackupImportModal'));
const TemplateImportModal = dynamic(() => import('./TemplateImportModal'));
const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
const Header = ({
hasTrainingData,
hasTrainingError,
onOpenTrainingErrorModal
}: {
hasTrainingData: boolean;
hasTrainingError: boolean;
onOpenTrainingErrorModal: () => void;
}) => {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
const { isPc } = useSystem();
......@@ -196,6 +204,23 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
{datasetDetail.type !== DatasetTypeEnum.websiteDataset &&
datasetDetail.permission.hasWritePer &&
feConfigs?.isPlus && <HeaderTagPopOver />}
{hasTrainingError && (
<Button
variant={'whiteBase'}
h={'36px'}
px={'14px'}
color={'#F97066'}
borderColor={'#F97066'}
_hover={{
color: '#F97066',
borderColor: '#F97066'
}}
onClick={onOpenTrainingErrorModal}
>
{t('dataset:training_error_list')}
</Button>
)}
</HStack>
{/* diff collection button */}
......
projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingErrorEditView.tsx 0 → 100644
import { Box, Button, Flex } from '@chakra-ui/react';
import type { GetTrainingDataDetailResponse } from '@fastgpt/global/openapi/core/dataset/training/api';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import { useTranslation } from 'next-i18next';
import MyTextarea from '@/components/common/Textarea/MyTextarea';
import MyImage from '@/components/MyImage';
import { useForm } from 'react-hook-form';
const TrainingErrorEditView = ({
loading,
editChunk,
onCancel,
onSave
}: {
loading: boolean;
editChunk: GetTrainingDataDetailResponse;
onCancel: () => void;
onSave: (data: { q: string; a?: string; chunkIndex?: number }) => void;
}) => {
const { t } = useTranslation();
const { register, handleSubmit } = useForm({
defaultValues: {
q: editChunk?.q || '',
a: editChunk?.a || ''
}
});
return (
<Flex flexDirection={'column'} gap={4}>
{editChunk?.imagePreviewUrl && (
<Box>
<FormLabel>{t('file:image')}</FormLabel>
<Box w={'100%'} h={'200px'} border={'base'} borderRadius={'md'}>
<MyImage src={editChunk.imagePreviewUrl} alt="image" w={'100%'} h={'100%'} />
</Box>
</Box>
)}
<Box>
{(editChunk?.a || editChunk?.imagePreviewUrl) && (
<FormLabel>
{editChunk?.a
? t('common:dataset_data_input_chunk_content')
: t('common:dataset_data_input_q')}
</FormLabel>
)}
<MyTextarea
{...register('q', { required: true })}
minH={editChunk?.a || editChunk?.imagePreviewUrl ? 200 : 400}
/>
</Box>
{editChunk?.a && (
<Box>
<Box>{t('common:dataset_data_input_a')}</Box>
<MyTextarea {...register('a')} minH={200} />
</Box>
)}
<Flex justifyContent={'flex-end'} gap={4}>
<Button variant={'outline'} onClick={onCancel}>
{t('common:Cancel')}
</Button>
<Button isLoading={loading} variant={'primary'} onClick={handleSubmit(onSave)}>
{t('common:Confirm')}
</Button>
</Flex>
</Flex>
);
};
export default TrainingErrorEditView;
projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingErrorModal.tsx 0 → 100644
import MyModal from '@fastgpt/web/components/v2/common/MyModal';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type';
import { useTranslation } from 'next-i18next';
import TrainingErrorList from './TrainingErrorList';
const TrainingErrorModal = ({
datasetId,
permission,
onClose,
onRefresh
}: {
datasetId: string;
permission: DatasetItemType['permission'];
onClose: () => void;
onRefresh?: () => void;
}) => {
const { t } = useTranslation();
return (
<MyModal
isOpen
onClose={onClose}
title={t('dataset:training_error_list')}
size={'lg'}
isCentered
borderRadius={'10px'}
sx={{
'.chakra-modal__close-btn': {
top: '8px',
right: '8px',
w: '36px',
h: '36px'
}
}}
>
<TrainingErrorList
scope={{ type: 'dataset', datasetId }}
permission={permission}
onClose={onClose}
onRefresh={onRefresh}
showFooter
/>
</MyModal>
);
};
export default TrainingErrorModal;
projects/app/src/pageComponents/dataset/detail/DataCard.tsx
......@@ -36,6 +36,10 @@ import { formatFileSize } from '@fastgpt/global/common/file/tools';
import MyImage from '@fastgpt/web/components/common/Image/MyImage';
import dynamic from 'next/dynamic';
import { downloadFetch } from '@/web/common/system/utils';
import {
getCollectionTrainingStatusColorSchema,
getCollectionTrainingStatusText
} from '@/web/core/dataset/trainingStatus';
const InsertImagesModal = dynamic(() => import('./data/InsertImageModal'), {
ssr: false
......@@ -101,6 +105,13 @@ const DataCard = () => {
);
const canWrite = useMemo(() => datasetDetail.permission.hasWritePer, [datasetDetail]);
const collectionTrainingStatus = useMemo(() => {
if (!collection) return;
return {
text: getCollectionTrainingStatusText(collection),
colorSchema: getCollectionTrainingStatusColorSchema(collection)
};
}, [collection]);
const [
isInsertImagesModalOpen,
......@@ -240,21 +251,19 @@ const DataCard = () => {
indexAmount: collection?.indexAmount ?? '-'
})}
</Box>
{!!collection?.errorCount && (
{!!collectionTrainingStatus && (
<MyTag
colorSchema={'red'}
type={'fill'}
cursor={'pointer'}
rounded={'full'}
ml={2}
colorSchema={collectionTrainingStatus.colorSchema}
onClick={() => {
setErrorModalId(collection._id);
setErrorModalId(collection?._id || '');
}}
>
<Flex fontWeight={'medium'} alignItems={'center'} gap={1}>
{t('dataset:data_error_amount', {
errorAmount: collection?.errorCount
})}
{t(collectionTrainingStatus.text as any)}
<MyIcon name={'common/maximize'} w={'11px'} />
</Flex>
</MyTag>
......@@ -462,10 +471,11 @@ const DataCard = () => {
}}
/>
)}
{errorModalId && (
{errorModalId && collection && (
<TrainingStates
defaultTab={'errors'}
defaultTab={collection?.hasError ? 'errors' : 'states'}
collectionId={errorModalId}
permission={collection.permission}
onClose={() => {
setErrorModalId('');
refreshList();
......
projects/app/src/pages/api/core/dataset/collection/detail.ts
......@@ -9,6 +9,7 @@ import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collecti
import { getVectorCount } from '@fastgpt/service/common/vectorDB/controller';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { Types } from '@fastgpt/service/common/mongo';
import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset';
import { isS3ObjectKey } from '@fastgpt/service/common/s3/utils';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
......@@ -18,6 +19,124 @@ import {
GetCollectionDetailQuerySchema,
GetCollectionDetailResponseSchema
} from '@fastgpt/global/openapi/core/dataset/collection/api';
import {
activeTrainingExpr,
finalErrorTrainingExpr,
getSlowestTrainingStatus,
remainingTrainingMatch,
trainingModeRanks
} from '@fastgpt/service/core/dataset/training/query';
import { CollectionTrainingStatusEnum } from '@fastgpt/global/core/dataset/constants';
const defaultCollectionTrainingStatus = {
trainingAmount: 0,
activeTrainingAmount: 0,
finalErrorAmount: 0,
hasError: false,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
/**
* 获取数据集集合的训练状态统计信息
* @param teamId - 团队ID
* @param datasetId - 数据集ID
* @param collectionId - 集合ID
* @returns 包含训练数量、活跃训练数、错误数及最慢训练状态的统计对象
*/
const getCollectionTrainingStatus = async ({
teamId,
datasetId,
collectionId
}: {
teamId: Types.ObjectId;
datasetId: Types.ObjectId;
collectionId: Types.ObjectId;
}) => {
const [trainingStatus] = await MongoDatasetTraining.aggregate(
[
{
$match: {
teamId,
datasetId,
collectionId,
...remainingTrainingMatch
}
},
{
$addFields: {
modeRank: {
$switch: {
branches: trainingModeRanks.map(({ mode, rank }) => ({
case: { $eq: ['$mode', mode] },
then: rank
})),
default: 999
}
},
isActiveTraining: activeTrainingExpr,
isFinalErrorTraining: finalErrorTrainingExpr
}
},
{
$group: {
_id: '$mode',
modeRank: { $first: '$modeRank' },
activeCount: { $sum: { $cond: ['$isActiveTraining', 1, 0] } },
finalErrorCount: { $sum: { $cond: ['$isFinalErrorTraining', 1, 0] } },
trainingAmount: { $sum: 1 }
}
},
{
$sort: {
modeRank: 1
}
},
{
$group: {
_id: null,
trainingAmount: { $sum: '$trainingAmount' },
activeTrainingAmount: { $sum: '$activeCount' },
finalErrorAmount: { $sum: '$finalErrorCount' },
modeCounts: {
$push: {
mode: '$_id',
activeCount: '$activeCount',
finalErrorCount: '$finalErrorCount'
}
}
}
}
],
readFromSecondary
);
if (!trainingStatus) return defaultCollectionTrainingStatus;
const { slowestTrainingMode, slowestTrainingStatus } = getSlowestTrainingStatus(
Object.fromEntries(
trainingStatus.modeCounts.map(
({
mode,
activeCount,
finalErrorCount
}: {
mode: any;
activeCount: number;
finalErrorCount: number;
}) => [mode, { activeCount, finalErrorCount }]
)
)
);
return {
trainingAmount: trainingStatus.trainingAmount,
activeTrainingAmount: trainingStatus.activeTrainingAmount,
finalErrorAmount: trainingStatus.finalErrorAmount,
hasError: trainingStatus.finalErrorAmount > 0,
slowestTrainingMode,
slowestTrainingStatus
};
};
async function handler(req: ApiRequestProps): Promise<GetCollectionDetailResponseType> {
const { id } = parseApiInput({ req, querySchema: GetCollectionDetailQuerySchema }).query;
......@@ -36,23 +155,18 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionDetailRespons
return Promise.reject('Invalid dataset file key');
}
const [file, indexAmount, errorCount] = await Promise.all([
const [file, indexAmount, trainingStatus] = await Promise.all([
fileId ? getS3DatasetSource().getFileMetadata(fileId) : undefined,
getVectorCount({
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id
}),
MongoDatasetTraining.countDocuments(
{
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: id,
errorMsg: { $exists: true },
retryCount: { $lte: 0 }
},
readFromSecondary
)
getCollectionTrainingStatus({
teamId: new Types.ObjectId(collection.teamId),
datasetId: new Types.ObjectId(collection.datasetId),
collectionId: new Types.ObjectId(collection._id)
})
]);
return GetCollectionDetailResponseSchema.parse({
......@@ -65,7 +179,8 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionDetailRespons
}),
permission,
file,
errorCount
...trainingStatus,
errorCount: trainingStatus.finalErrorAmount
});
}
......
projects/app/src/pages/api/core/dataset/collection/listV2.ts
......@@ -13,11 +13,64 @@ import { replaceRegChars } from '@fastgpt/global/common/string/tools';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import {
activeTrainingExpr,
finalErrorTrainingExpr,
getSlowestTrainingStatus,
remainingTrainingMatch,
trainingModeRanks
} from '@fastgpt/service/core/dataset/training/query';
import {
CollectionTrainingStatusEnum,
type TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import {
ListCollectionV2BodySchema,
ListCollectionV2ResponseSchema,
type ListCollectionV2ResponseType
} from '@fastgpt/global/openapi/core/dataset/collection/api';
const defaultCollectionTrainingStatus = {
trainingAmount: 0,
activeTrainingAmount: 0,
finalErrorAmount: 0,
hasError: false,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
type TrainingAmountAggregateItem = {
_id: string;
trainingAmount: number;
activeTrainingAmount: number;
finalErrorAmount: number;
modeCounts: {
mode: TrainingModeEnum;
activeCount: number;
finalErrorCount: number;
}[];
};
const formatTrainingStatus = (item?: TrainingAmountAggregateItem) => {
if (!item) return defaultCollectionTrainingStatus;
const { slowestTrainingMode, slowestTrainingStatus } = getSlowestTrainingStatus(
Object.fromEntries(
item.modeCounts.map(({ mode, activeCount, finalErrorCount }) => [
mode,
{ activeCount, finalErrorCount }
])
)
);
return {
trainingAmount: item.trainingAmount,
activeTrainingAmount: item.activeTrainingAmount,
finalErrorAmount: item.finalErrorAmount,
hasError: item.finalErrorAmount > 0,
slowestTrainingMode,
slowestTrainingStatus
};
};
async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseType> {
const {
datasetId,
......@@ -96,8 +149,7 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy
tags: item.tags
}),
dataAmount: 0,
trainingAmount: 0,
hasError: false,
...defaultCollectionTrainingStatus,
permission
}))
),
......@@ -118,7 +170,7 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy
// Compute data amount
const [trainingAmount, dataAmount]: [
{ _id: string; count: number; hasError: boolean }[],
TrainingAmountAggregateItem[],
{ _id: string; count: number }[]
] = await Promise.all([
MongoDatasetTraining.aggregate(
......@@ -127,14 +179,75 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy
$match: {
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(datasetId),
collectionId: { $in: collectionIds }
collectionId: { $in: collectionIds },
...remainingTrainingMatch
}
},
{
$addFields: {
modeRank: {
$switch: {
branches: trainingModeRanks.map(({ mode, rank }) => ({
case: { $eq: ['$mode', mode] },
then: rank
})),
default: 999
}
},
isActiveTraining: activeTrainingExpr,
isFinalErrorTraining: finalErrorTrainingExpr
}
},
{
$group: {
_id: '$collectionId',
count: { $sum: 1 },
hasError: { $max: { $cond: [{ $ifNull: ['$errorMsg', false] }, true, false] } }
trainingAmount: { $sum: 1 },
activeTrainingAmount: { $sum: { $cond: ['$isActiveTraining', 1, 0] } },
finalErrorAmount: { $sum: { $cond: ['$isFinalErrorTraining', 1, 0] } },
modeCounts: {
$push: {
mode: '$mode',
modeRank: '$modeRank',
activeCount: { $cond: ['$isActiveTraining', 1, 0] },
finalErrorCount: { $cond: ['$isFinalErrorTraining', 1, 0] }
}
}
}
},
{ $unwind: '$modeCounts' },
{
$group: {
_id: {
collectionId: '$_id',
mode: '$modeCounts.mode',
modeRank: '$modeCounts.modeRank'
},
trainingAmount: { $first: '$trainingAmount' },
activeTrainingAmount: { $first: '$activeTrainingAmount' },
finalErrorAmount: { $first: '$finalErrorAmount' },
activeCount: { $sum: '$modeCounts.activeCount' },
finalErrorCount: { $sum: '$modeCounts.finalErrorCount' }
}
},
{
$sort: {
'_id.collectionId': 1,
'_id.modeRank': 1
}
},
{
$group: {
_id: '$_id.collectionId',
trainingAmount: { $first: '$trainingAmount' },
activeTrainingAmount: { $first: '$activeTrainingAmount' },
finalErrorAmount: { $first: '$finalErrorAmount' },
modeCounts: {
$push: {
mode: '$_id.mode',
activeCount: '$activeCount',
finalErrorCount: '$finalErrorCount'
}
}
}
}
],
......@@ -171,10 +284,10 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy
datasetId,
tags: item.tags
}),
trainingAmount:
trainingAmount.find((amount) => String(amount._id) === String(item._id))?.count || 0,
dataAmount: dataAmount.find((amount) => String(amount._id) === String(item._id))?.count || 0,
hasError: trainingAmount.find((amount) => String(amount._id) === String(item._id))?.hasError,
...formatTrainingStatus(
trainingAmount.find((amount) => String(amount._id) === String(item._id))
),
permission
}))
);
......
projects/app/src/pages/api/core/dataset/collection/scrollList.ts
/**
* @deprecated Use /core/dataset/collection/listV2 instead.
*/
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { NextAPI } from '@/service/middleware/entry';
......@@ -5,7 +8,10 @@ import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/tra
import { Types } from '@fastgpt/service/common/mongo';
import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import {
CollectionTrainingStatusEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { type PaginationResponse } from '@fastgpt/global/openapi/api';
import type { DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api';
......@@ -14,6 +20,13 @@ import { replaceRegChars } from '@fastgpt/global/common/string/tools';
import { ScrollCollectionsBodySchema } from '@fastgpt/global/openapi/core/dataset/collection/api';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
const defaultCollectionTrainingStatus = {
activeTrainingAmount: 0,
finalErrorAmount: 0,
hasError: false,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
async function handler(
req: ApiRequestProps
): Promise<PaginationResponse<DatasetCollectionsListItemType>> {
......@@ -80,6 +93,7 @@ async function handler(
...item,
dataAmount: 0,
trainingAmount: 0,
...defaultCollectionTrainingStatus,
indexAmount: 0,
permission
}))
......@@ -150,7 +164,11 @@ async function handler(
},
trainingAmount: {
$ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0]
}
},
activeTrainingAmount: { $literal: 0 },
finalErrorAmount: { $literal: 0 },
hasError: { $literal: false },
slowestTrainingStatus: { $literal: CollectionTrainingStatusEnum.ready }
}
}
]),
......
projects/app/src/pages/api/core/dataset/collection/trainingDetail.ts
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import {
DatasetCollectionDataProcessModeEnum,
type TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { type TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
......@@ -15,6 +12,11 @@ import {
GetCollectionTrainingDetailResponseSchema,
type GetCollectionTrainingDetailResponseType
} from '@fastgpt/global/openapi/core/dataset/collection/api';
import {
BLOCKED_LOCK_TIME,
activeTrainingMatch,
finalErrorTrainingMatch
} from '@fastgpt/service/core/dataset/training/query';
const defaultCounts: Record<TrainingModeEnum, number> = {
parse: 0,
......@@ -57,7 +59,7 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai
$match: {
_id: { $lt: new Types.ObjectId(minId) },
retryCount: { $gt: 0 },
lockTime: { $lt: new Date('2050/1/1') }
lockTime: { $lt: BLOCKED_LOCK_TIME }
}
},
{
......@@ -72,8 +74,7 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai
{
$match: {
...match,
retryCount: { $gt: 0 },
lockTime: { $lt: new Date('2050/1/1') }
...activeTrainingMatch
}
},
{
......@@ -87,8 +88,7 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai
{
$match: {
...match,
// retryCount: { $lte: 0 },
errorMsg: { $exists: true }
...finalErrorTrainingMatch
}
},
{
......
projects/app/src/pages/api/core/dataset/training/deleteTrainingData.ts
......@@ -35,3 +35,6 @@ async function handler(req: ApiRequestProps): Promise<DeleteTrainingDataResponse
}
export default NextAPI(handler);
export type deleteTrainingDataBody =
import('@fastgpt/global/openapi/core/dataset/training/api').DeleteTrainingDataBody;
export type deleteTrainingDataResponse = DeleteTrainingDataResponse;
projects/app/src/pages/api/core/dataset/training/getDatasetTrainingError.ts 0 → 100644
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
import { Types, type PipelineStage } from '@fastgpt/service/common/mongo';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import {
DatasetTrainingErrorPaginationLimits,
GetDatasetTrainingErrorBodySchema,
GetDatasetTrainingErrorResponseSchema,
type GetDatasetTrainingErrorBody,
type GetDatasetTrainingErrorResponse,
type TrainingErrorGroupType,
type TrainingErrorItemType
} from '@fastgpt/global/openapi/core/dataset/training/api';
import {
finalErrorTrainingMatch,
trainingModeRanks
} from '@fastgpt/service/core/dataset/training/query';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
async function handler(req: ApiRequestProps): Promise<GetDatasetTrainingErrorResponse> {
const {
datasetId,
collectionId,
itemOffset = 0,
itemPageSize = DatasetTrainingErrorPaginationLimits.defaultItemPageSize
} = parseApiInput({
req,
bodySchema: GetDatasetTrainingErrorBodySchema
}).body;
const { offset, pageSize: rawPageSize } = parsePaginationRequest(req);
const pageSize = Math.min(rawPageSize, DatasetTrainingErrorPaginationLimits.maxPageSize);
const itemLimit = Math.min(itemPageSize, DatasetTrainingErrorPaginationLimits.maxItemPageSize);
const itemSkip = Math.min(itemOffset, DatasetTrainingErrorPaginationLimits.maxItemOffset);
const { teamId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: ReadPermissionVal
});
const baseTrainingMatch = {
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(dataset._id),
...finalErrorTrainingMatch
};
/**
* 复用集合内异常 chunk 查询流水线，确保首屏和“加载更多”的排序完全一致。
*/
const buildItemPipeline = ({
match,
skip,
limit
}: {
match: Record<string, any>;
skip: number;
limit: number;
}): PipelineStage[] => [
{ $match: match },
{
$addFields: {
modeRank: {
$switch: {
branches: trainingModeRanks.map(({ mode, rank }) => ({
case: { $eq: ['$mode', mode] },
then: rank
})),
default: 999
}
}
}
},
{ $sort: { modeRank: 1, chunkIndex: 1, _id: 1 } },
{ $skip: skip },
{ $limit: limit },
{ $project: { modeRank: 0 } }
];
if (collectionId) {
const collection = await MongoDatasetCollection.findOne(
{
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(dataset._id),
_id: new Types.ObjectId(collectionId),
type: { $ne: DatasetCollectionTypeEnum.folder }
},
undefined,
readFromSecondary
).lean();
if (!collection) {
return GetDatasetTrainingErrorResponseSchema.parse({
total: 0,
list: []
});
}
const collectionMatch = {
...baseTrainingMatch,
collectionId: new Types.ObjectId(collection._id)
};
const [items, errorCount] = await Promise.all([
MongoDatasetTraining.aggregate(
buildItemPipeline({
match: collectionMatch,
skip: itemSkip,
limit: itemLimit
}),
readFromSecondary
),
MongoDatasetTraining.countDocuments(collectionMatch, { ...readFromSecondary })
]);
const { sourceName, sourceId } = getCollectionSourceData(collection);
return GetDatasetTrainingErrorResponseSchema.parse({
total: errorCount > 0 ? 1 : 0,
list:
errorCount > 0
? [
{
collection: {
_id: collection._id,
collectionId: collection._id,
name: collection.name,
type: collection.type,
sourceName,
sourceId
},
items,
errorCount,
hasMoreItems: itemSkip + items.length < errorCount
}
]
: []
});
}
const [collectionStats, totalResult] = await Promise.all([
MongoDatasetTraining.aggregate(
[
{ $match: baseTrainingMatch },
{
$group: {
_id: '$collectionId',
errorCount: { $sum: 1 }
}
},
{ $sort: { _id: 1 } },
{ $skip: offset },
{ $limit: pageSize }
],
readFromSecondary
),
MongoDatasetTraining.aggregate(
[
{ $match: baseTrainingMatch },
{
$group: {
_id: '$collectionId'
}
},
{ $count: 'total' }
],
readFromSecondary
)
]);
const total = totalResult[0]?.total ?? 0;
const collectionIds = collectionStats.map((item: { _id: any }) => item._id);
const collections = collectionIds.length
? await MongoDatasetCollection.find(
{
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(dataset._id),
_id: { $in: collectionIds },
type: { $ne: DatasetCollectionTypeEnum.folder }
},
undefined,
readFromSecondary
).lean()
: [];
const collectionMap = new Map(
collections.map((collection) => [String(collection._id), collection])
);
const statsMap = new Map(
collectionStats.map((item: { _id: any; errorCount: number }) => [
String(item._id),
item.errorCount
])
);
const itemsList = collectionIds.length
? await Promise.all(
collectionIds.map((id: any) =>
MongoDatasetTraining.aggregate(
buildItemPipeline({
match: {
...baseTrainingMatch,
collectionId: id
},
skip: 0,
limit: itemLimit
}),
readFromSecondary
)
)
)
: [];
const list = collectionIds.reduce<TrainingErrorGroupType[]>((groups, id: any, index: number) => {
const collection = collectionMap.get(String(id));
const errorCount = statsMap.get(String(id)) ?? 0;
const items = (itemsList[index] ?? []) as TrainingErrorItemType[];
if (!collection || errorCount === 0 || items.length === 0) return groups;
const { sourceName, sourceId } = getCollectionSourceData(collection);
groups.push({
collection: {
_id: collection._id,
collectionId: collection._id,
name: collection.name,
type: collection.type,
sourceName,
sourceId
},
items,
errorCount,
hasMoreItems: items.length < errorCount
});
return groups;
}, []);
return GetDatasetTrainingErrorResponseSchema.parse({
total,
list
});
}
export default NextAPI(handler);
export type getDatasetTrainingErrorBody = GetDatasetTrainingErrorBody;
export type getDatasetTrainingErrorResponse = GetDatasetTrainingErrorResponse;
projects/app/src/pages/api/core/dataset/training/getTrainingDataDetail.ts
......@@ -41,6 +41,7 @@ async function handler(req: ApiRequestProps): Promise<GetTrainingDataDetailRespo
return GetTrainingDataDetailResponseSchema.parse({
_id: data._id,
datasetId: data.datasetId,
collectionId: data.collectionId,
mode: data.mode,
imagePreviewUrl:
data.imageId && isS3ObjectKey(data.imageId, 'dataset')
......
projects/app/src/pages/api/core/dataset/training/getTrainingError.ts
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
import { Types, type PipelineStage } from '@fastgpt/service/common/mongo';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
......@@ -9,11 +10,19 @@ import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import {
GetTrainingErrorBodySchema,
GetTrainingErrorResponseSchema,
type GetTrainingErrorBody,
type GetTrainingErrorResponse
} from '@fastgpt/global/openapi/core/dataset/training/api';
import {
finalErrorTrainingMatch,
trainingModeRanks
} from '@fastgpt/service/core/dataset/training/query';
async function handler(req: ApiRequestProps): Promise<GetTrainingErrorResponse> {
const { collectionId } = parseApiInput({ req, bodySchema: GetTrainingErrorBodySchema }).body;
const { collectionId } = parseApiInput({
req,
bodySchema: GetTrainingErrorBodySchema
}).body;
const { offset, pageSize } = parsePaginationRequest(req);
const { collection } = await authDatasetCollection({
......@@ -25,26 +34,43 @@ async function handler(req: ApiRequestProps): Promise<GetTrainingErrorResponse>
});
const match = {
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id,
errorMsg: { $exists: true }
teamId: new Types.ObjectId(collection.teamId),
datasetId: new Types.ObjectId(collection.datasetId),
collectionId: new Types.ObjectId(collection._id),
...finalErrorTrainingMatch
};
const pipeline: PipelineStage[] = [
{ $match: match },
{
$addFields: {
modeRank: {
$switch: {
branches: trainingModeRanks.map(({ mode, rank }) => ({
case: { $eq: ['$mode', mode] },
then: rank
})),
default: 999
}
}
}
},
{ $sort: { modeRank: 1, chunkIndex: 1, _id: 1 } },
{ $skip: offset },
{ $limit: pageSize },
{ $project: { modeRank: 0 } }
];
const [errorList, total] = await Promise.all([
MongoDatasetTraining.find(match, undefined, {
...readFromSecondary
})
.skip(offset)
.limit(pageSize)
.lean(),
MongoDatasetTraining.aggregate(pipeline, readFromSecondary),
MongoDatasetTraining.countDocuments(match, { ...readFromSecondary })
]);
return GetTrainingErrorResponseSchema.parse({
list: errorList,
total
total,
list: errorList
});
}
export default NextAPI(handler);
export type getTrainingErrorBody = GetTrainingErrorBody;
export type getTrainingErrorResponse = GetTrainingErrorResponse;
projects/app/src/pages/api/core/dataset/training/hasDatasetTrainingError.ts 0 → 100644
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { Types } from '@fastgpt/service/common/mongo';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { finalErrorTrainingMatch } from '@fastgpt/service/core/dataset/training/query';
import {
HasDatasetTrainingErrorQuerySchema,
HasDatasetTrainingErrorResponseSchema,
type HasDatasetTrainingErrorQuery,
type HasDatasetTrainingErrorResponse
} from '@fastgpt/global/openapi/core/dataset/training/api';
async function handler(req: ApiRequestProps): Promise<HasDatasetTrainingErrorResponse> {
const { datasetId } = parseApiInput({
req,
querySchema: HasDatasetTrainingErrorQuerySchema
}).query;
const { teamId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: ReadPermissionVal
});
const errorRecord = await MongoDatasetTraining.findOne(
{
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(dataset._id),
...finalErrorTrainingMatch
},
{ _id: 1 },
readFromSecondary
).lean();
return HasDatasetTrainingErrorResponseSchema.parse({
hasError: !!errorRecord
});
}
export default NextAPI(handler);
export type hasDatasetTrainingErrorQuery = HasDatasetTrainingErrorQuery;
export type hasDatasetTrainingErrorResponse = HasDatasetTrainingErrorResponse;
projects/app/src/pages/api/core/dataset/training/updateTrainingData.ts
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import {
authDataset,
authDatasetCollection
} from '@fastgpt/service/support/permission/dataset/auth';
import { NextAPI } from '@/service/middleware/entry';
import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
......@@ -10,33 +13,48 @@ import {
type UpdateTrainingDataResponse
} from '@fastgpt/global/openapi/core/dataset/training/api';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { finalErrorTrainingMatch } from '@fastgpt/service/core/dataset/training/query';
async function handler(req: ApiRequestProps): Promise<UpdateTrainingDataResponse> {
const { collectionId, dataId, q, a, chunkIndex } = parseApiInput({
req,
bodySchema: UpdateTrainingDataBodySchema
}).body;
const body = parseApiInput({ req, bodySchema: UpdateTrainingDataBodySchema }).body;
const { collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: WritePermissionVal
});
// 不传 dataId 时是批量重试：collectionId 和 datasetId 分别限定不同的重试范围。
if (!body.dataId) {
const retryMatch = await (async () => {
if (body.collectionId) {
const { collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId: body.collectionId,
per: WritePermissionVal
});
const trainingMatch = {
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id
};
return {
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id
};
}
const { teamId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId: body.datasetId!,
per: WritePermissionVal
});
return {
teamId,
datasetId: dataset._id
};
})();
// If dataId is not passed, all error data in this collection will be retried.
if (!dataId) {
await MongoDatasetTraining.updateMany(
{
...trainingMatch,
errorMsg: { $exists: true, $ne: null }
...retryMatch,
...finalErrorTrainingMatch
},
{
$unset: { errorMsg: '' },
......@@ -47,45 +65,57 @@ async function handler(req: ApiRequestProps): Promise<UpdateTrainingDataResponse
return UpdateTrainingDataResponseSchema.parse(undefined);
}
// Single data retry logic
const data = await MongoDatasetTraining.findOne({ ...trainingMatch, _id: dataId });
const { q, a, chunkIndex } = body;
// 单条重试只信任 dataId 找到的训练记录，再用记录所属 collection 做权限校验。
const data = await MongoDatasetTraining.findById(body.dataId);
if (!data) {
return Promise.reject('data not found');
}
const { collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId: data.collectionId,
per: WritePermissionVal
});
if (
String(collection.teamId) !== String(data.teamId) ||
String(collection.datasetId) !== String(data.datasetId) ||
String(collection._id) !== String(data.collectionId)
) {
return Promise.reject('data not found');
}
const trainingMatch = {
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id,
_id: data._id
};
// Add to chunk
if (data.imageId && q) {
await MongoDatasetTraining.updateOne(
{
...trainingMatch,
_id: dataId
},
{
$unset: { errorMsg: '' },
retryCount: 3,
mode: TrainingModeEnum.chunk,
...(q !== undefined && { q }),
...(a !== undefined && { a }),
...(chunkIndex !== undefined && { chunkIndex }),
lockTime: new Date('2000')
}
);
await MongoDatasetTraining.updateOne(trainingMatch, {
$unset: { errorMsg: '' },
retryCount: 3,
mode: TrainingModeEnum.chunk,
...(q !== undefined && { q }),
...(a !== undefined && { a }),
...(chunkIndex !== undefined && { chunkIndex }),
lockTime: new Date('2000')
});
} else {
await MongoDatasetTraining.updateOne(
{
...trainingMatch,
_id: dataId
},
{
$unset: { errorMsg: '' },
retryCount: 3,
...(q !== undefined && { q }),
...(a !== undefined && { a }),
...(chunkIndex !== undefined && { chunkIndex }),
lockTime: new Date('2000')
}
);
await MongoDatasetTraining.updateOne(trainingMatch, {
$unset: { errorMsg: '' },
retryCount: 3,
...(q !== undefined && { q }),
...(a !== undefined && { a }),
...(chunkIndex !== undefined && { chunkIndex }),
lockTime: new Date('2000')
});
}
return UpdateTrainingDataResponseSchema.parse(undefined);
......
projects/app/src/service/core/dataset/queues/datasetParse.ts
......@@ -167,7 +167,7 @@ export const datasetParseQueue = async (): Promise<any> => {
continue;
}
// Check team points and lock(No mistakes will be thrown here)
if (!(await checkTeamAiPointsAndLock(data.teamId))) {
if (!(await checkTeamAiPointsAndLock(data.teamId, String(data._id)))) {
continue;
}
......
projects/app/src/service/core/dataset/queues/generateQA.ts
......@@ -112,7 +112,7 @@ export async function generateQA(): Promise<any> {
continue;
}
// auth balance
if (!(await checkTeamAiPointsAndLock(data.teamId))) {
if (!(await checkTeamAiPointsAndLock(data.teamId, String(data._id)))) {
continue;
}
......
projects/app/src/service/core/dataset/queues/generateVector.ts
......@@ -160,7 +160,7 @@ export async function generateVector(): Promise<any> {
}
// auth balance
if (!(await checkTeamAiPointsAndLock(data.teamId))) {
if (!(await checkTeamAiPointsAndLock(data.teamId, String(data._id)))) {
continue;
}
......
projects/app/src/service/core/dataset/queues/utils.ts
......@@ -6,7 +6,7 @@ import { InformLevelEnum } from '@fastgpt/global/support/user/inform/constants';
import { getLogger, LogCategories } from '@fastgpt/service/common/logger';
const logger = getLogger(LogCategories.MODULE.DATASET.QUEUES);
export const checkTeamAiPointsAndLock = async (teamId: string) => {
export const checkTeamAiPointsAndLock = async (teamId: string, currentTrainingId?: string) => {
try {
await checkTeamAIPoints(teamId);
return true;
......@@ -21,7 +21,7 @@ export const checkTeamAiPointsAndLock = async (teamId: string) => {
teamId
});
logger.info('余额不足，暂停知识库处理');
await lockTrainingDataByTeamId(teamId);
await lockTrainingDataByTeamId(teamId, currentTrainingId);
} catch (error) {}
}
return false;
......
projects/app/src/web/core/dataset/api/training.ts
......@@ -7,7 +7,10 @@ import type {
GetTrainingDataDetailBody,
GetTrainingDataDetailResponse,
GetTrainingErrorBody,
GetTrainingErrorResponse
GetTrainingErrorResponse,
GetDatasetTrainingErrorBody,
GetDatasetTrainingErrorResponse,
HasDatasetTrainingErrorResponse
} from '@fastgpt/global/openapi/core/dataset/training/api';
export const postRebuildEmbedding = (data: RebuildEmbeddingBody) =>
......@@ -29,3 +32,11 @@ export const getTrainingDataDetail = (data: GetTrainingDataDetailBody) =>
export const getTrainingError = (data: GetTrainingErrorBody) =>
POST<GetTrainingErrorResponse>(`/core/dataset/training/getTrainingError`, data);
export const getDatasetTrainingError = (data: GetDatasetTrainingErrorBody) =>
POST<GetDatasetTrainingErrorResponse>(`/core/dataset/training/getDatasetTrainingError`, data);
export const hasDatasetTrainingError = (datasetId: string) =>
GET<HasDatasetTrainingErrorResponse>(`/core/dataset/training/hasDatasetTrainingError`, {
datasetId
});
projects/app/src/web/core/dataset/constants.ts
import { defaultQAModels, defaultVectorModels } from '@fastgpt/global/core/ai/constants';
import {
CollectionTrainingStatusEnum,
DatasetCollectionDataProcessModeEnum,
DatasetCollectionTypeEnum,
DatasetTypeEnum,
......@@ -60,7 +61,12 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
chunkSize: 0,
indexSize: 512,
permission: new DatasetPermission(),
indexAmount: 0
indexAmount: 0,
trainingAmount: 0,
activeTrainingAmount: 0,
finalErrorAmount: 0,
hasError: false,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
export const TrainingProcess = {
......
projects/app/src/web/core/dataset/trainingStatus.ts 0 → 100644
import {
CollectionTrainingStatusEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { i18nT } from '@fastgpt/global/common/i18n/utils';
import type { TFunction } from 'next-i18next';
import type { ColorSchemaType } from '@fastgpt/web/components/common/Tag';
/**
* 根据训练模式获取对应的阶段文本描述
* @param mode - 训练模式枚举，若未提供则返回“等待中”
* @returns 国际化后的阶段文本字符串
*/
export const getTrainingStageText = (mode?: TrainingModeEnum) => {
const textMap = {
[TrainingModeEnum.parse]: i18nT('dataset:process.Parsing'),
[TrainingModeEnum.imageParse]: i18nT('dataset:process.Parse_Image'),
[TrainingModeEnum.qa]: i18nT('dataset:process.Get QA'),
[TrainingModeEnum.image]: i18nT('dataset:process.Image_Index'),
[TrainingModeEnum.auto]: i18nT('dataset:process.Auto_Index'),
[TrainingModeEnum.chunk]: i18nT('dataset:process.Vectorizing')
};
return mode ? textMap[mode] : i18nT('dataset:process.Waiting');
};
/**
* 根据集合中最慢的训练任务和状态，获取整体训练状态文本
* @param params.slowestTrainingMode - 最慢任务的训练模式
* @param params.slowestTrainingStatus - 最慢任务的训练状态
* @returns 国际化后的状态文本（错误/进行中具体阶段/等待中/已就绪）
*/
export const getCollectionTrainingStatusText = ({
slowestTrainingMode,
slowestTrainingStatus
}: {
slowestTrainingMode?: TrainingModeEnum;
slowestTrainingStatus?: CollectionTrainingStatusEnum;
}) => {
if (slowestTrainingStatus === CollectionTrainingStatusEnum.error) {
return i18nT('dataset:training.status_error');
}
if (slowestTrainingStatus === CollectionTrainingStatusEnum.running && slowestTrainingMode) {
return getTrainingStageText(slowestTrainingMode);
}
if (slowestTrainingStatus === CollectionTrainingStatusEnum.running) {
return i18nT('dataset:process.Waiting');
}
return i18nT('dataset:process.Is_Ready');
};
/**
* 使用传入的翻译函数格式化训练阶段文本
* @param mode - 训练模式枚举
* @param t - i18next 翻译函数实例
* @returns 国际化后的阶段文本字符串
*/
export const formatTrainingStageText = (mode: TrainingModeEnum | undefined, t: TFunction) => {
return t(getTrainingStageText(mode) as any);
};
/**
* 根据集合中最慢的训练任务和状态，获取状态标签的颜色主题
* @param params.slowestTrainingMode - 最慢任务的训练模式
* @param params.slowestTrainingStatus - 最慢任务的训练状态
* @returns MyTag 可消费的颜色主题名
*/
export const getCollectionTrainingStatusColorSchema = ({
slowestTrainingMode,
slowestTrainingStatus
}: {
slowestTrainingMode?: TrainingModeEnum;
slowestTrainingStatus?: CollectionTrainingStatusEnum;
}): ColorSchemaType => {
if (slowestTrainingStatus === CollectionTrainingStatusEnum.error) {
return 'lightRed';
}
if (slowestTrainingStatus !== CollectionTrainingStatusEnum.running) {
return 'green';
}
switch (slowestTrainingMode) {
case TrainingModeEnum.parse:
case TrainingModeEnum.imageParse:
return 'blue';
case TrainingModeEnum.qa:
case TrainingModeEnum.image:
case TrainingModeEnum.auto:
return 'cyan';
case TrainingModeEnum.chunk:
return 'adora';
default:
return 'lightGray';
}
};
projects/app/test/api/core/dataset/collection/trainingStatus.test.ts 0 → 100644
import listHandler from '@/pages/api/core/dataset/collection/listV2';
import scrollListHandler from '@/pages/api/core/dataset/collection/scrollList';
import detailHandler from '@/pages/api/core/dataset/collection/detail';
import trainingDetailHandler from '@/pages/api/core/dataset/collection/trainingDetail';
import {
CollectionTrainingStatusEnum,
DatasetCollectionTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { DatasetCollectionsListItemSchema } from '@fastgpt/global/openapi/core/dataset/collection/api';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { getRootUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest';
describe('collection training status api', () => {
it('should expose unified active/final error/slowest status in list and detail', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 3
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'final error'
}
]);
const listRes = await Call(listHandler, {
auth: root,
body: {
datasetId: dataset._id,
pageSize: 10,
offset: 0,
filterTags: []
}
});
expect(listRes.code).toBe(200);
expect(listRes.data.list[0]).toMatchObject({
trainingAmount: 2,
activeTrainingAmount: 1,
finalErrorAmount: 1,
hasError: true,
slowestTrainingMode: TrainingModeEnum.parse,
slowestTrainingStatus: CollectionTrainingStatusEnum.running
});
const detailRes = await Call(detailHandler, {
auth: root,
query: {
id: collection._id
}
});
expect(detailRes.code).toBe(200);
expect(detailRes.data).toMatchObject({
trainingAmount: 2,
activeTrainingAmount: 1,
finalErrorAmount: 1,
errorCount: 1,
hasError: true,
slowestTrainingMode: TrainingModeEnum.parse,
slowestTrainingStatus: CollectionTrainingStatusEnum.running
});
});
it('should use active counts for progress and final errors for error tab', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 3,
errorMsg: 'temporary failed'
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'final failed'
}
]);
const res = await Call(trainingDetailHandler, {
auth: root,
query: {
collectionId: collection._id
}
});
expect(res.code).toBe(200);
expect(res.data.trainingCounts.qa).toBe(1);
expect(res.data.errorCounts.qa).toBe(0);
expect(res.data.errorCounts.chunk).toBe(1);
});
it('should keep deprecated scrollList compatible with the collection list item schema', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'final failed'
});
const res = await Call(scrollListHandler, {
auth: root,
body: {
datasetId: dataset._id,
pageSize: 10,
offset: 0,
filterTags: []
}
});
expect(res.code).toBe(200);
expect(() => DatasetCollectionsListItemSchema.parse(res.data.list[0])).not.toThrow();
expect(res.data.list[0]).toMatchObject({
trainingAmount: 1,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
});
});
});
projects/app/test/api/core/dataset/training/deleteTrainingData.test.ts
......@@ -39,7 +39,11 @@ describe('delete training data test', () => {
mode: TrainingModeEnum.chunk
});
const res = await Call<deleteTrainingDataBody, {}, deleteTrainingDataResponse>(handler, {
const res = await Call<
deleteTrainingDataBody,
Record<string, never>,
deleteTrainingDataResponse
>(handler, {
auth: root,
body: {
collectionId: collection._id,
......@@ -100,7 +104,11 @@ describe('delete training data test', () => {
mode: TrainingModeEnum.chunk
});
const res = await Call<deleteTrainingDataBody, {}, deleteTrainingDataResponse>(handler, {
const res = await Call<
deleteTrainingDataBody,
Record<string, never>,
deleteTrainingDataResponse
>(handler, {
auth: root,
body: {
datasetId: foreignDataset._id,
......@@ -114,4 +122,4 @@ describe('delete training data test', () => {
expect(res.code).toBe(200);
expect(existingTrainingData).toBeTruthy();
});
});
});
\ No newline at end of file
projects/app/test/api/core/dataset/training/getDatasetTrainingError.test.ts 0 → 100644
import handler, {
type getDatasetTrainingErrorBody,
type getDatasetTrainingErrorResponse
} from '@/pages/api/core/dataset/training/getDatasetTrainingError';
import {
DatasetCollectionTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { DatasetTrainingErrorPaginationLimits } from '@fastgpt/global/openapi/core/dataset/training/api';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { ApiRequestInputParseError } from '@fastgpt/service/common/zod/requestParseError';
import { getRootUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest';
type EmptyQuery = Record<string, never>;
describe('dataset training error list test', () => {
it('should paginate error collections and load more chunks inside one collection', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const [fileCollection, linkCollection] = await MongoDatasetCollection.create([
{
name: 'file',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
updateTime: new Date('2025-01-01')
},
{
name: 'link',
type: DatasetCollectionTypeEnum.link,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
rawLink: 'https://example.com',
updateTime: new Date('2025-01-02')
}
]);
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: fileCollection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 3,
errorMsg: 'temporary failed',
chunkIndex: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: fileCollection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'file error',
chunkIndex: 2
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: fileCollection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 0,
errorMsg: 'file parse error',
chunkIndex: 1
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: fileCollection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: ' ',
chunkIndex: 3
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: linkCollection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 0,
errorMsg: 'link error',
chunkIndex: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: linkCollection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 3,
errorMsg: 'link chunk error',
chunkIndex: 1
}
]);
const firstPageRes = await Call<
getDatasetTrainingErrorBody,
EmptyQuery,
getDatasetTrainingErrorResponse
>(handler, {
auth: root,
body: {
datasetId: dataset._id,
pageSize: 1,
offset: 0,
itemPageSize: 1
}
});
expect(firstPageRes.code).toBe(200);
expect(firstPageRes.data.total).toBe(2);
expect(firstPageRes.data.list).toHaveLength(1);
expect(firstPageRes.data.list[0].collection.name).toBe('file');
expect(firstPageRes.data.list[0].errorCount).toBe(2);
expect(firstPageRes.data.list[0].hasMoreItems).toBe(true);
expect(firstPageRes.data.list[0].items).toHaveLength(1);
expect(firstPageRes.data.list[0].items[0].mode).toBe(TrainingModeEnum.parse);
const secondPageRes = await Call<
getDatasetTrainingErrorBody,
EmptyQuery,
getDatasetTrainingErrorResponse
>(handler, {
auth: root,
body: {
datasetId: dataset._id,
pageSize: 1,
offset: 1,
itemPageSize: 1
}
});
expect(secondPageRes.code).toBe(200);
expect(secondPageRes.data.total).toBe(2);
expect(secondPageRes.data.list).toHaveLength(1);
expect(secondPageRes.data.list[0].collection.name).toBe('link');
expect(secondPageRes.data.list[0].errorCount).toBe(1);
expect(secondPageRes.data.list[0].hasMoreItems).toBe(false);
expect(secondPageRes.data.list[0].items[0].mode).toBe(TrainingModeEnum.qa);
const loadMoreRes = await Call<
getDatasetTrainingErrorBody,
EmptyQuery,
getDatasetTrainingErrorResponse
>(handler, {
auth: root,
body: {
datasetId: dataset._id,
collectionId: fileCollection._id,
pageSize: 1,
offset: 0,
itemOffset: 1,
itemPageSize: 1
}
});
expect(loadMoreRes.code).toBe(200);
expect(loadMoreRes.data.total).toBe(1);
expect(loadMoreRes.data.list).toHaveLength(1);
expect(loadMoreRes.data.list[0].collection.name).toBe('file');
expect(loadMoreRes.data.list[0].errorCount).toBe(2);
expect(loadMoreRes.data.list[0].hasMoreItems).toBe(false);
expect(loadMoreRes.data.list[0].items).toHaveLength(1);
expect(loadMoreRes.data.list[0].items[0].mode).toBe(TrainingModeEnum.chunk);
});
it.each([
{
name: 'invalid itemOffset',
body: {
datasetId: '507f1f77bcf86cd799439011',
pageSize: 1,
offset: 0,
itemOffset: 'not-a-number',
itemPageSize: 1
}
},
{
name: 'oversized itemPageSize',
body: {
datasetId: '507f1f77bcf86cd799439011',
pageSize: 1,
offset: 0,
itemPageSize: DatasetTrainingErrorPaginationLimits.maxItemPageSize + 1
}
},
{
name: 'invalid pageSize',
body: {
datasetId: '507f1f77bcf86cd799439011',
pageSize: 'not-a-number',
offset: 0,
itemPageSize: 1
}
},
{
name: 'oversized pageSize',
body: {
datasetId: '507f1f77bcf86cd799439011',
pageSize: DatasetTrainingErrorPaginationLimits.maxPageSize + 1,
offset: 0,
itemPageSize: 1
}
}
])('should reject $name before querying Mongo', async ({ body }) => {
const res = await Call<any, EmptyQuery, getDatasetTrainingErrorResponse>(handler, {
body
});
expect(res.code).not.toBe(200);
expect(res.error).toBeInstanceOf(ApiRequestInputParseError);
});
});
projects/app/test/api/core/dataset/training/getTrainingDataDetail.test.ts
import handler, {
type getTrainingDataDetailBody,
type getTrainingDataDetailResponse
} from '@/pages/api/core/dataset/training/getTrainingDataDetail';
import handler from '@/pages/api/core/dataset/training/getTrainingDataDetail';
import {
DatasetCollectionTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import type {
GetTrainingDataDetailBody,
GetTrainingDataDetailResponse
} from '@fastgpt/global/openapi/core/dataset/training/api';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
......@@ -41,7 +42,11 @@ describe('get training data detail test', () => {
a: 'test'
});
const res = await Call<getTrainingDataDetailBody, {}, getTrainingDataDetailResponse>(handler, {
const res = await Call<
GetTrainingDataDetailBody,
Record<string, never>,
GetTrainingDataDetailResponse
>(handler, {
auth: root,
body: {
collectionId: collection._id,
......@@ -103,7 +108,11 @@ describe('get training data detail test', () => {
a: 'foreign'
});
const res = await Call<getTrainingDataDetailBody, {}, getTrainingDataDetailResponse>(handler, {
const res = await Call<
GetTrainingDataDetailBody,
Record<string, never>,
GetTrainingDataDetailResponse
>(handler, {
auth: root,
body: {
datasetId: foreignDataset._id,
......
projects/app/test/api/core/dataset/training/getTrainingError.test.ts
......@@ -13,8 +13,10 @@ import { getRootUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest';
type EmptyQuery = Record<string, never>;
describe('training error list test', () => {
it('should return training error list', async () => {
it('should return final error list in collection scope', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
......@@ -30,19 +32,63 @@ describe('training error list test', () => {
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create(
[...Array(10).keys()].map((i) => ({
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'chunk first in insert order',
chunkIndex: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 0,
errorMsg: 'parse should sort before chunk',
chunkIndex: 9
},
...[...Array(10).keys()].map((i) => ({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
errorMsg: 'test'
}))
);
retryCount: 0,
errorMsg: 'test',
chunkIndex: i
})),
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 3,
errorMsg: 'temporary failed'
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 0,
errorMsg: ' '
}
]);
const res = await Call<getTrainingErrorBody, {}, getTrainingErrorResponse>(handler, {
const res = await Call<getTrainingErrorBody, EmptyQuery, getTrainingErrorResponse>(handler, {
auth: root,
body: {
collectionId: collection._id,
......@@ -52,7 +98,25 @@ describe('training error list test', () => {
});
expect(res.code).toBe(200);
expect(res.data.total).toBe(10);
expect(res.data.total).toBe(12);
expect(res.data.list.length).toBe(10);
expect(
res.data.list.map((item) => ('mode' in item ? item.mode : undefined)).slice(0, 2)
).toEqual([TrainingModeEnum.parse, TrainingModeEnum.chunk]);
expect(res.data.list.every((item) => 'mode' in item)).toBe(true);
});
it('should reject dataset scope request', async () => {
const root = await getRootUser();
const res = await Call<getTrainingErrorBody, EmptyQuery, getTrainingErrorResponse>(handler, {
auth: root,
body: {
datasetId: '507f1f77bcf86cd799439012',
pageSize: 10,
offset: 0
} as any
});
expect(res.code).not.toBe(200);
});
});
projects/app/test/api/core/dataset/training/hasDatasetTrainingError.test.ts 0 → 100644
import handler, {
type hasDatasetTrainingErrorQuery,
type hasDatasetTrainingErrorResponse
} from '@/pages/api/core/dataset/training/hasDatasetTrainingError';
import {
DatasetCollectionTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { getRootUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest';
type EmptyBody = Record<string, never>;
describe('dataset training error existence test', () => {
it('should only report final or blocked training errors', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 3,
errorMsg: 'temporary failed',
chunkIndex: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 0,
errorMsg: ' ',
chunkIndex: 1
}
]);
const noFinalErrorRes = await Call<
EmptyBody,
hasDatasetTrainingErrorQuery,
hasDatasetTrainingErrorResponse
>(handler, {
auth: root,
query: {
datasetId: dataset._id
}
});
expect(noFinalErrorRes.code).toBe(200);
expect(noFinalErrorRes.data.hasError).toBe(false);
await MongoDatasetTraining.create({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 0,
errorMsg: 'parse failed',
chunkIndex: 2
});
const finalErrorRes = await Call<
EmptyBody,
hasDatasetTrainingErrorQuery,
hasDatasetTrainingErrorResponse
>(handler, {
auth: root,
query: {
datasetId: dataset._id
}
});
expect(finalErrorRes.code).toBe(200);
expect(finalErrorRes.data.hasError).toBe(true);
});
});
projects/app/test/pages/api/core/dataset/training/updateTrainingData.test.ts
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { describe, expect, it, vi, beforeEach } from 'vitest';
import { handler } from '@/pages/api/core/dataset/training/updateTrainingData';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import {
authDataset,
authDatasetCollection
} from '@fastgpt/service/support/permission/dataset/auth';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
const datasetId = '507f1f77bcf86cd799439011';
......@@ -11,13 +14,14 @@ const foreignDatasetId = '507f1f77bcf86cd799439014';
vi.mock('@fastgpt/service/core/dataset/training/schema', () => ({
MongoDatasetTraining: {
findOne: vi.fn(),
findById: vi.fn(),
updateOne: vi.fn(),
updateMany: vi.fn()
}
}));
vi.mock('@fastgpt/service/support/permission/dataset/auth', () => ({
authDataset: vi.fn(),
authDatasetCollection: vi.fn()
}));
......@@ -30,25 +34,34 @@ describe('updateTrainingData', () => {
teamId: 'team1',
datasetId
}
});
} as any);
vi.mocked(authDataset).mockResolvedValue({
teamId: 'team1',
dataset: {
_id: datasetId
}
} as any);
});
it('should retry all error data when dataId is not provided', async () => {
const req = {
it('should retry only final errors in collection scope', async () => {
await handler({
body: {
collectionId
}
};
await handler(req as any);
} as any);
expect(authDatasetCollection).toHaveBeenCalledWith(
expect.objectContaining({
collectionId
})
);
expect(MongoDatasetTraining.updateMany).toHaveBeenCalledWith(
{
expect.objectContaining({
teamId: 'team1',
datasetId,
collectionId,
errorMsg: { $exists: true, $ne: null }
},
$expr: expect.any(Object)
}),
{
$unset: { errorMsg: '' },
retryCount: 3,
......@@ -57,118 +70,134 @@ describe('updateTrainingData', () => {
);
});
it('should update single training data with image', async () => {
vi.mocked(MongoDatasetTraining.findOne).mockResolvedValue({
imageId: 'image1'
});
const req = {
it('should retry only final errors in dataset scope', async () => {
await handler({
body: {
collectionId,
dataId,
q: 'question',
a: 'answer',
chunkIndex: 1
datasetId
}
};
await handler(req as any);
} as any);
expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith(
{
expect(authDataset).toHaveBeenCalledWith(
expect.objectContaining({
datasetId
})
);
expect(MongoDatasetTraining.updateMany).toHaveBeenCalledWith(
expect.objectContaining({
teamId: 'team1',
datasetId,
collectionId,
_id: dataId
},
$expr: expect.any(Object)
}),
{
$unset: { errorMsg: '' },
retryCount: 3,
mode: TrainingModeEnum.chunk,
q: 'question',
a: 'answer',
chunkIndex: 1,
lockTime: new Date('2000')
}
);
});
it('should update single training data without image', async () => {
vi.mocked(MongoDatasetTraining.findOne).mockResolvedValue({});
it('should update single training data with collection boundary', async () => {
vi.mocked(MongoDatasetTraining.findById).mockResolvedValue({
_id: dataId,
imageId: 'image1',
teamId: 'team1',
datasetId,
collectionId
});
const req = {
await handler({
body: {
collectionId,
dataId,
q: 'question',
a: 'answer',
chunkIndex: 1
}
};
} as any);
await handler(req as any);
const match = {
teamId: 'team1',
datasetId,
collectionId,
_id: dataId
};
expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith(
{
teamId: 'team1',
datasetId,
collectionId,
_id: dataId
},
{
$unset: { errorMsg: '' },
retryCount: 3,
q: 'question',
a: 'answer',
chunkIndex: 1,
lockTime: new Date('2000')
}
expect(MongoDatasetTraining.findById).toHaveBeenCalledWith(dataId);
expect(authDatasetCollection).toHaveBeenCalledWith(
expect.objectContaining({
collectionId
})
);
expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith(match, {
$unset: { errorMsg: '' },
retryCount: 3,
mode: TrainingModeEnum.chunk,
q: 'question',
a: 'answer',
chunkIndex: 1,
lockTime: new Date('2000')
});
});
it('should reject when data not found', async () => {
vi.mocked(MongoDatasetTraining.findOne).mockResolvedValue(null);
it('should reject when single training data is not found', async () => {
vi.mocked(MongoDatasetTraining.findById).mockResolvedValue(null);
const req = {
body: {
collectionId,
dataId
}
};
await expect(handler(req as any)).rejects.toBe('data not found');
await expect(
handler({
body: {
dataId
}
} as any)
).rejects.toBe('data not found');
});
it('should ignore legacy request datasetId and use the authorized collection datasetId', async () => {
vi.mocked(MongoDatasetTraining.findOne).mockResolvedValue({});
it('should ignore legacy request datasetId and use the item collection datasetId', async () => {
vi.mocked(MongoDatasetTraining.findById).mockResolvedValue({
_id: dataId,
teamId: 'team1',
datasetId,
collectionId
});
const req = {
await handler({
body: {
datasetId: foreignDatasetId,
collectionId,
dataId,
q: 'question'
}
};
await handler(req as any);
} as any);
expect(MongoDatasetTraining.findOne).toHaveBeenCalledWith({
const match = {
teamId: 'team1',
datasetId,
collectionId,
_id: dataId
});
};
expect(MongoDatasetTraining.findById).toHaveBeenCalledWith(dataId);
expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith(
{
teamId: 'team1',
datasetId,
collectionId,
_id: dataId
},
match,
expect.objectContaining({
q: 'question'
})
);
});
});
it('should reject when the item collection boundary is inconsistent', async () => {
vi.mocked(MongoDatasetTraining.findById).mockResolvedValue({
_id: dataId,
teamId: 'team1',
datasetId: foreignDatasetId,
collectionId
});
await expect(
handler({
body: {
dataId
}
} as any)
).rejects.toBe('data not found');
expect(MongoDatasetTraining.updateOne).not.toHaveBeenCalled();
});
});
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment