Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit b3ba5de1 by light5980 Committed by GitHub
Options

feat: dataset error list (#7084)

parent 86a0ffe4
Showing with 2451 additions and 244 deletions
packages/global/core/dataset/constants.ts
...@@ -245,6 +245,12 @@ export enum TrainingModeEnum { ...@@ -245,6 +245,12 @@ export enum TrainingModeEnum {
imageParse = 'imageParse' imageParse = 'imageParse'
} }
export enum CollectionTrainingStatusEnum {
running = 'running',
error = 'error',
ready = 'ready'
}
/* ------------ search -------------- */ /* ------------ search -------------- */
export enum DatasetSearchModeEnum { export enum DatasetSearchModeEnum {
embedding = 'embedding', embedding = 'embedding',
......
packages/global/core/dataset/type.ts
...@@ -7,6 +7,7 @@ import { ...@@ -7,6 +7,7 @@ import {
DatasetTypeEnum, DatasetTypeEnum,
SearchScoreTypeEnum, SearchScoreTypeEnum,
TrainingModeEnum, TrainingModeEnum,
CollectionTrainingStatusEnum,
ChunkSettingModeEnum, ChunkSettingModeEnum,
ChunkTriggerConfigTypeEnum, ChunkTriggerConfigTypeEnum,
ParagraphChunkAIModeEnum ParagraphChunkAIModeEnum
...@@ -262,6 +263,24 @@ export const CollectionWithDatasetSchema = DatasetCollectionSchema.extend({ ...@@ -262,6 +263,24 @@ export const CollectionWithDatasetSchema = DatasetCollectionSchema.extend({
}); });
export type CollectionWithDatasetType = z.infer<typeof CollectionWithDatasetSchema>; export type CollectionWithDatasetType = z.infer<typeof CollectionWithDatasetSchema>;
export const CollectionTrainingStatusSchema = z.object({
trainingAmount: z.number().optional().default(0).meta({ description: '剩余训练数量' }),
activeTrainingAmount: z
.number()
.optional()
.default(0)
.meta({ description: '仍会继续处理的训练数量' }),
finalErrorAmount: z.number().optional().default(0).meta({ description: '最终/阻塞异常训练数量' }),
hasError: z.boolean().optional().default(false).meta({ description: '是否存在最终/阻塞异常' }),
slowestTrainingMode: z.enum(TrainingModeEnum).optional().meta({ description: '最慢训练阶段' }),
slowestTrainingStatus: z
.enum(CollectionTrainingStatusEnum)
.optional()
.default(CollectionTrainingStatusEnum.ready)
.meta({ description: '最慢训练阶段状态' })
});
export type CollectionTrainingStatusType = z.infer<typeof CollectionTrainingStatusSchema>;
/* ====== service type ===== */ /* ====== service type ===== */
/* ================= dataset ===================== */ /* ================= dataset ===================== */
...@@ -329,7 +348,7 @@ export const DatasetCollectionItemSchema = CollectionWithDatasetSchema.extend({ ...@@ -329,7 +348,7 @@ export const DatasetCollectionItemSchema = CollectionWithDatasetSchema.extend({
permission: PermissionSchema, permission: PermissionSchema,
indexAmount: z.number().meta({ description: '索引数量' }), indexAmount: z.number().meta({ description: '索引数量' }),
errorCount: z.number().optional().meta({ description: '错误数量' }) errorCount: z.number().optional().meta({ description: '错误数量' })
}); }).merge(CollectionTrainingStatusSchema);
export type DatasetCollectionItemType = z.infer<typeof DatasetCollectionItemSchema>; export type DatasetCollectionItemType = z.infer<typeof DatasetCollectionItemSchema>;
/* ================= data ===================== */ /* ================= data ===================== */
......
packages/global/openapi/core/dataset/collection/api.ts
...@@ -7,12 +7,11 @@ import { ObjectIdSchema } from '../../../../common/type/mongo'; ...@@ -7,12 +7,11 @@ import { ObjectIdSchema } from '../../../../common/type/mongo';
import { OutLinkChatAuthSchema } from '../../../../support/permission/chat'; import { OutLinkChatAuthSchema } from '../../../../support/permission/chat';
import { import {
DatasetCollectionSyncResultEnum, DatasetCollectionSyncResultEnum,
DatasetCollectionTypeEnum,
DatasetCollectionDataProcessModeEnum, DatasetCollectionDataProcessModeEnum,
TrainingModeEnum TrainingModeEnum
} from '../../../../core/dataset/constants'; } from '../../../../core/dataset/constants';
import { import {
ChunkSettingsSchema, CollectionTrainingStatusSchema,
DatasetCollectionItemSchema, DatasetCollectionItemSchema,
DatasetCollectionSchema DatasetCollectionSchema
} from '../../../../core/dataset/type'; } from '../../../../core/dataset/type';
...@@ -21,6 +20,9 @@ import { PaginationResponseSchema, PaginationSchema } from '../../../api'; ...@@ -21,6 +20,9 @@ import { PaginationResponseSchema, PaginationSchema } from '../../../api';
import z from 'zod'; import z from 'zod';
// ============= Scroll Collections ============= // ============= Scroll Collections =============
/**
* @deprecated Use ListCollectionV2BodySchema and /core/dataset/collection/listV2 instead.
*/
export const ScrollCollectionsBodySchema = z.object({ export const ScrollCollectionsBodySchema = z.object({
datasetId: z.string(), datasetId: z.string(),
parentId: z.string().nullable().optional().default(null), parentId: z.string().nullable().optional().default(null),
...@@ -114,7 +116,8 @@ export const ListCollectionV2BodySchema = PaginationSchema.extend({ ...@@ -114,7 +116,8 @@ export const ListCollectionV2BodySchema = PaginationSchema.extend({
export type ListCollectionV2BodyType = z.infer<typeof ListCollectionV2BodySchema>; export type ListCollectionV2BodyType = z.infer<typeof ListCollectionV2BodySchema>;
// ============= List Collections V2 Response ============= // ============= List Collections V2 Response =============
export const DatasetCollectionsListItemSchema = z.object({ export const DatasetCollectionsListItemSchema = z
.object({
_id: ObjectIdSchema.meta({ description: '集合 ID' }), _id: ObjectIdSchema.meta({ description: '集合 ID' }),
parentId: DatasetCollectionSchema.shape.parentId, parentId: DatasetCollectionSchema.shape.parentId,
tmbId: DatasetCollectionSchema.shape.tmbId, tmbId: DatasetCollectionSchema.shape.tmbId,
...@@ -131,10 +134,9 @@ export const DatasetCollectionsListItemSchema = z.object({ ...@@ -131,10 +134,9 @@ export const DatasetCollectionsListItemSchema = z.object({
fileId: z.string().optional().meta({ description: '文件 ID' }), fileId: z.string().optional().meta({ description: '文件 ID' }),
rawLink: z.string().optional().meta({ description: '原始链接' }), rawLink: z.string().optional().meta({ description: '原始链接' }),
permission: PermissionSchema, permission: PermissionSchema,
dataAmount: z.number().meta({ description: '数据数量' }), dataAmount: z.number().meta({ description: '数据数量' })
trainingAmount: z.number().meta({ description: '训练数量' }), })
hasError: z.boolean().optional().meta({ description: '是否错误' }) .merge(CollectionTrainingStatusSchema);
});
export type DatasetCollectionsListItemType = z.infer<typeof DatasetCollectionsListItemSchema>; export type DatasetCollectionsListItemType = z.infer<typeof DatasetCollectionsListItemSchema>;
export const ListCollectionV2ResponseSchema = PaginationResponseSchema( export const ListCollectionV2ResponseSchema = PaginationResponseSchema(
DatasetCollectionsListItemSchema DatasetCollectionsListItemSchema
......
packages/global/openapi/core/dataset/collection/index.ts
...@@ -78,7 +78,8 @@ export const DatasetCollectionPath: OpenAPIPath = { ...@@ -78,7 +78,8 @@ export const DatasetCollectionPath: OpenAPIPath = {
'/core/dataset/collection/scrollList': { '/core/dataset/collection/scrollList': {
post: { post: {
summary: '获取数据集集合列表（滚动分页）', summary: '获取数据集集合列表（滚动分页）',
description: '获取数据集集合列表（滚动分页）', description: '已废弃：获取数据集集合列表（滚动分页）。请改用 /core/dataset/collection/listV2',
deprecated: true,
tags: [TagsMap.datasetCollection], tags: [TagsMap.datasetCollection],
requestBody: { requestBody: {
content: { content: {
......
packages/global/openapi/core/dataset/training/api.ts
import { z } from 'zod'; import { z } from 'zod';
import { ObjectIdSchema } from '../../../../common/type/mongo'; import { ObjectIdSchema } from '../../../../common/type/mongo';
import { TrainingModeEnum } from '../../../../core/dataset/constants'; import { DatasetCollectionTypeEnum, TrainingModeEnum } from '../../../../core/dataset/constants';
import { DatasetTrainingSchema } from '../../../../core/dataset/type'; import { DatasetTrainingSchema } from '../../../../core/dataset/type';
import { PaginationSchema, PaginationResponseSchema } from '../../../api'; import { PaginationSchema, PaginationResponseSchema } from '../../../api';
...@@ -8,14 +8,19 @@ import { PaginationSchema, PaginationResponseSchema } from '../../../api'; ...@@ -8,14 +8,19 @@ import { PaginationSchema, PaginationResponseSchema } from '../../../api';
* API: 更新训练数据（或重试所有错误数据） * API: 更新训练数据（或重试所有错误数据）
* Route: PUT /api/core/dataset/training/updateTrainingData * Route: PUT /api/core/dataset/training/updateTrainingData
* ============================================================================ */ * ============================================================================ */
export const UpdateTrainingDataBodySchema = z.object({ export const UpdateTrainingDataBodySchema = z
collectionId: ObjectIdSchema.meta({ .object({
datasetId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID，不传 dataId 时按知识库范围重试所有最终/阻塞异常'
}),
collectionId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a06', example: '68ad85a7463006c963799a06',
description: '集合 ID' description: '集合 ID，不传 dataId 时按集合范围重试所有最终/阻塞异常'
}), }),
dataId: ObjectIdSchema.optional().meta({ dataId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a07', example: '68ad85a7463006c963799a07',
description: '训练数据 ID，不传则重试集合内所有错误数据' description: '训练数据 ID，传入则只重试或更新该训练数据'
}), }),
q: z.string().optional().meta({ q: z.string().optional().meta({
example: '什么是 FastGPT？', example: '什么是 FastGPT？',
...@@ -29,7 +34,24 @@ export const UpdateTrainingDataBodySchema = z.object({ ...@@ -29,7 +34,24 @@ export const UpdateTrainingDataBodySchema = z.object({
example: 0, example: 0,
description: '块索引' description: '块索引'
}) })
}); })
.superRefine((data, ctx) => {
if (!data.collectionId && !data.datasetId && !data.dataId) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
path: ['collectionId'],
message: 'collectionId, datasetId or dataId is required'
});
}
if (!data.dataId && data.collectionId && data.datasetId) {
ctx.addIssue({
code: z.ZodIssueCode.custom,
path: ['collectionId'],
message: 'collectionId and datasetId cannot be used together without dataId'
});
}
});
export type UpdateTrainingDataBody = z.infer<typeof UpdateTrainingDataBodySchema>; export type UpdateTrainingDataBody = z.infer<typeof UpdateTrainingDataBodySchema>;
export const UpdateTrainingDataResponseSchema = z.undefined().meta({ description: '更新成功' }); export const UpdateTrainingDataResponseSchema = z.undefined().meta({ description: '更新成功' });
...@@ -99,6 +121,10 @@ export const GetTrainingDataDetailResponseSchema = z ...@@ -99,6 +121,10 @@ export const GetTrainingDataDetailResponseSchema = z
example: '68ad85a7463006c963799a05', example: '68ad85a7463006c963799a05',
description: '知识库 ID' description: '知识库 ID'
}), }),
collectionId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID'
}),
mode: z.enum(TrainingModeEnum).meta({ mode: z.enum(TrainingModeEnum).meta({
example: TrainingModeEnum.chunk, example: TrainingModeEnum.chunk,
description: '训练模式' description: '训练模式'
...@@ -132,13 +158,135 @@ export const GetTrainingErrorBodySchema = PaginationSchema.extend({ ...@@ -132,13 +158,135 @@ export const GetTrainingErrorBodySchema = PaginationSchema.extend({
}); });
export type GetTrainingErrorBody = z.infer<typeof GetTrainingErrorBodySchema>; export type GetTrainingErrorBody = z.infer<typeof GetTrainingErrorBodySchema>;
export const GetTrainingErrorResponseSchema = PaginationResponseSchema( export const TrainingErrorItemSchema = DatasetTrainingSchema.omit({ billId: true }).extend({
DatasetTrainingSchema.omit({ billId: true }).extend({
billId: z.string().optional() billId: z.string().optional()
});
export type TrainingErrorItemType = z.infer<typeof TrainingErrorItemSchema>;
export const TrainingErrorCollectionSchema = z.object({
_id: ObjectIdSchema.meta({ description: '集合 ID' }),
collectionId: ObjectIdSchema.meta({ description: '集合 ID' }),
name: z.string().meta({ description: '集合名称' }),
type: z.enum(DatasetCollectionTypeEnum).meta({ description: '集合类型' }),
sourceName: z.string().optional().meta({ description: '来源名称' }),
sourceId: z.string().optional().meta({ description: '来源 ID' })
});
export type TrainingErrorCollectionType = z.infer<typeof TrainingErrorCollectionSchema>;
export const TrainingErrorGroupSchema = z.object({
collection: TrainingErrorCollectionSchema.meta({ description: '集合信息' }),
items: z.array(TrainingErrorItemSchema).meta({ description: '异常训练记录列表' }),
errorCount: z.number().meta({
description: '该集合最终/阻塞异常总数'
}),
hasMoreItems: z.boolean().meta({
description: '该集合是否还有更多异常项未返回'
}) })
); });
export type TrainingErrorGroupType = z.infer<typeof TrainingErrorGroupSchema>;
export const GetTrainingErrorResponseSchema = PaginationResponseSchema(TrainingErrorItemSchema);
export type GetTrainingErrorResponse = z.infer<typeof GetTrainingErrorResponseSchema>; export type GetTrainingErrorResponse = z.infer<typeof GetTrainingErrorResponseSchema>;
export const DatasetTrainingErrorPaginationLimits = {
maxPageSize: 20,
maxOffset: 100000,
maxPageNum: 5000,
defaultItemPageSize: 5,
maxItemOffset: 100000,
maxItemPageSize: 20
} as const;
/* ============================================================================
* API: 获取知识库训练错误列表（分页）
* Route: POST /api/core/dataset/training/getDatasetTrainingError
* ============================================================================ */
export const GetDatasetTrainingErrorBodySchema = PaginationSchema.extend({
pageSize: z.coerce
.number()
.int()
.min(1)
.max(DatasetTrainingErrorPaginationLimits.maxPageSize)
.optional()
.meta({
example: 10,
description: `每页集合数量，范围 [1, ${DatasetTrainingErrorPaginationLimits.maxPageSize}]`
}),
offset: z.coerce
.number()
.int()
.min(0)
.max(DatasetTrainingErrorPaginationLimits.maxOffset)
.optional()
.meta({
example: 0,
description: `集合分页偏移量，范围 [0, ${DatasetTrainingErrorPaginationLimits.maxOffset}]`
}),
pageNum: z.coerce
.number()
.int()
.min(1)
.max(DatasetTrainingErrorPaginationLimits.maxPageNum)
.optional()
.meta({
example: 1,
description: `集合分页页码，范围 [1, ${DatasetTrainingErrorPaginationLimits.maxPageNum}]`
}),
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
}),
collectionId: ObjectIdSchema.optional().meta({
example: '68ad85a7463006c963799a06',
description: '集合 ID。传入时只分页加载该集合内的异常 chunk'
}),
itemOffset: z.coerce
.number()
.int()
.min(0)
.max(DatasetTrainingErrorPaginationLimits.maxItemOffset)
.optional()
.meta({
example: 5,
description: `集合内异常 chunk 偏移量，用于加载更多，范围 [0, ${DatasetTrainingErrorPaginationLimits.maxItemOffset}]`
}),
itemPageSize: z.coerce
.number()
.int()
.min(1)
.max(DatasetTrainingErrorPaginationLimits.maxItemPageSize)
.optional()
.meta({
example: DatasetTrainingErrorPaginationLimits.defaultItemPageSize,
description: `每个集合返回的异常 chunk 数量，范围 [1, ${DatasetTrainingErrorPaginationLimits.maxItemPageSize}]`
})
});
export type GetDatasetTrainingErrorBody = z.infer<typeof GetDatasetTrainingErrorBodySchema>;
export const GetDatasetTrainingErrorResponseSchema =
PaginationResponseSchema(TrainingErrorGroupSchema);
export type GetDatasetTrainingErrorResponse = z.infer<typeof GetDatasetTrainingErrorResponseSchema>;
/* ============================================================================
* API: 检查知识库是否存在训练错误
* Route: GET /api/core/dataset/training/hasDatasetTrainingError
* ============================================================================ */
export const HasDatasetTrainingErrorQuerySchema = z.object({
datasetId: ObjectIdSchema.meta({
example: '68ad85a7463006c963799a05',
description: '知识库 ID'
})
});
export type HasDatasetTrainingErrorQuery = z.infer<typeof HasDatasetTrainingErrorQuerySchema>;
export const HasDatasetTrainingErrorResponseSchema = z.object({
hasError: z.boolean().meta({
example: true,
description: '知识库内是否存在最终/阻塞异常训练记录'
})
});
export type HasDatasetTrainingErrorResponse = z.infer<typeof HasDatasetTrainingErrorResponseSchema>;
/* ============================================================================ /* ============================================================================
* API: 获取数据集训练队列状态 * API: 获取数据集训练队列状态
* Route: GET /api/core/dataset/training/getDatasetTrainingQueue * Route: GET /api/core/dataset/training/getDatasetTrainingQueue
......
packages/global/openapi/core/dataset/training/index.ts
...@@ -6,14 +6,17 @@ import { ...@@ -6,14 +6,17 @@ import {
DeleteTrainingDataBodySchema, DeleteTrainingDataBodySchema,
GetTrainingDataDetailBodySchema, GetTrainingDataDetailBodySchema,
GetTrainingErrorBodySchema, GetTrainingErrorBodySchema,
GetDatasetTrainingErrorBodySchema,
HasDatasetTrainingErrorQuerySchema,
GetDatasetTrainingQueueQuerySchema GetDatasetTrainingQueueQuerySchema
} from './api'; } from './api';
export const DatasetTrainingPath: OpenAPIPath = { export const DatasetTrainingPath: OpenAPIPath = {
'/core/dataset/training/updateTrainingData': { '/core/dataset/training/updateTrainingData': {
put: { put: {
summary: '更新训练数据', summary: '更新训练数据或重试训练异常',
description: '更新单条训练数据，或批量重试集合内所有错误数据（不传 dataId）', description:
'更新或重试训练异常：传 dataId 时重试或编辑后重训单个 chunk；不传 dataId 时，可按 collectionId 或 datasetId 重试全部最终/阻塞异常',
tags: [TagsMap.datasetTraining], tags: [TagsMap.datasetTraining],
requestBody: { requestBody: {
content: { content: {
...@@ -92,8 +95,8 @@ export const DatasetTrainingPath: OpenAPIPath = { ...@@ -92,8 +95,8 @@ export const DatasetTrainingPath: OpenAPIPath = {
'/core/dataset/training/getTrainingError': { '/core/dataset/training/getTrainingError': {
post: { post: {
summary: '获取训练错误列表', summary: '获取集合训练错误列表',
description: '分页查询集合内训练失败的数据列表', description: '分页查询集合内最终/阻塞异常训练记录',
tags: [TagsMap.datasetTraining], tags: [TagsMap.datasetTraining],
requestBody: { requestBody: {
content: { content: {
...@@ -110,6 +113,43 @@ export const DatasetTrainingPath: OpenAPIPath = { ...@@ -110,6 +113,43 @@ export const DatasetTrainingPath: OpenAPIPath = {
} }
}, },
'/core/dataset/training/getDatasetTrainingError': {
post: {
summary: '获取知识库训练错误列表',
description:
'分页查询知识库内存在最终/阻塞异常的集合，并返回每个集合内的部分异常 chunk；传 collectionId 时加载该集合更多异常 chunk',
tags: [TagsMap.datasetTraining],
requestBody: {
content: {
'application/json': {
schema: GetDatasetTrainingErrorBodySchema
}
}
},
responses: {
200: {
description: '成功返回知识库错误数据分页列表'
}
}
}
},
'/core/dataset/training/hasDatasetTrainingError': {
get: {
summary: '检查知识库是否存在训练错误',
description: '轻量检查知识库内是否存在最终/阻塞异常训练记录，用于列表徽章状态探测',
tags: [TagsMap.datasetTraining],
requestParams: {
query: HasDatasetTrainingErrorQuerySchema
},
responses: {
200: {
description: '成功返回是否存在训练错误'
}
}
}
},
'/core/dataset/training/getDatasetTrainingQueue': { '/core/dataset/training/getDatasetTrainingQueue': {
get: { get: {
summary: '获取训练队列状态', summary: '获取训练队列状态',
......
packages/service/core/dataset/training/constants.ts deleted 100644 → 0
export enum ImportDataSourceEnum {
fileLocal = 'fileLocal',
fileLink = 'fileLink',
fileCustom = 'fileCustom'
}
packages/service/core/dataset/training/controller.ts
...@@ -12,23 +12,54 @@ import { getLLMMaxChunkSize } from '../../../../global/core/dataset/training/uti ...@@ -12,23 +12,54 @@ import { getLLMMaxChunkSize } from '../../../../global/core/dataset/training/uti
import { retryFn } from '@fastgpt/global/common/system/utils'; import { retryFn } from '@fastgpt/global/common/system/utils';
import { getLogger, LogCategories } from '../../../common/logger'; import { getLogger, LogCategories } from '../../../common/logger';
import { checkTimerLock, deleteTimerLock } from '../../../common/system/timerLock/utils'; import { checkTimerLock, deleteTimerLock } from '../../../common/system/timerLock/utils';
import { BLOCKED_LOCK_TIME } from './query';
const logger = getLogger(LogCategories.MODULE.DATASET.TRAINING); const logger = getLogger(LogCategories.MODULE.DATASET.TRAINING);
export const lockTrainingDataByTeamId = async (teamId: string): Promise<any> => { export const lockTrainingDataByTeamId = async (
teamId: string,
currentTrainingId?: string
): Promise<any> => {
const timerId = `lock_training_data--${teamId}`; const timerId = `lock_training_data--${teamId}`;
const errorMsg = i18nT('common:code_error.team_error.ai_points_not_enough');
const lockCurrentTraining = () => {
if (!currentTrainingId) return Promise.resolve();
return MongoDatasetTraining.updateOne(
{
teamId,
_id: currentTrainingId
},
{
lockTime: BLOCKED_LOCK_TIME,
errorMsg
}
);
};
// 5 分钟闸门：并发/多节点调用时，只有首个抢到锁的会执行；TTL 作为兜底 // 5 分钟闸门：并发/多节点调用时，只有首个抢到锁的会执行；TTL 作为兜底
const acquired = await checkTimerLock({ timerId, lockMinuted: 30 }); const acquired = await checkTimerLock({ timerId, lockMinuted: 30 });
if (!acquired) return; if (!acquired) {
// 其它 worker 已在执行团队级锁定时，当前已领取任务仍需要单独标记，避免最后一次重试被扣到 0 后不可见。
await lockCurrentTraining().catch((error) => {
logger.error('lock current training data failed', { teamId, currentTrainingId, error });
});
return;
}
try { try {
await MongoDatasetTraining.updateMany( await MongoDatasetTraining.updateMany(
{ {
teamId teamId,
$or: [
{ retryCount: { $gt: 0 } },
...(currentTrainingId ? [{ _id: currentTrainingId }] : [])
]
}, },
{ {
lockTime: new Date('2999/5/5') lockTime: BLOCKED_LOCK_TIME,
errorMsg
} }
); );
} catch (error) { } catch (error) {
......
packages/service/core/dataset/training/query.ts 0 → 100644
import {
CollectionTrainingStatusEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import type { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
type TrainingStatusCount = {
activeCount: number;
finalErrorCount: number;
};
export const BLOCKED_LOCK_TIME = new Date('2050-01-01');
export const trainingModeRankMap: Record<TrainingModeEnum, number> = {
[TrainingModeEnum.parse]: 0,
[TrainingModeEnum.imageParse]: 1,
[TrainingModeEnum.qa]: 2,
[TrainingModeEnum.image]: 3,
[TrainingModeEnum.auto]: 4,
[TrainingModeEnum.chunk]: 5
};
export const trainingModeRanks = Object.values(TrainingModeEnum).map((mode) => ({
mode,
rank: trainingModeRankMap[mode]
}));
const trimmedErrorMsgExpr = (fieldPath = '$errorMsg') => ({
$trim: {
input: {
$ifNull: [fieldPath, '']
}
}
});
/**
* 判断训练记录是否有有效错误信息。空字符串和纯空白字符串都视为无错误，
* 避免自动重试或历史脏数据被错误计入最终异常。
*/
export const hasEffectiveErrorMsg = (training?: Pick<DatasetTrainingSchemaType, 'errorMsg'>) => {
return typeof training?.errorMsg === 'string' && training.errorMsg.trim() !== '';
};
/**
* active 表示仍可能被训练队列继续处理的剩余任务，包含普通排队/训练中和自动重试中。
* 这里不判断 lockTime 是否已经到达队列可消费时间，只判断未被永久锁定。
*/
export const isActiveTraining = (
training?: Pick<DatasetTrainingSchemaType, 'retryCount' | 'lockTime'>
) => {
return (training?.retryCount ?? 0) > 0 && new Date(training?.lockTime ?? 0) < BLOCKED_LOCK_TIME;
};
export const isTemporarilyFailedTraining = (
training?: Pick<DatasetTrainingSchemaType, 'retryCount' | 'lockTime' | 'errorMsg'>
) => {
return hasEffectiveErrorMsg(training) && isActiveTraining(training);
};
export const isFinalErrorTraining = (
training?: Pick<DatasetTrainingSchemaType, 'retryCount' | 'lockTime' | 'errorMsg'>
) => {
return (
hasEffectiveErrorMsg(training) &&
((training?.retryCount ?? 0) <= 0 || new Date(training?.lockTime ?? 0) >= BLOCKED_LOCK_TIME)
);
};
export const isRemainingTraining = (
training?: Pick<DatasetTrainingSchemaType, 'retryCount' | 'lockTime' | 'errorMsg'>
) => {
return isActiveTraining(training) || isFinalErrorTraining(training);
};
export const hasEffectiveErrorMsgExpr = { $gt: [{ $strLenCP: trimmedErrorMsgExpr() }, 0] };
export const activeTrainingExpr = {
$and: [{ $gt: ['$retryCount', 0] }, { $lt: ['$lockTime', BLOCKED_LOCK_TIME] }]
};
export const finalErrorTrainingExpr = {
$and: [
hasEffectiveErrorMsgExpr,
{
$or: [{ $lte: ['$retryCount', 0] }, { $gte: ['$lockTime', BLOCKED_LOCK_TIME] }]
}
]
};
export const remainingTrainingExpr = {
$or: [activeTrainingExpr, finalErrorTrainingExpr]
};
export const hasEffectiveErrorMsgMatch = {
$expr: hasEffectiveErrorMsgExpr
};
export const activeTrainingMatch = {
retryCount: { $gt: 0 },
lockTime: { $lt: BLOCKED_LOCK_TIME }
};
export const finalErrorTrainingMatch = {
$expr: finalErrorTrainingExpr
};
export const remainingTrainingMatch = {
$or: [activeTrainingMatch, finalErrorTrainingMatch]
};
/**
* rank 越小表示流程越早；collection 的“最慢阶段”就是剩余任务里流程最早的阶段。
*/
export const getTrainingModeRank = (mode?: TrainingModeEnum) => {
if (!mode) return Number.MAX_SAFE_INTEGER;
return trainingModeRankMap[mode] ?? Number.MAX_SAFE_INTEGER;
};
/**
* 返回流程中更早的训练阶段，用于计算用户感知上的“最慢阶段”。
*/
export const compareTrainingModeBySlowest = (a?: TrainingModeEnum, b?: TrainingModeEnum) => {
return getTrainingModeRank(a) - getTrainingModeRank(b);
};
export const getSlowestTrainingMode = (modes: Array<TrainingModeEnum | undefined>) => {
return modes.filter(Boolean).sort((a, b) => compareTrainingModeBySlowest(a, b))[0] as
| TrainingModeEnum
| undefined;
};
/**
* 根据各阶段 active/final error 数量计算 collection 级最慢阶段状态。
* 最慢阶段只有最终异常时才展示 error。
*/
export const getSlowestTrainingStatus = (
modeCounts: Partial<Record<TrainingModeEnum, TrainingStatusCount>>
) => {
const slowestTrainingMode = getSlowestTrainingMode(
Object.entries(modeCounts)
.filter(([, count]) => (count?.activeCount ?? 0) + (count?.finalErrorCount ?? 0) > 0)
.map(([mode]) => mode as TrainingModeEnum)
);
if (!slowestTrainingMode) {
return {
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
}
const slowestCounts = modeCounts[slowestTrainingMode];
return {
slowestTrainingMode,
slowestTrainingStatus:
(slowestCounts?.activeCount ?? 0) > 0
? CollectionTrainingStatusEnum.running
: CollectionTrainingStatusEnum.error
};
};
packages/service/core/dataset/training/schema.ts
...@@ -11,7 +11,6 @@ import { ...@@ -11,7 +11,6 @@ import {
} from '@fastgpt/global/support/user/team/constant'; } from '@fastgpt/global/support/user/team/constant';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants'; import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/data/constants';
import { DatasetDataCollectionName } from '../data/schema'; import { DatasetDataCollectionName } from '../data/schema';
import { getLogger, LogCategories } from '../../../common/logger';
export const DatasetTrainingCollectionName = 'dataset_trainings'; export const DatasetTrainingCollectionName = 'dataset_trainings';
...@@ -121,6 +120,12 @@ TrainingDataSchema.virtual('data', { ...@@ -121,6 +120,12 @@ TrainingDataSchema.virtual('data', {
// lock training data(teamId); delete training data // lock training data(teamId); delete training data
TrainingDataSchema.index({ teamId: 1, datasetId: 1 }); TrainingDataSchema.index({ teamId: 1, datasetId: 1 });
// collection 级状态、错误列表、删除、详情
TrainingDataSchema.index({
teamId: 1,
datasetId: 1,
collectionId: 1
});
// get training data and sort // get training data and sort
TrainingDataSchema.index({ mode: 1, retryCount: 1, lockTime: 1, weight: -1 }); TrainingDataSchema.index({ mode: 1, retryCount: 1, lockTime: 1, weight: -1 });
TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 * 60 }); // 7 days TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 * 60 }); // 7 days
......
packages/service/support/permission/dataset/auth.ts
...@@ -138,7 +138,6 @@ export const authDataset = async ({ ...@@ -138,7 +138,6 @@ export const authDataset = async ({
export async function authDatasetCollection({ export async function authDatasetCollection({
collectionId, collectionId,
per = NullPermissionVal, per = NullPermissionVal,
isRoot = false,
...props ...props
}: AuthModeType & { }: AuthModeType & {
collectionId: string; collectionId: string;
......
packages/service/test/core/dataset/training/controller.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import { i18nT } from '@fastgpt/global/common/i18n/utils';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { lockTrainingDataByTeamId } from '@fastgpt/service/core/dataset/training/controller';
import {
BLOCKED_LOCK_TIME,
finalErrorTrainingMatch,
isFinalErrorTraining
} from '@fastgpt/service/core/dataset/training/query';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { getRootUser } from '@test/datas/users';
describe('dataset training controller', () => {
it('should lock retryable team trainings with AI points error message', async () => {
const root = await getRootUser();
const otherRoot = await getRootUser();
const datasetId = '507f1f77bcf86cd799439011';
const collectionId = '507f1f77bcf86cd799439012';
const billId = 'test';
const [retryable, exhausted, otherTeam] = await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 3
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 0
},
{
teamId: otherRoot.teamId,
tmbId: otherRoot.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 3
}
]);
await lockTrainingDataByTeamId(String(root.teamId));
const lockedTraining = await MongoDatasetTraining.findById(retryable._id).lean();
const exhaustedTraining = await MongoDatasetTraining.findById(exhausted._id).lean();
const otherTeamTraining = await MongoDatasetTraining.findById(otherTeam._id).lean();
const finalErrorLockedTrainingCount = await MongoDatasetTraining.countDocuments({
_id: retryable._id,
...finalErrorTrainingMatch
});
const errorMsg = i18nT('common:code_error.team_error.ai_points_not_enough');
expect(lockedTraining?.lockTime).toEqual(BLOCKED_LOCK_TIME);
expect(lockedTraining?.errorMsg).toBe(errorMsg);
expect(finalErrorLockedTrainingCount).toBe(1);
expect(
isFinalErrorTraining({
retryCount: lockedTraining?.retryCount,
lockTime: lockedTraining?.lockTime,
errorMsg: lockedTraining?.errorMsg
})
).toBe(true);
expect(exhaustedTraining?.lockTime).not.toEqual(BLOCKED_LOCK_TIME);
expect(exhaustedTraining?.errorMsg).toBeUndefined();
expect(otherTeamTraining?.lockTime).not.toEqual(BLOCKED_LOCK_TIME);
expect(otherTeamTraining?.errorMsg).toBeUndefined();
});
it('should lock the current picked training even when retry count is exhausted', async () => {
const root = await getRootUser();
const datasetId = '507f1f77bcf86cd799439021';
const collectionId = '507f1f77bcf86cd799439022';
const billId = 'test';
const [pickedTraining, exhaustedHistory] = await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId,
collectionId,
billId,
mode: TrainingModeEnum.chunk,
retryCount: 0
}
]);
await lockTrainingDataByTeamId(String(root.teamId), String(pickedTraining._id));
const lockedTraining = await MongoDatasetTraining.findById(pickedTraining._id).lean();
const untouchedTraining = await MongoDatasetTraining.findById(exhaustedHistory._id).lean();
const errorMsg = i18nT('common:code_error.team_error.ai_points_not_enough');
expect(lockedTraining?.lockTime).toEqual(BLOCKED_LOCK_TIME);
expect(lockedTraining?.errorMsg).toBe(errorMsg);
expect(untouchedTraining?.lockTime).not.toEqual(BLOCKED_LOCK_TIME);
expect(untouchedTraining?.errorMsg).toBeUndefined();
});
});
packages/service/test/core/dataset/training/query.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import {
CollectionTrainingStatusEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import {
BLOCKED_LOCK_TIME,
compareTrainingModeBySlowest,
getSlowestTrainingStatus,
getTrainingModeRank,
hasEffectiveErrorMsg,
isActiveTraining,
isFinalErrorTraining,
isRemainingTraining,
isTemporarilyFailedTraining
} from '@fastgpt/service/core/dataset/training/query';
describe('dataset training query helpers', () => {
it('treats empty or whitespace errorMsg as ineffective', () => {
expect(hasEffectiveErrorMsg({ errorMsg: undefined })).toBe(false);
expect(hasEffectiveErrorMsg({ errorMsg: '' })).toBe(false);
expect(hasEffectiveErrorMsg({ errorMsg: ' \n\t' })).toBe(false);
expect(hasEffectiveErrorMsg({ errorMsg: 'failed' })).toBe(true);
});
it('separates active, temporary failed and final error records', () => {
const active = {
retryCount: 1,
lockTime: new Date('2049-12-31'),
errorMsg: ''
};
const temporaryFailed = {
retryCount: 1,
lockTime: new Date('2049-12-31'),
errorMsg: ' failed '
};
const retryExhausted = {
retryCount: 0,
lockTime: new Date('2049-12-31'),
errorMsg: 'failed'
};
const permanentlyLocked = {
retryCount: 3,
lockTime: BLOCKED_LOCK_TIME,
errorMsg: 'failed'
};
expect(isActiveTraining(active)).toBe(true);
expect(isRemainingTraining(active)).toBe(true);
expect(isActiveTraining(temporaryFailed)).toBe(true);
expect(isTemporarilyFailedTraining(temporaryFailed)).toBe(true);
expect(isFinalErrorTraining(temporaryFailed)).toBe(false);
expect(isActiveTraining(retryExhausted)).toBe(false);
expect(isFinalErrorTraining(retryExhausted)).toBe(true);
expect(isRemainingTraining(retryExhausted)).toBe(true);
expect(isActiveTraining(permanentlyLocked)).toBe(false);
expect(isFinalErrorTraining(permanentlyLocked)).toBe(true);
});
it('orders modes by the earliest remaining stage as slowest', () => {
expect(getTrainingModeRank(TrainingModeEnum.parse)).toBeLessThan(
getTrainingModeRank(TrainingModeEnum.imageParse)
);
expect(getTrainingModeRank(TrainingModeEnum.imageParse)).toBeLessThan(
getTrainingModeRank(TrainingModeEnum.qa)
);
expect(
compareTrainingModeBySlowest(TrainingModeEnum.image, TrainingModeEnum.chunk)
).toBeLessThan(0);
});
it('returns running when the slowest stage still has active records', () => {
const status = getSlowestTrainingStatus({
[TrainingModeEnum.parse]: { activeCount: 1, finalErrorCount: 0 },
[TrainingModeEnum.chunk]: { activeCount: 0, finalErrorCount: 2 }
});
expect(status).toEqual({
slowestTrainingMode: TrainingModeEnum.parse,
slowestTrainingStatus: CollectionTrainingStatusEnum.running
});
});
it('returns error only when the slowest stage has final errors and no active records', () => {
const status = getSlowestTrainingStatus({
[TrainingModeEnum.image]: { activeCount: 0, finalErrorCount: 1 },
[TrainingModeEnum.chunk]: { activeCount: 3, finalErrorCount: 0 }
});
expect(status).toEqual({
slowestTrainingMode: TrainingModeEnum.image,
slowestTrainingStatus: CollectionTrainingStatusEnum.error
});
});
it('returns ready when there are no remaining records', () => {
expect(getSlowestTrainingStatus({})).toEqual({
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
});
});
});
packages/web/components/common/Tag/index.tsx
...@@ -4,10 +4,13 @@ import { Box, type BoxProps, Flex, type FlexProps } from '@chakra-ui/react'; ...@@ -4,10 +4,13 @@ import { Box, type BoxProps, Flex, type FlexProps } from '@chakra-ui/react';
export type ColorSchemaType = export type ColorSchemaType =
| 'white' | 'white'
| 'blue' | 'blue'
| 'cyan'
| 'green' | 'green'
| 'red' | 'red'
| 'lightRed'
| 'yellow' | 'yellow'
| 'gray' | 'gray'
| 'lightGray'
| 'purple' | 'purple'
| 'adora'; | 'adora';
...@@ -57,6 +60,21 @@ const colorMap: Record< ...@@ -57,6 +60,21 @@ const colorMap: Record<
bg: 'primary.50', bg: 'primary.50',
color: 'primary.600' color: 'primary.600'
}, },
cyan: {
borderColor: '#BCE7FF',
bg: '#F0FBFF',
color: '#0884DD'
},
lightRed: {
borderColor: 'red.200',
bg: 'red.50',
color: 'red.500'
},
lightGray: {
borderColor: 'myGray.200',
bg: 'myGray.100',
color: 'myGray.500'
},
purple: { purple: {
borderColor: '#ECF', borderColor: '#ECF',
bg: '#F6EEFA', bg: '#F6EEFA',
......
packages/web/hooks/useScrollPagination.tsx
import React, { type ReactNode, type RefObject, useMemo, useRef, useState } from 'react'; import React, { type ReactNode, type RefObject, useRef, useState } from 'react';
import { Box, type BoxProps } from '@chakra-ui/react'; import { Box, type BoxProps } from '@chakra-ui/react';
import { useToast } from './useToast'; import { useToast } from './useToast';
import { getErrText } from '@fastgpt/global/common/error/utils'; import { getErrText } from '@fastgpt/global/common/error/utils';
...@@ -190,6 +190,7 @@ export function useScrollPagination< ...@@ -190,6 +190,7 @@ export function useScrollPagination<
EmptyTip, EmptyTip,
showErrorToast = true, showErrorToast = true,
disabled = false, disabled = false,
showNoMoreTip = true,
...props ...props
}: { }: {
...@@ -200,6 +201,7 @@ export function useScrollPagination< ...@@ -200,6 +201,7 @@ export function useScrollPagination<
EmptyTip?: React.JSX.Element; EmptyTip?: React.JSX.Element;
showErrorToast?: boolean; showErrorToast?: boolean;
disabled?: boolean; disabled?: boolean;
showNoMoreTip?: boolean;
} & Parameters<typeof useRequest>[1] } & Parameters<typeof useRequest>[1]
) { ) {
const { t } = useTranslation(); const { t } = useTranslation();
...@@ -286,7 +288,7 @@ export function useScrollPagination< ...@@ -286,7 +288,7 @@ export function useScrollPagination<
} }
); );
let ScrollRef = useRef<HTMLDivElement>(null); const ScrollRef = useRef<HTMLDivElement>(null);
const ScrollData = useMemoizedFn( const ScrollData = useMemoizedFn(
({ ({
children, children,
...@@ -299,11 +301,11 @@ export function useScrollPagination< ...@@ -299,11 +301,11 @@ export function useScrollPagination<
ScrollContainerRef?: RefObject<HTMLDivElement>; ScrollContainerRef?: RefObject<HTMLDivElement>;
} & BoxProps) => { } & BoxProps) => {
const ref = ScrollContainerRef || ScrollRef; const ref = ScrollContainerRef || ScrollRef;
const loadText = useMemo(() => { const loadText = (() => {
if (isLoading || isLoadingProp) return t('common:is_requesting'); if (isLoading || isLoadingProp) return t('common:is_requesting');
if (noMore) return t('common:request_end'); if (noMore) return t('common:request_end');
return t('common:request_more'); return t('common:request_more');
}, [isLoading, noMore]); })();
const scroll = useScroll(ref); const scroll = useScroll(ref);
...@@ -339,7 +341,7 @@ export function useScrollPagination< ...@@ -339,7 +341,7 @@ export function useScrollPagination<
</Box> </Box>
)} )}
{children} {children}
{scrollLoadType === 'bottom' && !isEmpty && ( {scrollLoadType === 'bottom' && !isEmpty && (showNoMoreTip || !noMore) && (
<Box <Box
mt={2} mt={2}
fontSize={'xs'} fontSize={'xs'}
......
packages/web/i18n/en/dataset.json
...@@ -201,5 +201,9 @@ ...@@ -201,5 +201,9 @@
"website_info": "Website Information", "website_info": "Website Information",
"yuque_dataset": "Yuque Knowledge Base", "yuque_dataset": "Yuque Knowledge Base",
"yuque_dataset_config": "Configure Yuque Knowledge Base", "yuque_dataset_config": "Configure Yuque Knowledge Base",
"yuque_dataset_desc": "Build knowledge base using Yuque documents by configuring document permissions, documents will not be stored twice" "yuque_dataset_desc": "Build knowledge base using Yuque documents by configuring document permissions, documents will not be stored twice",
"training.stage_error": "{{stage}} error",
"training.status_error": "Training error",
"training_error_list": "Error list",
"training_error_empty": "No training errors"
} }
packages/web/i18n/zh-CN/dataset.json
...@@ -201,5 +201,9 @@ ...@@ -201,5 +201,9 @@
"website_info": "网站信息", "website_info": "网站信息",
"yuque_dataset": "语雀知识库", "yuque_dataset": "语雀知识库",
"yuque_dataset_config": "配置语雀知识库", "yuque_dataset_config": "配置语雀知识库",
"yuque_dataset_desc": "可通过配置语雀文档权限，使用语雀文档构建知识库，文档不会进行二次存储" "yuque_dataset_desc": "可通过配置语雀文档权限，使用语雀文档构建知识库，文档不会进行二次存储",
"training.stage_error": "{{stage}}异常",
"training.status_error": "训练异常",
"training_error_list": "异常列表",
"training_error_empty": "暂无训练异常"
} }
packages/web/i18n/zh-Hant/dataset.json
...@@ -201,5 +201,9 @@ ...@@ -201,5 +201,9 @@
"website_info": "網站資訊", "website_info": "網站資訊",
"yuque_dataset": "語雀知識庫", "yuque_dataset": "語雀知識庫",
"yuque_dataset_config": "設定語雀知識庫", "yuque_dataset_config": "設定語雀知識庫",
"yuque_dataset_desc": "可透過設定語雀文件權限，使用語雀文件建構知識庫，文件不會進行二次儲存" "yuque_dataset_desc": "可透過設定語雀文件權限，使用語雀文件建構知識庫，文件不會進行二次儲存",
"training.stage_error": "{{stage}}異常",
"training.status_error": "訓練異常",
"training_error_list": "異常列表",
"training_error_empty": "暫無訓練異常"
} }
projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx
...@@ -40,7 +40,15 @@ const FileSourceSelector = dynamic(() => import('../Import/components/FileSource ...@@ -40,7 +40,15 @@ const FileSourceSelector = dynamic(() => import('../Import/components/FileSource
const BackupImportModal = dynamic(() => import('./BackupImportModal')); const BackupImportModal = dynamic(() => import('./BackupImportModal'));
const TemplateImportModal = dynamic(() => import('./TemplateImportModal')); const TemplateImportModal = dynamic(() => import('./TemplateImportModal'));
const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => { const Header = ({
hasTrainingData,
hasTrainingError,
onOpenTrainingErrorModal
}: {
hasTrainingData: boolean;
hasTrainingError: boolean;
onOpenTrainingErrorModal: () => void;
}) => {
const { t } = useTranslation(); const { t } = useTranslation();
const { feConfigs } = useSystemStore(); const { feConfigs } = useSystemStore();
const { isPc } = useSystem(); const { isPc } = useSystem();
...@@ -196,6 +204,23 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => { ...@@ -196,6 +204,23 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
{datasetDetail.type !== DatasetTypeEnum.websiteDataset && {datasetDetail.type !== DatasetTypeEnum.websiteDataset &&
datasetDetail.permission.hasWritePer && datasetDetail.permission.hasWritePer &&
feConfigs?.isPlus && <HeaderTagPopOver />} feConfigs?.isPlus && <HeaderTagPopOver />}
{hasTrainingError && (
<Button
variant={'whiteBase'}
h={'36px'}
px={'14px'}
color={'#F97066'}
borderColor={'#F97066'}
_hover={{
color: '#F97066',
borderColor: '#F97066'
}}
onClick={onOpenTrainingErrorModal}
>
{t('dataset:training_error_list')}
</Button>
)}
</HStack> </HStack>
{/* diff collection button */} {/* diff collection button */}
......
projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingErrorEditView.tsx 0 → 100644
import { Box, Button, Flex } from '@chakra-ui/react';
import type { GetTrainingDataDetailResponse } from '@fastgpt/global/openapi/core/dataset/training/api';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import { useTranslation } from 'next-i18next';
import MyTextarea from '@/components/common/Textarea/MyTextarea';
import MyImage from '@/components/MyImage';
import { useForm } from 'react-hook-form';
const TrainingErrorEditView = ({
loading,
editChunk,
onCancel,
onSave
}: {
loading: boolean;
editChunk: GetTrainingDataDetailResponse;
onCancel: () => void;
onSave: (data: { q: string; a?: string; chunkIndex?: number }) => void;
}) => {
const { t } = useTranslation();
const { register, handleSubmit } = useForm({
defaultValues: {
q: editChunk?.q || '',
a: editChunk?.a || ''
}
});
return (
<Flex flexDirection={'column'} gap={4}>
{editChunk?.imagePreviewUrl && (
<Box>
<FormLabel>{t('file:image')}</FormLabel>
<Box w={'100%'} h={'200px'} border={'base'} borderRadius={'md'}>
<MyImage src={editChunk.imagePreviewUrl} alt="image" w={'100%'} h={'100%'} />
</Box>
</Box>
)}
<Box>
{(editChunk?.a || editChunk?.imagePreviewUrl) && (
<FormLabel>
{editChunk?.a
? t('common:dataset_data_input_chunk_content')
: t('common:dataset_data_input_q')}
</FormLabel>
)}
<MyTextarea
{...register('q', { required: true })}
minH={editChunk?.a || editChunk?.imagePreviewUrl ? 200 : 400}
/>
</Box>
{editChunk?.a && (
<Box>
<Box>{t('common:dataset_data_input_a')}</Box>
<MyTextarea {...register('a')} minH={200} />
</Box>
)}
<Flex justifyContent={'flex-end'} gap={4}>
<Button variant={'outline'} onClick={onCancel}>
{t('common:Cancel')}
</Button>
<Button isLoading={loading} variant={'primary'} onClick={handleSubmit(onSave)}>
{t('common:Confirm')}
</Button>
</Flex>
</Flex>
);
};
export default TrainingErrorEditView;
projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingErrorModal.tsx 0 → 100644
import MyModal from '@fastgpt/web/components/v2/common/MyModal';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type';
import { useTranslation } from 'next-i18next';
import TrainingErrorList from './TrainingErrorList';
const TrainingErrorModal = ({
datasetId,
permission,
onClose,
onRefresh
}: {
datasetId: string;
permission: DatasetItemType['permission'];
onClose: () => void;
onRefresh?: () => void;
}) => {
const { t } = useTranslation();
return (
<MyModal
isOpen
onClose={onClose}
title={t('dataset:training_error_list')}
size={'lg'}
isCentered
borderRadius={'10px'}
sx={{
'.chakra-modal__close-btn': {
top: '8px',
right: '8px',
w: '36px',
h: '36px'
}
}}
>
<TrainingErrorList
scope={{ type: 'dataset', datasetId }}
permission={permission}
onClose={onClose}
onRefresh={onRefresh}
showFooter
/>
</MyModal>
);
};
export default TrainingErrorModal;
projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx
...@@ -52,6 +52,13 @@ import TagsPopOver from './TagsPopOver'; ...@@ -52,6 +52,13 @@ import TagsPopOver from './TagsPopOver';
import { useSystemStore } from '@/web/common/system/useSystemStore'; import { useSystemStore } from '@/web/common/system/useSystemStore';
import TrainingStates from './TrainingStates'; import TrainingStates from './TrainingStates';
import { useTableMultipleSelect } from '@fastgpt/web/hooks/useTableMultipleSelect'; import { useTableMultipleSelect } from '@fastgpt/web/hooks/useTableMultipleSelect';
import {
getCollectionTrainingStatusColorSchema,
getCollectionTrainingStatusText
} from '@/web/core/dataset/trainingStatus';
import TrainingErrorModal from './TrainingErrorModal';
import type { DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api';
import { hasDatasetTrainingError as checkDatasetTrainingError } from '@/web/core/dataset/api/training';
const Header = dynamic(() => import('./Header')); const Header = dynamic(() => import('./Header'));
const EmptyCollectionTip = dynamic(() => import('./EmptyCollectionTip')); const EmptyCollectionTip = dynamic(() => import('./EmptyCollectionTip'));
...@@ -66,7 +73,10 @@ const CollectionCard = () => { ...@@ -66,7 +73,10 @@ const CollectionCard = () => {
const [trainingStatesCollection, setTrainingStatesCollection] = useState<{ const [trainingStatesCollection, setTrainingStatesCollection] = useState<{
collectionId: string; collectionId: string;
permission: DatasetCollectionsListItemType['permission'];
}>(); }>();
const [isTrainingErrorModalOpen, setIsTrainingErrorModalOpen] = useState(false);
const [hasDatasetTrainingError, setHasDatasetTrainingError] = useState(false);
const { collections, Pagination, total, getData, isGetting, pageNum, pageSize } = const { collections, Pagination, total, getData, isGetting, pageNum, pageSize } =
useContextSelector(CollectionPageContext, (v) => v); useContextSelector(CollectionPageContext, (v) => v);
...@@ -76,34 +86,17 @@ const CollectionCard = () => { ...@@ -76,34 +86,17 @@ const CollectionCard = () => {
() => () =>
collections.map((collection) => { collections.map((collection) => {
const icon = getCollectionIcon({ type: collection.type, name: collection.name }); const icon = getCollectionIcon({ type: collection.type, name: collection.name });
const status = (() => { const statusColorSchema = getCollectionTrainingStatusColorSchema(collection);
if (collection.hasError) { const statusText = getCollectionTrainingStatusText(collection);
return {
statusText: t('common:core.dataset.collection.status.error'),
colorSchema: 'red'
};
}
if (collection.trainingAmount > 0) {
return {
statusText: t('common:dataset.collections.Collection Embedding', {
total: collection.trainingAmount
}),
colorSchema: 'gray'
};
}
return {
statusText: t('common:core.dataset.collection.status.active'),
colorSchema: 'green'
};
})();
return { return {
...collection, ...collection,
icon, icon,
...status statusText,
statusColorSchema
}; };
}), }),
[collections, t] [collections]
); );
const { const {
...@@ -124,6 +117,22 @@ const CollectionCard = () => { ...@@ -124,6 +117,22 @@ const CollectionCard = () => {
const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({ const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
title: t('common:Rename') title: t('common:Rename')
}); });
const { runAsync: refreshDatasetTrainingError } = useRequest(
async () => {
const res = await checkDatasetTrainingError(datasetDetail._id);
return res.hasError;
},
{
manual: false,
refreshDeps: [datasetDetail._id],
errorToast: '',
onSuccess(hasError) {
setHasDatasetTrainingError(hasError);
}
}
);
const { runAsync: onUpdateCollection, loading: isUpdating } = useRequest( const { runAsync: onUpdateCollection, loading: isUpdating } = useRequest(
putDatasetCollectionById, putDatasetCollectionById,
{ {
...@@ -147,6 +156,7 @@ const CollectionCard = () => { ...@@ -147,6 +156,7 @@ const CollectionCard = () => {
{ {
onSuccess() { onSuccess() {
getData(pageNum); getData(pageNum);
refreshDatasetTrainingError().catch(() => undefined);
}, },
successToast: t('common:delete_success'), successToast: t('common:delete_success'),
errorToast: t('common:delete_failed') errorToast: t('common:delete_failed')
...@@ -174,12 +184,18 @@ const CollectionCard = () => { ...@@ -174,12 +184,18 @@ const CollectionCard = () => {
useRequest( useRequest(
async () => { async () => {
const shouldRefreshTrainingError =
hasTrainingData || datasetDetail.status !== DatasetStatusEnum.active;
if (datasetDetail.status !== DatasetStatusEnum.active) { if (datasetDetail.status !== DatasetStatusEnum.active) {
loadDatasetDetail(datasetDetail._id); loadDatasetDetail(datasetDetail._id);
} }
if (hasTrainingData) { if (hasTrainingData) {
getData(pageNum); getData(pageNum);
} }
if (shouldRefreshTrainingError) {
await refreshDatasetTrainingError().catch(() => undefined);
}
}, },
{ {
pollingInterval: 6000, pollingInterval: 6000,
...@@ -198,7 +214,9 @@ const CollectionCard = () => { ...@@ -198,7 +214,9 @@ const CollectionCard = () => {
parentId: targetId parentId: targetId
}); });
getData(pageNum); getData(pageNum);
} catch {} } catch {
// Drag failures are handled by the request layer toast; keep the list state unchanged here.
}
} }
}); });
...@@ -208,7 +226,11 @@ const CollectionCard = () => { ...@@ -208,7 +226,11 @@ const CollectionCard = () => {
<MyBox isLoading={isLoading} h={'100%'} py={[2, 4]} overflow={'hidden'}> <MyBox isLoading={isLoading} h={'100%'} py={[2, 4]} overflow={'hidden'}>
<Flex ref={BoxRef} flexDirection={'column'} py={[1, 0]} h={'100%'} px={[2, 6]}> <Flex ref={BoxRef} flexDirection={'column'} py={[1, 0]} h={'100%'} px={[2, 6]}>
{/* header */} {/* header */}
<Header hasTrainingData={hasTrainingData} /> <Header
hasTrainingData={hasTrainingData}
hasTrainingError={hasDatasetTrainingError}
onOpenTrainingErrorModal={() => setIsTrainingErrorModalOpen(true)}
/>
{/* collection table */} {/* collection table */}
<TableContainer mt={3} overflowY={'auto'} fontSize={'sm'} flex={'1 0 0'} h={0}> <TableContainer mt={3} overflowY={'auto'} fontSize={'sm'} flex={'1 0 0'} h={0}>
...@@ -311,16 +333,21 @@ const CollectionCard = () => { ...@@ -311,16 +333,21 @@ const CollectionCard = () => {
<MyTooltip label={t('common:Click_to_expand')}> <MyTooltip label={t('common:Click_to_expand')}>
<MyTag <MyTag
showDot showDot
colorSchema={collection.colorSchema as any} colorSchema={collection.statusColorSchema}
type={'fill'} type={'fill'}
fontSize={'mini'}
letterSpacing={'0.5px'}
onClick={(e) => { onClick={(e) => {
e.stopPropagation(); e.stopPropagation();
setTrainingStatesCollection({ collectionId: collection._id }); setTrainingStatesCollection({
collectionId: collection._id,
permission: collection.permission
});
}} }}
> >
<Flex fontWeight={'medium'} alignItems={'center'} gap={1}> <Flex fontWeight={'medium'} alignItems={'center'} gap={1}>
{t(collection.statusText as any)} {t(collection.statusText as any)}
<MyIcon name={'common/maximize'} w={'11px'} /> <MyIcon name={'common/maximize'} w={'10px'} h={'10px'} />
</Flex> </Flex>
</MyTag> </MyTag>
</MyTooltip> </MyTooltip>
...@@ -497,10 +524,23 @@ const CollectionCard = () => { ...@@ -497,10 +524,23 @@ const CollectionCard = () => {
{!!trainingStatesCollection && ( {!!trainingStatesCollection && (
<TrainingStates <TrainingStates
collectionId={trainingStatesCollection.collectionId} collectionId={trainingStatesCollection.collectionId}
permission={trainingStatesCollection.permission}
onClose={() => setTrainingStatesCollection(undefined)} onClose={() => setTrainingStatesCollection(undefined)}
/> />
)} )}
{isTrainingErrorModalOpen && (
<TrainingErrorModal
datasetId={datasetDetail._id}
permission={datasetDetail.permission}
onClose={() => setIsTrainingErrorModalOpen(false)}
onRefresh={() => {
getData(pageNum);
refreshDatasetTrainingError().catch(() => undefined);
}}
/>
)}
{!!moveCollectionData && ( {!!moveCollectionData && (
<SelectCollections <SelectCollections
datasetId={datasetDetail._id} datasetId={datasetDetail._id}
......
projects/app/src/pageComponents/dataset/detail/DataCard.tsx
...@@ -36,6 +36,10 @@ import { formatFileSize } from '@fastgpt/global/common/file/tools'; ...@@ -36,6 +36,10 @@ import { formatFileSize } from '@fastgpt/global/common/file/tools';
import MyImage from '@fastgpt/web/components/common/Image/MyImage'; import MyImage from '@fastgpt/web/components/common/Image/MyImage';
import dynamic from 'next/dynamic'; import dynamic from 'next/dynamic';
import { downloadFetch } from '@/web/common/system/utils'; import { downloadFetch } from '@/web/common/system/utils';
import {
getCollectionTrainingStatusColorSchema,
getCollectionTrainingStatusText
} from '@/web/core/dataset/trainingStatus';
const InsertImagesModal = dynamic(() => import('./data/InsertImageModal'), { const InsertImagesModal = dynamic(() => import('./data/InsertImageModal'), {
ssr: false ssr: false
...@@ -101,6 +105,13 @@ const DataCard = () => { ...@@ -101,6 +105,13 @@ const DataCard = () => {
); );
const canWrite = useMemo(() => datasetDetail.permission.hasWritePer, [datasetDetail]); const canWrite = useMemo(() => datasetDetail.permission.hasWritePer, [datasetDetail]);
const collectionTrainingStatus = useMemo(() => {
if (!collection) return;
return {
text: getCollectionTrainingStatusText(collection),
colorSchema: getCollectionTrainingStatusColorSchema(collection)
};
}, [collection]);
const [ const [
isInsertImagesModalOpen, isInsertImagesModalOpen,
...@@ -240,21 +251,19 @@ const DataCard = () => { ...@@ -240,21 +251,19 @@ const DataCard = () => {
indexAmount: collection?.indexAmount ?? '-' indexAmount: collection?.indexAmount ?? '-'
})} })}
</Box> </Box>
{!!collection?.errorCount && ( {!!collectionTrainingStatus && (
<MyTag <MyTag
colorSchema={'red'}
type={'fill'} type={'fill'}
cursor={'pointer'} cursor={'pointer'}
rounded={'full'} rounded={'full'}
ml={2} ml={2}
colorSchema={collectionTrainingStatus.colorSchema}
onClick={() => { onClick={() => {
setErrorModalId(collection._id); setErrorModalId(collection?._id || '');
}} }}
> >
<Flex fontWeight={'medium'} alignItems={'center'} gap={1}> <Flex fontWeight={'medium'} alignItems={'center'} gap={1}>
{t('dataset:data_error_amount', { {t(collectionTrainingStatus.text as any)}
errorAmount: collection?.errorCount
})}
<MyIcon name={'common/maximize'} w={'11px'} /> <MyIcon name={'common/maximize'} w={'11px'} />
</Flex> </Flex>
</MyTag> </MyTag>
...@@ -462,10 +471,11 @@ const DataCard = () => { ...@@ -462,10 +471,11 @@ const DataCard = () => {
}} }}
/> />
)} )}
{errorModalId && ( {errorModalId && collection && (
<TrainingStates <TrainingStates
defaultTab={'errors'} defaultTab={collection?.hasError ? 'errors' : 'states'}
collectionId={errorModalId} collectionId={errorModalId}
permission={collection.permission}
onClose={() => { onClose={() => {
setErrorModalId(''); setErrorModalId('');
refreshList(); refreshList();
......
projects/app/src/pages/api/core/dataset/collection/detail.ts
...@@ -9,6 +9,7 @@ import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collecti ...@@ -9,6 +9,7 @@ import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collecti
import { getVectorCount } from '@fastgpt/service/common/vectorDB/controller'; import { getVectorCount } from '@fastgpt/service/common/vectorDB/controller';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { Types } from '@fastgpt/service/common/mongo';
import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset'; import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset';
import { isS3ObjectKey } from '@fastgpt/service/common/s3/utils'; import { isS3ObjectKey } from '@fastgpt/service/common/s3/utils';
import type { ApiRequestProps } from '@fastgpt/service/type/next'; import type { ApiRequestProps } from '@fastgpt/service/type/next';
...@@ -18,6 +19,124 @@ import { ...@@ -18,6 +19,124 @@ import {
GetCollectionDetailQuerySchema, GetCollectionDetailQuerySchema,
GetCollectionDetailResponseSchema GetCollectionDetailResponseSchema
} from '@fastgpt/global/openapi/core/dataset/collection/api'; } from '@fastgpt/global/openapi/core/dataset/collection/api';
import {
activeTrainingExpr,
finalErrorTrainingExpr,
getSlowestTrainingStatus,
remainingTrainingMatch,
trainingModeRanks
} from '@fastgpt/service/core/dataset/training/query';
import { CollectionTrainingStatusEnum } from '@fastgpt/global/core/dataset/constants';
const defaultCollectionTrainingStatus = {
trainingAmount: 0,
activeTrainingAmount: 0,
finalErrorAmount: 0,
hasError: false,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
/**
* 获取数据集集合的训练状态统计信息
* @param teamId - 团队ID
* @param datasetId - 数据集ID
* @param collectionId - 集合ID
* @returns 包含训练数量、活跃训练数、错误数及最慢训练状态的统计对象
*/
const getCollectionTrainingStatus = async ({
teamId,
datasetId,
collectionId
}: {
teamId: Types.ObjectId;
datasetId: Types.ObjectId;
collectionId: Types.ObjectId;
}) => {
const [trainingStatus] = await MongoDatasetTraining.aggregate(
[
{
$match: {
teamId,
datasetId,
collectionId,
...remainingTrainingMatch
}
},
{
$addFields: {
modeRank: {
$switch: {
branches: trainingModeRanks.map(({ mode, rank }) => ({
case: { $eq: ['$mode', mode] },
then: rank
})),
default: 999
}
},
isActiveTraining: activeTrainingExpr,
isFinalErrorTraining: finalErrorTrainingExpr
}
},
{
$group: {
_id: '$mode',
modeRank: { $first: '$modeRank' },
activeCount: { $sum: { $cond: ['$isActiveTraining', 1, 0] } },
finalErrorCount: { $sum: { $cond: ['$isFinalErrorTraining', 1, 0] } },
trainingAmount: { $sum: 1 }
}
},
{
$sort: {
modeRank: 1
}
},
{
$group: {
_id: null,
trainingAmount: { $sum: '$trainingAmount' },
activeTrainingAmount: { $sum: '$activeCount' },
finalErrorAmount: { $sum: '$finalErrorCount' },
modeCounts: {
$push: {
mode: '$_id',
activeCount: '$activeCount',
finalErrorCount: '$finalErrorCount'
}
}
}
}
],
readFromSecondary
);
if (!trainingStatus) return defaultCollectionTrainingStatus;
const { slowestTrainingMode, slowestTrainingStatus } = getSlowestTrainingStatus(
Object.fromEntries(
trainingStatus.modeCounts.map(
({
mode,
activeCount,
finalErrorCount
}: {
mode: any;
activeCount: number;
finalErrorCount: number;
}) => [mode, { activeCount, finalErrorCount }]
)
)
);
return {
trainingAmount: trainingStatus.trainingAmount,
activeTrainingAmount: trainingStatus.activeTrainingAmount,
finalErrorAmount: trainingStatus.finalErrorAmount,
hasError: trainingStatus.finalErrorAmount > 0,
slowestTrainingMode,
slowestTrainingStatus
};
};
async function handler(req: ApiRequestProps): Promise<GetCollectionDetailResponseType> { async function handler(req: ApiRequestProps): Promise<GetCollectionDetailResponseType> {
const { id } = parseApiInput({ req, querySchema: GetCollectionDetailQuerySchema }).query; const { id } = parseApiInput({ req, querySchema: GetCollectionDetailQuerySchema }).query;
...@@ -36,23 +155,18 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionDetailRespons ...@@ -36,23 +155,18 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionDetailRespons
return Promise.reject('Invalid dataset file key'); return Promise.reject('Invalid dataset file key');
} }
const [file, indexAmount, errorCount] = await Promise.all([ const [file, indexAmount, trainingStatus] = await Promise.all([
fileId ? getS3DatasetSource().getFileMetadata(fileId) : undefined, fileId ? getS3DatasetSource().getFileMetadata(fileId) : undefined,
getVectorCount({ getVectorCount({
teamId: collection.teamId, teamId: collection.teamId,
datasetId: collection.datasetId, datasetId: collection.datasetId,
collectionId: collection._id collectionId: collection._id
}), }),
MongoDatasetTraining.countDocuments( getCollectionTrainingStatus({
{ teamId: new Types.ObjectId(collection.teamId),
teamId: collection.teamId, datasetId: new Types.ObjectId(collection.datasetId),
datasetId: collection.datasetId, collectionId: new Types.ObjectId(collection._id)
collectionId: id, })
errorMsg: { $exists: true },
retryCount: { $lte: 0 }
},
readFromSecondary
)
]); ]);
return GetCollectionDetailResponseSchema.parse({ return GetCollectionDetailResponseSchema.parse({
...@@ -65,7 +179,8 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionDetailRespons ...@@ -65,7 +179,8 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionDetailRespons
}), }),
permission, permission,
file, file,
errorCount ...trainingStatus,
errorCount: trainingStatus.finalErrorAmount
}); });
} }
......
projects/app/src/pages/api/core/dataset/collection/listV2.ts
...@@ -13,11 +13,64 @@ import { replaceRegChars } from '@fastgpt/global/common/string/tools'; ...@@ -13,11 +13,64 @@ import { replaceRegChars } from '@fastgpt/global/common/string/tools';
import type { ApiRequestProps } from '@fastgpt/service/type/next'; import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError'; import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { import {
activeTrainingExpr,
finalErrorTrainingExpr,
getSlowestTrainingStatus,
remainingTrainingMatch,
trainingModeRanks
} from '@fastgpt/service/core/dataset/training/query';
import {
CollectionTrainingStatusEnum,
type TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import {
ListCollectionV2BodySchema, ListCollectionV2BodySchema,
ListCollectionV2ResponseSchema, ListCollectionV2ResponseSchema,
type ListCollectionV2ResponseType type ListCollectionV2ResponseType
} from '@fastgpt/global/openapi/core/dataset/collection/api'; } from '@fastgpt/global/openapi/core/dataset/collection/api';
const defaultCollectionTrainingStatus = {
trainingAmount: 0,
activeTrainingAmount: 0,
finalErrorAmount: 0,
hasError: false,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
type TrainingAmountAggregateItem = {
_id: string;
trainingAmount: number;
activeTrainingAmount: number;
finalErrorAmount: number;
modeCounts: {
mode: TrainingModeEnum;
activeCount: number;
finalErrorCount: number;
}[];
};
const formatTrainingStatus = (item?: TrainingAmountAggregateItem) => {
if (!item) return defaultCollectionTrainingStatus;
const { slowestTrainingMode, slowestTrainingStatus } = getSlowestTrainingStatus(
Object.fromEntries(
item.modeCounts.map(({ mode, activeCount, finalErrorCount }) => [
mode,
{ activeCount, finalErrorCount }
])
)
);
return {
trainingAmount: item.trainingAmount,
activeTrainingAmount: item.activeTrainingAmount,
finalErrorAmount: item.finalErrorAmount,
hasError: item.finalErrorAmount > 0,
slowestTrainingMode,
slowestTrainingStatus
};
};
async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseType> { async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseType> {
const { const {
datasetId, datasetId,
...@@ -96,8 +149,7 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy ...@@ -96,8 +149,7 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy
tags: item.tags tags: item.tags
}), }),
dataAmount: 0, dataAmount: 0,
trainingAmount: 0, ...defaultCollectionTrainingStatus,
hasError: false,
permission permission
})) }))
), ),
...@@ -118,7 +170,7 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy ...@@ -118,7 +170,7 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy
// Compute data amount // Compute data amount
const [trainingAmount, dataAmount]: [ const [trainingAmount, dataAmount]: [
{ _id: string; count: number; hasError: boolean }[], TrainingAmountAggregateItem[],
{ _id: string; count: number }[] { _id: string; count: number }[]
] = await Promise.all([ ] = await Promise.all([
MongoDatasetTraining.aggregate( MongoDatasetTraining.aggregate(
...@@ -127,14 +179,75 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy ...@@ -127,14 +179,75 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy
$match: { $match: {
teamId: new Types.ObjectId(teamId), teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(datasetId), datasetId: new Types.ObjectId(datasetId),
collectionId: { $in: collectionIds } collectionId: { $in: collectionIds },
...remainingTrainingMatch
}
},
{
$addFields: {
modeRank: {
$switch: {
branches: trainingModeRanks.map(({ mode, rank }) => ({
case: { $eq: ['$mode', mode] },
then: rank
})),
default: 999
}
},
isActiveTraining: activeTrainingExpr,
isFinalErrorTraining: finalErrorTrainingExpr
} }
}, },
{ {
$group: { $group: {
_id: '$collectionId', _id: '$collectionId',
count: { $sum: 1 }, trainingAmount: { $sum: 1 },
hasError: { $max: { $cond: [{ $ifNull: ['$errorMsg', false] }, true, false] } } activeTrainingAmount: { $sum: { $cond: ['$isActiveTraining', 1, 0] } },
finalErrorAmount: { $sum: { $cond: ['$isFinalErrorTraining', 1, 0] } },
modeCounts: {
$push: {
mode: '$mode',
modeRank: '$modeRank',
activeCount: { $cond: ['$isActiveTraining', 1, 0] },
finalErrorCount: { $cond: ['$isFinalErrorTraining', 1, 0] }
}
}
}
},
{ $unwind: '$modeCounts' },
{
$group: {
_id: {
collectionId: '$_id',
mode: '$modeCounts.mode',
modeRank: '$modeCounts.modeRank'
},
trainingAmount: { $first: '$trainingAmount' },
activeTrainingAmount: { $first: '$activeTrainingAmount' },
finalErrorAmount: { $first: '$finalErrorAmount' },
activeCount: { $sum: '$modeCounts.activeCount' },
finalErrorCount: { $sum: '$modeCounts.finalErrorCount' }
}
},
{
$sort: {
'_id.collectionId': 1,
'_id.modeRank': 1
}
},
{
$group: {
_id: '$_id.collectionId',
trainingAmount: { $first: '$trainingAmount' },
activeTrainingAmount: { $first: '$activeTrainingAmount' },
finalErrorAmount: { $first: '$finalErrorAmount' },
modeCounts: {
$push: {
mode: '$_id.mode',
activeCount: '$activeCount',
finalErrorCount: '$finalErrorCount'
}
}
} }
} }
], ],
...@@ -171,10 +284,10 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy ...@@ -171,10 +284,10 @@ async function handler(req: ApiRequestProps): Promise<ListCollectionV2ResponseTy
datasetId, datasetId,
tags: item.tags tags: item.tags
}), }),
trainingAmount:
trainingAmount.find((amount) => String(amount._id) === String(item._id))?.count || 0,
dataAmount: dataAmount.find((amount) => String(amount._id) === String(item._id))?.count || 0, dataAmount: dataAmount.find((amount) => String(amount._id) === String(item._id))?.count || 0,
hasError: trainingAmount.find((amount) => String(amount._id) === String(item._id))?.hasError, ...formatTrainingStatus(
trainingAmount.find((amount) => String(amount._id) === String(item._id))
),
permission permission
})) }))
); );
......
projects/app/src/pages/api/core/dataset/collection/scrollList.ts
/**
* @deprecated Use /core/dataset/collection/listV2 instead.
*/
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { NextAPI } from '@/service/middleware/entry'; import { NextAPI } from '@/service/middleware/entry';
...@@ -5,7 +8,10 @@ import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/tra ...@@ -5,7 +8,10 @@ import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/tra
import { Types } from '@fastgpt/service/common/mongo'; import { Types } from '@fastgpt/service/common/mongo';
import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema'; import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import {
CollectionTrainingStatusEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { type PaginationResponse } from '@fastgpt/global/openapi/api'; import { type PaginationResponse } from '@fastgpt/global/openapi/api';
import type { DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api'; import type { DatasetCollectionsListItemType } from '@fastgpt/global/openapi/core/dataset/collection/api';
...@@ -14,6 +20,13 @@ import { replaceRegChars } from '@fastgpt/global/common/string/tools'; ...@@ -14,6 +20,13 @@ import { replaceRegChars } from '@fastgpt/global/common/string/tools';
import { ScrollCollectionsBodySchema } from '@fastgpt/global/openapi/core/dataset/collection/api'; import { ScrollCollectionsBodySchema } from '@fastgpt/global/openapi/core/dataset/collection/api';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError'; import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
const defaultCollectionTrainingStatus = {
activeTrainingAmount: 0,
finalErrorAmount: 0,
hasError: false,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
};
async function handler( async function handler(
req: ApiRequestProps req: ApiRequestProps
): Promise<PaginationResponse<DatasetCollectionsListItemType>> { ): Promise<PaginationResponse<DatasetCollectionsListItemType>> {
...@@ -80,6 +93,7 @@ async function handler( ...@@ -80,6 +93,7 @@ async function handler(
...item, ...item,
dataAmount: 0, dataAmount: 0,
trainingAmount: 0, trainingAmount: 0,
...defaultCollectionTrainingStatus,
indexAmount: 0, indexAmount: 0,
permission permission
})) }))
...@@ -150,7 +164,11 @@ async function handler( ...@@ -150,7 +164,11 @@ async function handler(
}, },
trainingAmount: { trainingAmount: {
$ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0] $ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0]
} },
activeTrainingAmount: { $literal: 0 },
finalErrorAmount: { $literal: 0 },
hasError: { $literal: false },
slowestTrainingStatus: { $literal: CollectionTrainingStatusEnum.ready }
} }
} }
]), ]),
......
projects/app/src/pages/api/core/dataset/collection/trainingDetail.ts
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { import { type TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
DatasetCollectionDataProcessModeEnum,
type TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { NextAPI } from '@/service/middleware/entry'; import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
...@@ -15,6 +12,11 @@ import { ...@@ -15,6 +12,11 @@ import {
GetCollectionTrainingDetailResponseSchema, GetCollectionTrainingDetailResponseSchema,
type GetCollectionTrainingDetailResponseType type GetCollectionTrainingDetailResponseType
} from '@fastgpt/global/openapi/core/dataset/collection/api'; } from '@fastgpt/global/openapi/core/dataset/collection/api';
import {
BLOCKED_LOCK_TIME,
activeTrainingMatch,
finalErrorTrainingMatch
} from '@fastgpt/service/core/dataset/training/query';
const defaultCounts: Record<TrainingModeEnum, number> = { const defaultCounts: Record<TrainingModeEnum, number> = {
parse: 0, parse: 0,
...@@ -57,7 +59,7 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai ...@@ -57,7 +59,7 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai
$match: { $match: {
_id: { $lt: new Types.ObjectId(minId) }, _id: { $lt: new Types.ObjectId(minId) },
retryCount: { $gt: 0 }, retryCount: { $gt: 0 },
lockTime: { $lt: new Date('2050/1/1') } lockTime: { $lt: BLOCKED_LOCK_TIME }
} }
}, },
{ {
...@@ -72,8 +74,7 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai ...@@ -72,8 +74,7 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai
{ {
$match: { $match: {
...match, ...match,
retryCount: { $gt: 0 }, ...activeTrainingMatch
lockTime: { $lt: new Date('2050/1/1') }
} }
}, },
{ {
...@@ -87,8 +88,7 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai ...@@ -87,8 +88,7 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai
{ {
$match: { $match: {
...match, ...match,
// retryCount: { $lte: 0 }, ...finalErrorTrainingMatch
errorMsg: { $exists: true }
} }
}, },
{ {
......
projects/app/src/pages/api/core/dataset/training/deleteTrainingData.ts
...@@ -35,3 +35,6 @@ async function handler(req: ApiRequestProps): Promise<DeleteTrainingDataResponse ...@@ -35,3 +35,6 @@ async function handler(req: ApiRequestProps): Promise<DeleteTrainingDataResponse
} }
export default NextAPI(handler); export default NextAPI(handler);
export type deleteTrainingDataBody =
import('@fastgpt/global/openapi/core/dataset/training/api').DeleteTrainingDataBody;
export type deleteTrainingDataResponse = DeleteTrainingDataResponse;
projects/app/src/pages/api/core/dataset/training/getDatasetTrainingError.ts 0 → 100644
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
import { Types, type PipelineStage } from '@fastgpt/service/common/mongo';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import {
DatasetTrainingErrorPaginationLimits,
GetDatasetTrainingErrorBodySchema,
GetDatasetTrainingErrorResponseSchema,
type GetDatasetTrainingErrorBody,
type GetDatasetTrainingErrorResponse,
type TrainingErrorGroupType,
type TrainingErrorItemType
} from '@fastgpt/global/openapi/core/dataset/training/api';
import {
finalErrorTrainingMatch,
trainingModeRanks
} from '@fastgpt/service/core/dataset/training/query';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
async function handler(req: ApiRequestProps): Promise<GetDatasetTrainingErrorResponse> {
const {
datasetId,
collectionId,
itemOffset = 0,
itemPageSize = DatasetTrainingErrorPaginationLimits.defaultItemPageSize
} = parseApiInput({
req,
bodySchema: GetDatasetTrainingErrorBodySchema
}).body;
const { offset, pageSize: rawPageSize } = parsePaginationRequest(req);
const pageSize = Math.min(rawPageSize, DatasetTrainingErrorPaginationLimits.maxPageSize);
const itemLimit = Math.min(itemPageSize, DatasetTrainingErrorPaginationLimits.maxItemPageSize);
const itemSkip = Math.min(itemOffset, DatasetTrainingErrorPaginationLimits.maxItemOffset);
const { teamId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: ReadPermissionVal
});
const baseTrainingMatch = {
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(dataset._id),
...finalErrorTrainingMatch
};
/**
* 复用集合内异常 chunk 查询流水线，确保首屏和“加载更多”的排序完全一致。
*/
const buildItemPipeline = ({
match,
skip,
limit
}: {
match: Record<string, any>;
skip: number;
limit: number;
}): PipelineStage[] => [
{ $match: match },
{
$addFields: {
modeRank: {
$switch: {
branches: trainingModeRanks.map(({ mode, rank }) => ({
case: { $eq: ['$mode', mode] },
then: rank
})),
default: 999
}
}
}
},
{ $sort: { modeRank: 1, chunkIndex: 1, _id: 1 } },
{ $skip: skip },
{ $limit: limit },
{ $project: { modeRank: 0 } }
];
if (collectionId) {
const collection = await MongoDatasetCollection.findOne(
{
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(dataset._id),
_id: new Types.ObjectId(collectionId),
type: { $ne: DatasetCollectionTypeEnum.folder }
},
undefined,
readFromSecondary
).lean();
if (!collection) {
return GetDatasetTrainingErrorResponseSchema.parse({
total: 0,
list: []
});
}
const collectionMatch = {
...baseTrainingMatch,
collectionId: new Types.ObjectId(collection._id)
};
const [items, errorCount] = await Promise.all([
MongoDatasetTraining.aggregate(
buildItemPipeline({
match: collectionMatch,
skip: itemSkip,
limit: itemLimit
}),
readFromSecondary
),
MongoDatasetTraining.countDocuments(collectionMatch, { ...readFromSecondary })
]);
const { sourceName, sourceId } = getCollectionSourceData(collection);
return GetDatasetTrainingErrorResponseSchema.parse({
total: errorCount > 0 ? 1 : 0,
list:
errorCount > 0
? [
{
collection: {
_id: collection._id,
collectionId: collection._id,
name: collection.name,
type: collection.type,
sourceName,
sourceId
},
items,
errorCount,
hasMoreItems: itemSkip + items.length < errorCount
}
]
: []
});
}
const [collectionStats, totalResult] = await Promise.all([
MongoDatasetTraining.aggregate(
[
{ $match: baseTrainingMatch },
{
$group: {
_id: '$collectionId',
errorCount: { $sum: 1 }
}
},
{ $sort: { _id: 1 } },
{ $skip: offset },
{ $limit: pageSize }
],
readFromSecondary
),
MongoDatasetTraining.aggregate(
[
{ $match: baseTrainingMatch },
{
$group: {
_id: '$collectionId'
}
},
{ $count: 'total' }
],
readFromSecondary
)
]);
const total = totalResult[0]?.total ?? 0;
const collectionIds = collectionStats.map((item: { _id: any }) => item._id);
const collections = collectionIds.length
? await MongoDatasetCollection.find(
{
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(dataset._id),
_id: { $in: collectionIds },
type: { $ne: DatasetCollectionTypeEnum.folder }
},
undefined,
readFromSecondary
).lean()
: [];
const collectionMap = new Map(
collections.map((collection) => [String(collection._id), collection])
);
const statsMap = new Map(
collectionStats.map((item: { _id: any; errorCount: number }) => [
String(item._id),
item.errorCount
])
);
const itemsList = collectionIds.length
? await Promise.all(
collectionIds.map((id: any) =>
MongoDatasetTraining.aggregate(
buildItemPipeline({
match: {
...baseTrainingMatch,
collectionId: id
},
skip: 0,
limit: itemLimit
}),
readFromSecondary
)
)
)
: [];
const list = collectionIds.reduce<TrainingErrorGroupType[]>((groups, id: any, index: number) => {
const collection = collectionMap.get(String(id));
const errorCount = statsMap.get(String(id)) ?? 0;
const items = (itemsList[index] ?? []) as TrainingErrorItemType[];
if (!collection || errorCount === 0 || items.length === 0) return groups;
const { sourceName, sourceId } = getCollectionSourceData(collection);
groups.push({
collection: {
_id: collection._id,
collectionId: collection._id,
name: collection.name,
type: collection.type,
sourceName,
sourceId
},
items,
errorCount,
hasMoreItems: items.length < errorCount
});
return groups;
}, []);
return GetDatasetTrainingErrorResponseSchema.parse({
total,
list
});
}
export default NextAPI(handler);
export type getDatasetTrainingErrorBody = GetDatasetTrainingErrorBody;
export type getDatasetTrainingErrorResponse = GetDatasetTrainingErrorResponse;
projects/app/src/pages/api/core/dataset/training/getTrainingDataDetail.ts
...@@ -41,6 +41,7 @@ async function handler(req: ApiRequestProps): Promise<GetTrainingDataDetailRespo ...@@ -41,6 +41,7 @@ async function handler(req: ApiRequestProps): Promise<GetTrainingDataDetailRespo
return GetTrainingDataDetailResponseSchema.parse({ return GetTrainingDataDetailResponseSchema.parse({
_id: data._id, _id: data._id,
datasetId: data.datasetId, datasetId: data.datasetId,
collectionId: data.collectionId,
mode: data.mode, mode: data.mode,
imagePreviewUrl: imagePreviewUrl:
data.imageId && isS3ObjectKey(data.imageId, 'dataset') data.imageId && isS3ObjectKey(data.imageId, 'dataset')
......
projects/app/src/pages/api/core/dataset/training/getTrainingError.ts
import { NextAPI } from '@/service/middleware/entry'; import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination'; import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
import { Types, type PipelineStage } from '@fastgpt/service/common/mongo';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
...@@ -9,11 +10,19 @@ import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError'; ...@@ -9,11 +10,19 @@ import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { import {
GetTrainingErrorBodySchema, GetTrainingErrorBodySchema,
GetTrainingErrorResponseSchema, GetTrainingErrorResponseSchema,
type GetTrainingErrorBody,
type GetTrainingErrorResponse type GetTrainingErrorResponse
} from '@fastgpt/global/openapi/core/dataset/training/api'; } from '@fastgpt/global/openapi/core/dataset/training/api';
import {
finalErrorTrainingMatch,
trainingModeRanks
} from '@fastgpt/service/core/dataset/training/query';
async function handler(req: ApiRequestProps): Promise<GetTrainingErrorResponse> { async function handler(req: ApiRequestProps): Promise<GetTrainingErrorResponse> {
const { collectionId } = parseApiInput({ req, bodySchema: GetTrainingErrorBodySchema }).body; const { collectionId } = parseApiInput({
req,
bodySchema: GetTrainingErrorBodySchema
}).body;
const { offset, pageSize } = parsePaginationRequest(req); const { offset, pageSize } = parsePaginationRequest(req);
const { collection } = await authDatasetCollection({ const { collection } = await authDatasetCollection({
...@@ -25,26 +34,43 @@ async function handler(req: ApiRequestProps): Promise<GetTrainingErrorResponse> ...@@ -25,26 +34,43 @@ async function handler(req: ApiRequestProps): Promise<GetTrainingErrorResponse>
}); });
const match = { const match = {
teamId: collection.teamId, teamId: new Types.ObjectId(collection.teamId),
datasetId: collection.datasetId, datasetId: new Types.ObjectId(collection.datasetId),
collectionId: collection._id, collectionId: new Types.ObjectId(collection._id),
errorMsg: { $exists: true } ...finalErrorTrainingMatch
}; };
const pipeline: PipelineStage[] = [
{ $match: match },
{
$addFields: {
modeRank: {
$switch: {
branches: trainingModeRanks.map(({ mode, rank }) => ({
case: { $eq: ['$mode', mode] },
then: rank
})),
default: 999
}
}
}
},
{ $sort: { modeRank: 1, chunkIndex: 1, _id: 1 } },
{ $skip: offset },
{ $limit: pageSize },
{ $project: { modeRank: 0 } }
];
const [errorList, total] = await Promise.all([ const [errorList, total] = await Promise.all([
MongoDatasetTraining.find(match, undefined, { MongoDatasetTraining.aggregate(pipeline, readFromSecondary),
...readFromSecondary
})
.skip(offset)
.limit(pageSize)
.lean(),
MongoDatasetTraining.countDocuments(match, { ...readFromSecondary }) MongoDatasetTraining.countDocuments(match, { ...readFromSecondary })
]); ]);
return GetTrainingErrorResponseSchema.parse({ return GetTrainingErrorResponseSchema.parse({
list: errorList, total,
total list: errorList
}); });
} }
export default NextAPI(handler); export default NextAPI(handler);
export type getTrainingErrorBody = GetTrainingErrorBody;
export type getTrainingErrorResponse = GetTrainingErrorResponse;
projects/app/src/pages/api/core/dataset/training/hasDatasetTrainingError.ts 0 → 100644
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { Types } from '@fastgpt/service/common/mongo';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { finalErrorTrainingMatch } from '@fastgpt/service/core/dataset/training/query';
import {
HasDatasetTrainingErrorQuerySchema,
HasDatasetTrainingErrorResponseSchema,
type HasDatasetTrainingErrorQuery,
type HasDatasetTrainingErrorResponse
} from '@fastgpt/global/openapi/core/dataset/training/api';
async function handler(req: ApiRequestProps): Promise<HasDatasetTrainingErrorResponse> {
const { datasetId } = parseApiInput({
req,
querySchema: HasDatasetTrainingErrorQuerySchema
}).query;
const { teamId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: ReadPermissionVal
});
const errorRecord = await MongoDatasetTraining.findOne(
{
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(dataset._id),
...finalErrorTrainingMatch
},
{ _id: 1 },
readFromSecondary
).lean();
return HasDatasetTrainingErrorResponseSchema.parse({
hasError: !!errorRecord
});
}
export default NextAPI(handler);
export type hasDatasetTrainingErrorQuery = HasDatasetTrainingErrorQuery;
export type hasDatasetTrainingErrorResponse = HasDatasetTrainingErrorResponse;
projects/app/src/pages/api/core/dataset/training/updateTrainingData.ts
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant'; import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import {
authDataset,
authDatasetCollection
} from '@fastgpt/service/support/permission/dataset/auth';
import { NextAPI } from '@/service/middleware/entry'; import { NextAPI } from '@/service/middleware/entry';
import { type ApiRequestProps } from '@fastgpt/service/type/next'; import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
...@@ -10,33 +13,48 @@ import { ...@@ -10,33 +13,48 @@ import {
type UpdateTrainingDataResponse type UpdateTrainingDataResponse
} from '@fastgpt/global/openapi/core/dataset/training/api'; } from '@fastgpt/global/openapi/core/dataset/training/api';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError'; import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { finalErrorTrainingMatch } from '@fastgpt/service/core/dataset/training/query';
async function handler(req: ApiRequestProps): Promise<UpdateTrainingDataResponse> { async function handler(req: ApiRequestProps): Promise<UpdateTrainingDataResponse> {
const { collectionId, dataId, q, a, chunkIndex } = parseApiInput({ const body = parseApiInput({ req, bodySchema: UpdateTrainingDataBodySchema }).body;
req,
bodySchema: UpdateTrainingDataBodySchema
}).body;
// 不传 dataId 时是批量重试：collectionId 和 datasetId 分别限定不同的重试范围。
if (!body.dataId) {
const retryMatch = await (async () => {
if (body.collectionId) {
const { collection } = await authDatasetCollection({ const { collection } = await authDatasetCollection({
req, req,
authToken: true, authToken: true,
authApiKey: true, authApiKey: true,
collectionId, collectionId: body.collectionId,
per: WritePermissionVal per: WritePermissionVal
}); });
const trainingMatch = { return {
teamId: collection.teamId, teamId: collection.teamId,
datasetId: collection.datasetId, datasetId: collection.datasetId,
collectionId: collection._id collectionId: collection._id
}; };
}
const { teamId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId: body.datasetId!,
per: WritePermissionVal
});
return {
teamId,
datasetId: dataset._id
};
})();
// If dataId is not passed, all error data in this collection will be retried.
if (!dataId) {
await MongoDatasetTraining.updateMany( await MongoDatasetTraining.updateMany(
{ {
...trainingMatch, ...retryMatch,
errorMsg: { $exists: true, $ne: null } ...finalErrorTrainingMatch
}, },
{ {
$unset: { errorMsg: '' }, $unset: { errorMsg: '' },
...@@ -47,21 +65,40 @@ async function handler(req: ApiRequestProps): Promise<UpdateTrainingDataResponse ...@@ -47,21 +65,40 @@ async function handler(req: ApiRequestProps): Promise<UpdateTrainingDataResponse
return UpdateTrainingDataResponseSchema.parse(undefined); return UpdateTrainingDataResponseSchema.parse(undefined);
} }
// Single data retry logic const { q, a, chunkIndex } = body;
const data = await MongoDatasetTraining.findOne({ ...trainingMatch, _id: dataId }); // 单条重试只信任 dataId 找到的训练记录，再用记录所属 collection 做权限校验。
const data = await MongoDatasetTraining.findById(body.dataId);
if (!data) { if (!data) {
return Promise.reject('data not found'); return Promise.reject('data not found');
} }
const { collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId: data.collectionId,
per: WritePermissionVal
});
if (
String(collection.teamId) !== String(data.teamId) ||
String(collection.datasetId) !== String(data.datasetId) ||
String(collection._id) !== String(data.collectionId)
) {
return Promise.reject('data not found');
}
const trainingMatch = {
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id,
_id: data._id
};
// Add to chunk // Add to chunk
if (data.imageId && q) { if (data.imageId && q) {
await MongoDatasetTraining.updateOne( await MongoDatasetTraining.updateOne(trainingMatch, {
{
...trainingMatch,
_id: dataId
},
{
$unset: { errorMsg: '' }, $unset: { errorMsg: '' },
retryCount: 3, retryCount: 3,
mode: TrainingModeEnum.chunk, mode: TrainingModeEnum.chunk,
...@@ -69,23 +106,16 @@ async function handler(req: ApiRequestProps): Promise<UpdateTrainingDataResponse ...@@ -69,23 +106,16 @@ async function handler(req: ApiRequestProps): Promise<UpdateTrainingDataResponse
...(a !== undefined && { a }), ...(a !== undefined && { a }),
...(chunkIndex !== undefined && { chunkIndex }), ...(chunkIndex !== undefined && { chunkIndex }),
lockTime: new Date('2000') lockTime: new Date('2000')
} });
);
} else { } else {
await MongoDatasetTraining.updateOne( await MongoDatasetTraining.updateOne(trainingMatch, {
{
...trainingMatch,
_id: dataId
},
{
$unset: { errorMsg: '' }, $unset: { errorMsg: '' },
retryCount: 3, retryCount: 3,
...(q !== undefined && { q }), ...(q !== undefined && { q }),
...(a !== undefined && { a }), ...(a !== undefined && { a }),
...(chunkIndex !== undefined && { chunkIndex }), ...(chunkIndex !== undefined && { chunkIndex }),
lockTime: new Date('2000') lockTime: new Date('2000')
} });
);
} }
return UpdateTrainingDataResponseSchema.parse(undefined); return UpdateTrainingDataResponseSchema.parse(undefined);
......
projects/app/src/service/core/dataset/queues/datasetParse.ts
...@@ -167,7 +167,7 @@ export const datasetParseQueue = async (): Promise<any> => { ...@@ -167,7 +167,7 @@ export const datasetParseQueue = async (): Promise<any> => {
continue; continue;
} }
// Check team points and lock(No mistakes will be thrown here) // Check team points and lock(No mistakes will be thrown here)
if (!(await checkTeamAiPointsAndLock(data.teamId))) { if (!(await checkTeamAiPointsAndLock(data.teamId, String(data._id)))) {
continue; continue;
} }
......
projects/app/src/service/core/dataset/queues/generateQA.ts
...@@ -112,7 +112,7 @@ export async function generateQA(): Promise<any> { ...@@ -112,7 +112,7 @@ export async function generateQA(): Promise<any> {
continue; continue;
} }
// auth balance // auth balance
if (!(await checkTeamAiPointsAndLock(data.teamId))) { if (!(await checkTeamAiPointsAndLock(data.teamId, String(data._id)))) {
continue; continue;
} }
......
projects/app/src/service/core/dataset/queues/generateVector.ts
...@@ -160,7 +160,7 @@ export async function generateVector(): Promise<any> { ...@@ -160,7 +160,7 @@ export async function generateVector(): Promise<any> {
} }
// auth balance // auth balance
if (!(await checkTeamAiPointsAndLock(data.teamId))) { if (!(await checkTeamAiPointsAndLock(data.teamId, String(data._id)))) {
continue; continue;
} }
......
projects/app/src/service/core/dataset/queues/utils.ts
...@@ -6,7 +6,7 @@ import { InformLevelEnum } from '@fastgpt/global/support/user/inform/constants'; ...@@ -6,7 +6,7 @@ import { InformLevelEnum } from '@fastgpt/global/support/user/inform/constants';
import { getLogger, LogCategories } from '@fastgpt/service/common/logger'; import { getLogger, LogCategories } from '@fastgpt/service/common/logger';
const logger = getLogger(LogCategories.MODULE.DATASET.QUEUES); const logger = getLogger(LogCategories.MODULE.DATASET.QUEUES);
export const checkTeamAiPointsAndLock = async (teamId: string) => { export const checkTeamAiPointsAndLock = async (teamId: string, currentTrainingId?: string) => {
try { try {
await checkTeamAIPoints(teamId); await checkTeamAIPoints(teamId);
return true; return true;
...@@ -21,7 +21,7 @@ export const checkTeamAiPointsAndLock = async (teamId: string) => { ...@@ -21,7 +21,7 @@ export const checkTeamAiPointsAndLock = async (teamId: string) => {
teamId teamId
}); });
logger.info('余额不足，暂停知识库处理'); logger.info('余额不足，暂停知识库处理');
await lockTrainingDataByTeamId(teamId); await lockTrainingDataByTeamId(teamId, currentTrainingId);
} catch (error) {} } catch (error) {}
} }
return false; return false;
......
projects/app/src/web/core/dataset/api/training.ts
...@@ -7,7 +7,10 @@ import type { ...@@ -7,7 +7,10 @@ import type {
GetTrainingDataDetailBody, GetTrainingDataDetailBody,
GetTrainingDataDetailResponse, GetTrainingDataDetailResponse,
GetTrainingErrorBody, GetTrainingErrorBody,
GetTrainingErrorResponse GetTrainingErrorResponse,
GetDatasetTrainingErrorBody,
GetDatasetTrainingErrorResponse,
HasDatasetTrainingErrorResponse
} from '@fastgpt/global/openapi/core/dataset/training/api'; } from '@fastgpt/global/openapi/core/dataset/training/api';
export const postRebuildEmbedding = (data: RebuildEmbeddingBody) => export const postRebuildEmbedding = (data: RebuildEmbeddingBody) =>
...@@ -29,3 +32,11 @@ export const getTrainingDataDetail = (data: GetTrainingDataDetailBody) => ...@@ -29,3 +32,11 @@ export const getTrainingDataDetail = (data: GetTrainingDataDetailBody) =>
export const getTrainingError = (data: GetTrainingErrorBody) => export const getTrainingError = (data: GetTrainingErrorBody) =>
POST<GetTrainingErrorResponse>(`/core/dataset/training/getTrainingError`, data); POST<GetTrainingErrorResponse>(`/core/dataset/training/getTrainingError`, data);
export const getDatasetTrainingError = (data: GetDatasetTrainingErrorBody) =>
POST<GetDatasetTrainingErrorResponse>(`/core/dataset/training/getDatasetTrainingError`, data);
export const hasDatasetTrainingError = (datasetId: string) =>
GET<HasDatasetTrainingErrorResponse>(`/core/dataset/training/hasDatasetTrainingError`, {
datasetId
});
projects/app/src/web/core/dataset/constants.ts
import { defaultQAModels, defaultVectorModels } from '@fastgpt/global/core/ai/constants'; import { defaultQAModels, defaultVectorModels } from '@fastgpt/global/core/ai/constants';
import { import {
CollectionTrainingStatusEnum,
DatasetCollectionDataProcessModeEnum, DatasetCollectionDataProcessModeEnum,
DatasetCollectionTypeEnum, DatasetCollectionTypeEnum,
DatasetTypeEnum, DatasetTypeEnum,
...@@ -60,7 +61,12 @@ export const defaultCollectionDetail: DatasetCollectionItemType = { ...@@ -60,7 +61,12 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
chunkSize: 0, chunkSize: 0,
indexSize: 512, indexSize: 512,
permission: new DatasetPermission(), permission: new DatasetPermission(),
indexAmount: 0 indexAmount: 0,
trainingAmount: 0,
activeTrainingAmount: 0,
finalErrorAmount: 0,
hasError: false,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
}; };
export const TrainingProcess = { export const TrainingProcess = {
......
projects/app/src/web/core/dataset/trainingStatus.ts 0 → 100644
import {
CollectionTrainingStatusEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { i18nT } from '@fastgpt/global/common/i18n/utils';
import type { TFunction } from 'next-i18next';
import type { ColorSchemaType } from '@fastgpt/web/components/common/Tag';
/**
* 根据训练模式获取对应的阶段文本描述
* @param mode - 训练模式枚举，若未提供则返回“等待中”
* @returns 国际化后的阶段文本字符串
*/
export const getTrainingStageText = (mode?: TrainingModeEnum) => {
const textMap = {
[TrainingModeEnum.parse]: i18nT('dataset:process.Parsing'),
[TrainingModeEnum.imageParse]: i18nT('dataset:process.Parse_Image'),
[TrainingModeEnum.qa]: i18nT('dataset:process.Get QA'),
[TrainingModeEnum.image]: i18nT('dataset:process.Image_Index'),
[TrainingModeEnum.auto]: i18nT('dataset:process.Auto_Index'),
[TrainingModeEnum.chunk]: i18nT('dataset:process.Vectorizing')
};
return mode ? textMap[mode] : i18nT('dataset:process.Waiting');
};
/**
* 根据集合中最慢的训练任务和状态，获取整体训练状态文本
* @param params.slowestTrainingMode - 最慢任务的训练模式
* @param params.slowestTrainingStatus - 最慢任务的训练状态
* @returns 国际化后的状态文本（错误/进行中具体阶段/等待中/已就绪）
*/
export const getCollectionTrainingStatusText = ({
slowestTrainingMode,
slowestTrainingStatus
}: {
slowestTrainingMode?: TrainingModeEnum;
slowestTrainingStatus?: CollectionTrainingStatusEnum;
}) => {
if (slowestTrainingStatus === CollectionTrainingStatusEnum.error) {
return i18nT('dataset:training.status_error');
}
if (slowestTrainingStatus === CollectionTrainingStatusEnum.running && slowestTrainingMode) {
return getTrainingStageText(slowestTrainingMode);
}
if (slowestTrainingStatus === CollectionTrainingStatusEnum.running) {
return i18nT('dataset:process.Waiting');
}
return i18nT('dataset:process.Is_Ready');
};
/**
* 使用传入的翻译函数格式化训练阶段文本
* @param mode - 训练模式枚举
* @param t - i18next 翻译函数实例
* @returns 国际化后的阶段文本字符串
*/
export const formatTrainingStageText = (mode: TrainingModeEnum | undefined, t: TFunction) => {
return t(getTrainingStageText(mode) as any);
};
/**
* 根据集合中最慢的训练任务和状态，获取状态标签的颜色主题
* @param params.slowestTrainingMode - 最慢任务的训练模式
* @param params.slowestTrainingStatus - 最慢任务的训练状态
* @returns MyTag 可消费的颜色主题名
*/
export const getCollectionTrainingStatusColorSchema = ({
slowestTrainingMode,
slowestTrainingStatus
}: {
slowestTrainingMode?: TrainingModeEnum;
slowestTrainingStatus?: CollectionTrainingStatusEnum;
}): ColorSchemaType => {
if (slowestTrainingStatus === CollectionTrainingStatusEnum.error) {
return 'lightRed';
}
if (slowestTrainingStatus !== CollectionTrainingStatusEnum.running) {
return 'green';
}
switch (slowestTrainingMode) {
case TrainingModeEnum.parse:
case TrainingModeEnum.imageParse:
return 'blue';
case TrainingModeEnum.qa:
case TrainingModeEnum.image:
case TrainingModeEnum.auto:
return 'cyan';
case TrainingModeEnum.chunk:
return 'adora';
default:
return 'lightGray';
}
};
projects/app/test/api/core/dataset/collection/trainingStatus.test.ts 0 → 100644
import listHandler from '@/pages/api/core/dataset/collection/listV2';
import scrollListHandler from '@/pages/api/core/dataset/collection/scrollList';
import detailHandler from '@/pages/api/core/dataset/collection/detail';
import trainingDetailHandler from '@/pages/api/core/dataset/collection/trainingDetail';
import {
CollectionTrainingStatusEnum,
DatasetCollectionTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { DatasetCollectionsListItemSchema } from '@fastgpt/global/openapi/core/dataset/collection/api';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { getRootUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest';
describe('collection training status api', () => {
it('should expose unified active/final error/slowest status in list and detail', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 3
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'final error'
}
]);
const listRes = await Call(listHandler, {
auth: root,
body: {
datasetId: dataset._id,
pageSize: 10,
offset: 0,
filterTags: []
}
});
expect(listRes.code).toBe(200);
expect(listRes.data.list[0]).toMatchObject({
trainingAmount: 2,
activeTrainingAmount: 1,
finalErrorAmount: 1,
hasError: true,
slowestTrainingMode: TrainingModeEnum.parse,
slowestTrainingStatus: CollectionTrainingStatusEnum.running
});
const detailRes = await Call(detailHandler, {
auth: root,
query: {
id: collection._id
}
});
expect(detailRes.code).toBe(200);
expect(detailRes.data).toMatchObject({
trainingAmount: 2,
activeTrainingAmount: 1,
finalErrorAmount: 1,
errorCount: 1,
hasError: true,
slowestTrainingMode: TrainingModeEnum.parse,
slowestTrainingStatus: CollectionTrainingStatusEnum.running
});
});
it('should use active counts for progress and final errors for error tab', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 3,
errorMsg: 'temporary failed'
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'final failed'
}
]);
const res = await Call(trainingDetailHandler, {
auth: root,
query: {
collectionId: collection._id
}
});
expect(res.code).toBe(200);
expect(res.data.trainingCounts.qa).toBe(1);
expect(res.data.errorCounts.qa).toBe(0);
expect(res.data.errorCounts.chunk).toBe(1);
});
it('should keep deprecated scrollList compatible with the collection list item schema', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'final failed'
});
const res = await Call(scrollListHandler, {
auth: root,
body: {
datasetId: dataset._id,
pageSize: 10,
offset: 0,
filterTags: []
}
});
expect(res.code).toBe(200);
expect(() => DatasetCollectionsListItemSchema.parse(res.data.list[0])).not.toThrow();
expect(res.data.list[0]).toMatchObject({
trainingAmount: 1,
slowestTrainingStatus: CollectionTrainingStatusEnum.ready
});
});
});
projects/app/test/api/core/dataset/training/deleteTrainingData.test.ts
...@@ -39,7 +39,11 @@ describe('delete training data test', () => { ...@@ -39,7 +39,11 @@ describe('delete training data test', () => {
mode: TrainingModeEnum.chunk mode: TrainingModeEnum.chunk
}); });
const res = await Call<deleteTrainingDataBody, {}, deleteTrainingDataResponse>(handler, { const res = await Call<
deleteTrainingDataBody,
Record<string, never>,
deleteTrainingDataResponse
>(handler, {
auth: root, auth: root,
body: { body: {
collectionId: collection._id, collectionId: collection._id,
...@@ -100,7 +104,11 @@ describe('delete training data test', () => { ...@@ -100,7 +104,11 @@ describe('delete training data test', () => {
mode: TrainingModeEnum.chunk mode: TrainingModeEnum.chunk
}); });
const res = await Call<deleteTrainingDataBody, {}, deleteTrainingDataResponse>(handler, { const res = await Call<
deleteTrainingDataBody,
Record<string, never>,
deleteTrainingDataResponse
>(handler, {
auth: root, auth: root,
body: { body: {
datasetId: foreignDataset._id, datasetId: foreignDataset._id,
......
projects/app/test/api/core/dataset/training/getDatasetTrainingError.test.ts 0 → 100644
import handler, {
type getDatasetTrainingErrorBody,
type getDatasetTrainingErrorResponse
} from '@/pages/api/core/dataset/training/getDatasetTrainingError';
import {
DatasetCollectionTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { DatasetTrainingErrorPaginationLimits } from '@fastgpt/global/openapi/core/dataset/training/api';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { ApiRequestInputParseError } from '@fastgpt/service/common/zod/requestParseError';
import { getRootUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest';
type EmptyQuery = Record<string, never>;
describe('dataset training error list test', () => {
it('should paginate error collections and load more chunks inside one collection', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const [fileCollection, linkCollection] = await MongoDatasetCollection.create([
{
name: 'file',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
updateTime: new Date('2025-01-01')
},
{
name: 'link',
type: DatasetCollectionTypeEnum.link,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
rawLink: 'https://example.com',
updateTime: new Date('2025-01-02')
}
]);
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: fileCollection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 3,
errorMsg: 'temporary failed',
chunkIndex: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: fileCollection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'file error',
chunkIndex: 2
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: fileCollection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 0,
errorMsg: 'file parse error',
chunkIndex: 1
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: fileCollection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: ' ',
chunkIndex: 3
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: linkCollection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 0,
errorMsg: 'link error',
chunkIndex: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: linkCollection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 3,
errorMsg: 'link chunk error',
chunkIndex: 1
}
]);
const firstPageRes = await Call<
getDatasetTrainingErrorBody,
EmptyQuery,
getDatasetTrainingErrorResponse
>(handler, {
auth: root,
body: {
datasetId: dataset._id,
pageSize: 1,
offset: 0,
itemPageSize: 1
}
});
expect(firstPageRes.code).toBe(200);
expect(firstPageRes.data.total).toBe(2);
expect(firstPageRes.data.list).toHaveLength(1);
expect(firstPageRes.data.list[0].collection.name).toBe('file');
expect(firstPageRes.data.list[0].errorCount).toBe(2);
expect(firstPageRes.data.list[0].hasMoreItems).toBe(true);
expect(firstPageRes.data.list[0].items).toHaveLength(1);
expect(firstPageRes.data.list[0].items[0].mode).toBe(TrainingModeEnum.parse);
const secondPageRes = await Call<
getDatasetTrainingErrorBody,
EmptyQuery,
getDatasetTrainingErrorResponse
>(handler, {
auth: root,
body: {
datasetId: dataset._id,
pageSize: 1,
offset: 1,
itemPageSize: 1
}
});
expect(secondPageRes.code).toBe(200);
expect(secondPageRes.data.total).toBe(2);
expect(secondPageRes.data.list).toHaveLength(1);
expect(secondPageRes.data.list[0].collection.name).toBe('link');
expect(secondPageRes.data.list[0].errorCount).toBe(1);
expect(secondPageRes.data.list[0].hasMoreItems).toBe(false);
expect(secondPageRes.data.list[0].items[0].mode).toBe(TrainingModeEnum.qa);
const loadMoreRes = await Call<
getDatasetTrainingErrorBody,
EmptyQuery,
getDatasetTrainingErrorResponse
>(handler, {
auth: root,
body: {
datasetId: dataset._id,
collectionId: fileCollection._id,
pageSize: 1,
offset: 0,
itemOffset: 1,
itemPageSize: 1
}
});
expect(loadMoreRes.code).toBe(200);
expect(loadMoreRes.data.total).toBe(1);
expect(loadMoreRes.data.list).toHaveLength(1);
expect(loadMoreRes.data.list[0].collection.name).toBe('file');
expect(loadMoreRes.data.list[0].errorCount).toBe(2);
expect(loadMoreRes.data.list[0].hasMoreItems).toBe(false);
expect(loadMoreRes.data.list[0].items).toHaveLength(1);
expect(loadMoreRes.data.list[0].items[0].mode).toBe(TrainingModeEnum.chunk);
});
it.each([
{
name: 'invalid itemOffset',
body: {
datasetId: '507f1f77bcf86cd799439011',
pageSize: 1,
offset: 0,
itemOffset: 'not-a-number',
itemPageSize: 1
}
},
{
name: 'oversized itemPageSize',
body: {
datasetId: '507f1f77bcf86cd799439011',
pageSize: 1,
offset: 0,
itemPageSize: DatasetTrainingErrorPaginationLimits.maxItemPageSize + 1
}
},
{
name: 'invalid pageSize',
body: {
datasetId: '507f1f77bcf86cd799439011',
pageSize: 'not-a-number',
offset: 0,
itemPageSize: 1
}
},
{
name: 'oversized pageSize',
body: {
datasetId: '507f1f77bcf86cd799439011',
pageSize: DatasetTrainingErrorPaginationLimits.maxPageSize + 1,
offset: 0,
itemPageSize: 1
}
}
])('should reject $name before querying Mongo', async ({ body }) => {
const res = await Call<any, EmptyQuery, getDatasetTrainingErrorResponse>(handler, {
body
});
expect(res.code).not.toBe(200);
expect(res.error).toBeInstanceOf(ApiRequestInputParseError);
});
});
projects/app/test/api/core/dataset/training/getTrainingDataDetail.test.ts
import handler, { import handler from '@/pages/api/core/dataset/training/getTrainingDataDetail';
type getTrainingDataDetailBody,
type getTrainingDataDetailResponse
} from '@/pages/api/core/dataset/training/getTrainingDataDetail';
import { import {
DatasetCollectionTypeEnum, DatasetCollectionTypeEnum,
TrainingModeEnum TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants'; } from '@fastgpt/global/core/dataset/constants';
import type {
GetTrainingDataDetailBody,
GetTrainingDataDetailResponse
} from '@fastgpt/global/openapi/core/dataset/training/api';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
...@@ -41,7 +42,11 @@ describe('get training data detail test', () => { ...@@ -41,7 +42,11 @@ describe('get training data detail test', () => {
a: 'test' a: 'test'
}); });
const res = await Call<getTrainingDataDetailBody, {}, getTrainingDataDetailResponse>(handler, { const res = await Call<
GetTrainingDataDetailBody,
Record<string, never>,
GetTrainingDataDetailResponse
>(handler, {
auth: root, auth: root,
body: { body: {
collectionId: collection._id, collectionId: collection._id,
...@@ -103,7 +108,11 @@ describe('get training data detail test', () => { ...@@ -103,7 +108,11 @@ describe('get training data detail test', () => {
a: 'foreign' a: 'foreign'
}); });
const res = await Call<getTrainingDataDetailBody, {}, getTrainingDataDetailResponse>(handler, { const res = await Call<
GetTrainingDataDetailBody,
Record<string, never>,
GetTrainingDataDetailResponse
>(handler, {
auth: root, auth: root,
body: { body: {
datasetId: foreignDataset._id, datasetId: foreignDataset._id,
......
projects/app/test/api/core/dataset/training/getTrainingError.test.ts
...@@ -13,8 +13,10 @@ import { getRootUser } from '@test/datas/users'; ...@@ -13,8 +13,10 @@ import { getRootUser } from '@test/datas/users';
import { Call } from '@test/utils/request'; import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest'; import { describe, expect, it } from 'vitest';
type EmptyQuery = Record<string, never>;
describe('training error list test', () => { describe('training error list test', () => {
it('should return training error list', async () => { it('should return final error list in collection scope', async () => {
const root = await getRootUser(); const root = await getRootUser();
const dataset = await MongoDataset.create({ const dataset = await MongoDataset.create({
name: 'test', name: 'test',
...@@ -30,19 +32,63 @@ describe('training error list test', () => { ...@@ -30,19 +32,63 @@ describe('training error list test', () => {
tmbId: root.tmbId, tmbId: root.tmbId,
datasetId: dataset._id datasetId: dataset._id
}); });
await MongoDatasetTraining.create( await MongoDatasetTraining.create([
[...Array(10).keys()].map((i) => ({ {
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 0,
errorMsg: 'chunk first in insert order',
chunkIndex: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 0,
errorMsg: 'parse should sort before chunk',
chunkIndex: 9
},
...[...Array(10).keys()].map((i) => ({
teamId: root.teamId, teamId: root.teamId,
tmbId: root.tmbId, tmbId: root.tmbId,
datasetId: dataset._id, datasetId: dataset._id,
collectionId: collection._id, collectionId: collection._id,
billId: 'test', billId: 'test',
mode: TrainingModeEnum.chunk, mode: TrainingModeEnum.chunk,
errorMsg: 'test' retryCount: 0,
})) errorMsg: 'test',
); chunkIndex: i
})),
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 3,
errorMsg: 'temporary failed'
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 0,
errorMsg: ' '
}
]);
const res = await Call<getTrainingErrorBody, {}, getTrainingErrorResponse>(handler, { const res = await Call<getTrainingErrorBody, EmptyQuery, getTrainingErrorResponse>(handler, {
auth: root, auth: root,
body: { body: {
collectionId: collection._id, collectionId: collection._id,
...@@ -52,7 +98,25 @@ describe('training error list test', () => { ...@@ -52,7 +98,25 @@ describe('training error list test', () => {
}); });
expect(res.code).toBe(200); expect(res.code).toBe(200);
expect(res.data.total).toBe(10); expect(res.data.total).toBe(12);
expect(res.data.list.length).toBe(10); expect(res.data.list.length).toBe(10);
expect(
res.data.list.map((item) => ('mode' in item ? item.mode : undefined)).slice(0, 2)
).toEqual([TrainingModeEnum.parse, TrainingModeEnum.chunk]);
expect(res.data.list.every((item) => 'mode' in item)).toBe(true);
});
it('should reject dataset scope request', async () => {
const root = await getRootUser();
const res = await Call<getTrainingErrorBody, EmptyQuery, getTrainingErrorResponse>(handler, {
auth: root,
body: {
datasetId: '507f1f77bcf86cd799439012',
pageSize: 10,
offset: 0
} as any
});
expect(res.code).not.toBe(200);
}); });
}); });
projects/app/test/api/core/dataset/training/hasDatasetTrainingError.test.ts 0 → 100644
import handler, {
type hasDatasetTrainingErrorQuery,
type hasDatasetTrainingErrorResponse
} from '@/pages/api/core/dataset/training/hasDatasetTrainingError';
import {
DatasetCollectionTypeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { getRootUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest';
type EmptyBody = Record<string, never>;
describe('dataset training error existence test', () => {
it('should only report final or blocked training errors', async () => {
const root = await getRootUser();
const dataset = await MongoDataset.create({
name: 'test',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
});
const collection = await MongoDatasetCollection.create({
name: 'test',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
});
await MongoDatasetTraining.create([
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 3,
errorMsg: 'temporary failed',
chunkIndex: 0
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.qa,
retryCount: 0,
errorMsg: ' ',
chunkIndex: 1
}
]);
const noFinalErrorRes = await Call<
EmptyBody,
hasDatasetTrainingErrorQuery,
hasDatasetTrainingErrorResponse
>(handler, {
auth: root,
query: {
datasetId: dataset._id
}
});
expect(noFinalErrorRes.code).toBe(200);
expect(noFinalErrorRes.data.hasError).toBe(false);
await MongoDatasetTraining.create({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 0,
errorMsg: 'parse failed',
chunkIndex: 2
});
const finalErrorRes = await Call<
EmptyBody,
hasDatasetTrainingErrorQuery,
hasDatasetTrainingErrorResponse
>(handler, {
auth: root,
query: {
datasetId: dataset._id
}
});
expect(finalErrorRes.code).toBe(200);
expect(finalErrorRes.data.hasError).toBe(true);
});
});
projects/app/test/pages/api/core/dataset/training/updateTrainingData.test.ts
import { beforeEach, describe, expect, it, vi } from 'vitest'; import { describe, expect, it, vi, beforeEach } from 'vitest';
import { handler } from '@/pages/api/core/dataset/training/updateTrainingData'; import { handler } from '@/pages/api/core/dataset/training/updateTrainingData';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; import {
authDataset,
authDatasetCollection
} from '@fastgpt/service/support/permission/dataset/auth';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
const datasetId = '507f1f77bcf86cd799439011'; const datasetId = '507f1f77bcf86cd799439011';
...@@ -11,13 +14,14 @@ const foreignDatasetId = '507f1f77bcf86cd799439014'; ...@@ -11,13 +14,14 @@ const foreignDatasetId = '507f1f77bcf86cd799439014';
vi.mock('@fastgpt/service/core/dataset/training/schema', () => ({ vi.mock('@fastgpt/service/core/dataset/training/schema', () => ({
MongoDatasetTraining: { MongoDatasetTraining: {
findOne: vi.fn(), findById: vi.fn(),
updateOne: vi.fn(), updateOne: vi.fn(),
updateMany: vi.fn() updateMany: vi.fn()
} }
})); }));
vi.mock('@fastgpt/service/support/permission/dataset/auth', () => ({ vi.mock('@fastgpt/service/support/permission/dataset/auth', () => ({
authDataset: vi.fn(),
authDatasetCollection: vi.fn() authDatasetCollection: vi.fn()
})); }));
...@@ -30,25 +34,34 @@ describe('updateTrainingData', () => { ...@@ -30,25 +34,34 @@ describe('updateTrainingData', () => {
teamId: 'team1', teamId: 'team1',
datasetId datasetId
} }
}); } as any);
vi.mocked(authDataset).mockResolvedValue({
teamId: 'team1',
dataset: {
_id: datasetId
}
} as any);
}); });
it('should retry all error data when dataId is not provided', async () => { it('should retry only final errors in collection scope', async () => {
const req = { await handler({
body: { body: {
collectionId collectionId
} }
}; } as any);
await handler(req as any);
expect(authDatasetCollection).toHaveBeenCalledWith(
expect.objectContaining({
collectionId
})
);
expect(MongoDatasetTraining.updateMany).toHaveBeenCalledWith( expect(MongoDatasetTraining.updateMany).toHaveBeenCalledWith(
{ expect.objectContaining({
teamId: 'team1', teamId: 'team1',
datasetId, datasetId,
collectionId, collectionId,
errorMsg: { $exists: true, $ne: null } $expr: expect.any(Object)
}, }),
{ {
$unset: { errorMsg: '' }, $unset: { errorMsg: '' },
retryCount: 3, retryCount: 3,
...@@ -57,118 +70,134 @@ describe('updateTrainingData', () => { ...@@ -57,118 +70,134 @@ describe('updateTrainingData', () => {
); );
}); });
it('should update single training data with image', async () => { it('should retry only final errors in dataset scope', async () => {
vi.mocked(MongoDatasetTraining.findOne).mockResolvedValue({ await handler({
imageId: 'image1'
});
const req = {
body: { body: {
collectionId, datasetId
dataId,
q: 'question',
a: 'answer',
chunkIndex: 1
} }
}; } as any);
await handler(req as any);
expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith( expect(authDataset).toHaveBeenCalledWith(
{ expect.objectContaining({
datasetId
})
);
expect(MongoDatasetTraining.updateMany).toHaveBeenCalledWith(
expect.objectContaining({
teamId: 'team1', teamId: 'team1',
datasetId, datasetId,
collectionId, $expr: expect.any(Object)
_id: dataId }),
},
{ {
$unset: { errorMsg: '' }, $unset: { errorMsg: '' },
retryCount: 3, retryCount: 3,
mode: TrainingModeEnum.chunk,
q: 'question',
a: 'answer',
chunkIndex: 1,
lockTime: new Date('2000') lockTime: new Date('2000')
} }
); );
}); });
it('should update single training data without image', async () => { it('should update single training data with collection boundary', async () => {
vi.mocked(MongoDatasetTraining.findOne).mockResolvedValue({}); vi.mocked(MongoDatasetTraining.findById).mockResolvedValue({
_id: dataId,
imageId: 'image1',
teamId: 'team1',
datasetId,
collectionId
});
const req = { await handler({
body: { body: {
collectionId,
dataId, dataId,
q: 'question', q: 'question',
a: 'answer', a: 'answer',
chunkIndex: 1 chunkIndex: 1
} }
}; } as any);
await handler(req as any);
expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith( const match = {
{
teamId: 'team1', teamId: 'team1',
datasetId, datasetId,
collectionId, collectionId,
_id: dataId _id: dataId
}, };
{
expect(MongoDatasetTraining.findById).toHaveBeenCalledWith(dataId);
expect(authDatasetCollection).toHaveBeenCalledWith(
expect.objectContaining({
collectionId
})
);
expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith(match, {
$unset: { errorMsg: '' }, $unset: { errorMsg: '' },
retryCount: 3, retryCount: 3,
mode: TrainingModeEnum.chunk,
q: 'question', q: 'question',
a: 'answer', a: 'answer',
chunkIndex: 1, chunkIndex: 1,
lockTime: new Date('2000') lockTime: new Date('2000')
} });
);
}); });
it('should reject when data not found', async () => { it('should reject when single training data is not found', async () => {
vi.mocked(MongoDatasetTraining.findOne).mockResolvedValue(null); vi.mocked(MongoDatasetTraining.findById).mockResolvedValue(null);
const req = { await expect(
handler({
body: { body: {
collectionId,
dataId dataId
} }
}; } as any)
).rejects.toBe('data not found');
await expect(handler(req as any)).rejects.toBe('data not found');
}); });
it('should ignore legacy request datasetId and use the authorized collection datasetId', async () => { it('should ignore legacy request datasetId and use the item collection datasetId', async () => {
vi.mocked(MongoDatasetTraining.findOne).mockResolvedValue({}); vi.mocked(MongoDatasetTraining.findById).mockResolvedValue({
_id: dataId,
teamId: 'team1',
datasetId,
collectionId
});
const req = { await handler({
body: { body: {
datasetId: foreignDatasetId, datasetId: foreignDatasetId,
collectionId,
dataId, dataId,
q: 'question' q: 'question'
} }
}; } as any);
await handler(req as any);
expect(MongoDatasetTraining.findOne).toHaveBeenCalledWith({ const match = {
teamId: 'team1', teamId: 'team1',
datasetId, datasetId,
collectionId, collectionId,
_id: dataId _id: dataId
}); };
expect(MongoDatasetTraining.findById).toHaveBeenCalledWith(dataId);
expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith( expect(MongoDatasetTraining.updateOne).toHaveBeenCalledWith(
{ match,
teamId: 'team1',
datasetId,
collectionId,
_id: dataId
},
expect.objectContaining({ expect.objectContaining({
q: 'question' q: 'question'
}) })
); );
}); });
it('should reject when the item collection boundary is inconsistent', async () => {
vi.mocked(MongoDatasetTraining.findById).mockResolvedValue({
_id: dataId,
teamId: 'team1',
datasetId: foreignDatasetId,
collectionId
});
await expect(
handler({
body: {
dataId
}
} as any)
).rejects.toBe('data not found');
expect(MongoDatasetTraining.updateOne).not.toHaveBeenCalled();
});
}); });
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment