Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 2eeea151 by light5980 Committed by GitHub
Options

debug: training count error (#7169)

parent b46748e1
Showing with 343 additions and 24 deletions
projects/app/src/pageComponents/dataset/detail/CollectionCard/trainingStatesUtils.ts 0 → 100644
import type { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import type { GetCollectionTrainingDetailResponseType } from '@fastgpt/global/openapi/core/dataset/collection/api';
export enum TrainingStatus {
NotStart = 'NotStart',
Queued = 'Queued',
Running = 'Running',
Ready = 'Ready',
Error = 'Error'
}
/**
* 训练进度弹窗只有未开始阶段置灰；已完成、排队中、处理中和异常阶段都需要保持视觉激活。
*/
export const isTrainingStepHighlighted = (status: TrainingStatus) =>
status !== TrainingStatus.NotStart;
/**
* 判断当前集合是否已经没有任何剩余训练或最终异常。
*/
export const isTrainingDetailReady = (trainingDetail: GetCollectionTrainingDetailResponseType) =>
Object.values(trainingDetail.queuedCounts).every((count) => count === 0) &&
Object.values(trainingDetail.trainingCounts).every((count) => count === 0) &&
Object.values(trainingDetail.errorCounts).every((count) => count === 0);
/**
* 根据当前集合各训练阶段的计数计算单个阶段的展示状态。
* 已进入后续阶段时，前序阶段展示为完成；仍被前序阶段阻塞时，后续阶段保持未开始。
*/
export const getTrainingStepStatus = ({
trainingDetail,
mode,
modeOrder
}: {
trainingDetail: GetCollectionTrainingDetailResponseType;
mode: TrainingModeEnum;
modeOrder: TrainingModeEnum[];
}) => {
if (isTrainingDetailReady(trainingDetail)) return TrainingStatus.Ready;
if (trainingDetail.errorCounts[mode] > 0) return TrainingStatus.Error;
if (trainingDetail.trainingCounts[mode] > 0) return TrainingStatus.Running;
if (trainingDetail.queuedCounts[mode] > 0) return TrainingStatus.Queued;
const modeIndex = modeOrder.indexOf(mode);
if (modeIndex === -1) return TrainingStatus.NotStart;
const hasLaterProgress = modeOrder.slice(modeIndex + 1).some((nextMode) => {
return (
trainingDetail.queuedCounts[nextMode] > 0 ||
trainingDetail.trainingCounts[nextMode] > 0 ||
trainingDetail.errorCounts[nextMode] > 0
);
});
return hasLaterProgress ? TrainingStatus.Ready : TrainingStatus.NotStart;
};
projects/app/src/pages/api/core/dataset/collection/trainingDetail.ts
...@@ -14,9 +14,9 @@ import { ...@@ -14,9 +14,9 @@ import {
} from '@fastgpt/global/openapi/core/dataset/collection/api'; } from '@fastgpt/global/openapi/core/dataset/collection/api';
import { import {
BLOCKED_LOCK_TIME, BLOCKED_LOCK_TIME,
activeTrainingMatch,
finalErrorTrainingMatch finalErrorTrainingMatch
} from '@fastgpt/service/core/dataset/training/query'; } from '@fastgpt/service/core/dataset/training/query';
import { subMinutes } from 'date-fns';
const defaultCounts: Record<TrainingModeEnum, number> = { const defaultCounts: Record<TrainingModeEnum, number> = {
parse: 0, parse: 0,
...@@ -27,6 +27,15 @@ const defaultCounts: Record<TrainingModeEnum, number> = { ...@@ -27,6 +27,15 @@ const defaultCounts: Record<TrainingModeEnum, number> = {
imageParse: 0 imageParse: 0
}; };
const MODE_LOCK_TIMEOUT_MINUTES: Record<TrainingModeEnum, number> = {
parse: 10,
qa: 10,
chunk: 3,
image: 10,
auto: 10,
imageParse: 10
};
async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetailResponseType> { async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetailResponseType> {
const { collectionId } = parseApiInput({ const { collectionId } = parseApiInput({
req, req,
...@@ -47,34 +56,38 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai ...@@ -47,34 +56,38 @@ async function handler(req: ApiRequestProps): Promise<GetCollectionTrainingDetai
collectionId: new Types.ObjectId(collection._id) collectionId: new Types.ObjectId(collection._id)
}; };
// Computed global queue const now = new Date();
const minId = ( const activeTrainingExpr = Object.entries(MODE_LOCK_TIMEOUT_MINUTES).map(
await MongoDatasetTraining.findOne(match, { sort: { _id: 1 }, select: '_id' }).lean() ([mode, timeoutMinutes]) => ({
)?._id; mode,
lockTime: { $gt: subMinutes(now, timeoutMinutes), $lt: BLOCKED_LOCK_TIME }
})
);
const [ququedCountData, trainingCountData, errorCountData, trainedCount] = (await Promise.all([ const [ququedCountData, trainingCountData, errorCountData, trainedCount] = (await Promise.all([
minId
? MongoDatasetTraining.aggregate([
{
$match: {
_id: { $lt: new Types.ObjectId(minId) },
retryCount: { $gt: 0 },
lockTime: { $lt: BLOCKED_LOCK_TIME }
}
},
{
$group: {
_id: '$mode',
count: { $sum: 1 }
}
}
])
: Promise.resolve([]),
MongoDatasetTraining.aggregate([ MongoDatasetTraining.aggregate([
{ {
$match: { $match: {
...match, ...match,
...activeTrainingMatch retryCount: { $gt: 0 },
lockTime: { $lt: BLOCKED_LOCK_TIME },
// 只统计当前集合里未被 worker 领取或锁超时后可重试的任务，避免跨知识库队列污染状态展示。
$nor: activeTrainingExpr
}
},
{
$group: {
_id: '$mode',
count: { $sum: 1 }
}
}
]),
MongoDatasetTraining.aggregate([
{
$match: {
...match,
retryCount: { $gt: 0 },
$or: activeTrainingExpr
} }
}, },
{ {
......
projects/app/test/api/core/dataset/collection/trainingStatus.test.ts
...@@ -94,7 +94,7 @@ describe('collection training status api', () => { ...@@ -94,7 +94,7 @@ describe('collection training status api', () => {
}); });
}); });
it('should use active counts for progress and final errors for error tab', async () => { it('should split current collection queued/running counts and final errors', async () => {
const root = await getRootUser(); const root = await getRootUser();
const dataset = await MongoDataset.create({ const dataset = await MongoDataset.create({
name: 'test', name: 'test',
...@@ -120,6 +120,7 @@ describe('collection training status api', () => { ...@@ -120,6 +120,7 @@ describe('collection training status api', () => {
billId: 'test', billId: 'test',
mode: TrainingModeEnum.qa, mode: TrainingModeEnum.qa,
retryCount: 3, retryCount: 3,
lockTime: new Date(),
errorMsg: 'temporary failed' errorMsg: 'temporary failed'
}, },
{ {
...@@ -128,6 +129,16 @@ describe('collection training status api', () => { ...@@ -128,6 +129,16 @@ describe('collection training status api', () => {
datasetId: dataset._id, datasetId: dataset._id,
collectionId: collection._id, collectionId: collection._id,
billId: 'test', billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 3,
lockTime: new Date('2000')
},
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.chunk, mode: TrainingModeEnum.chunk,
retryCount: 0, retryCount: 0,
errorMsg: 'final failed' errorMsg: 'final failed'
...@@ -142,11 +153,84 @@ describe('collection training status api', () => { ...@@ -142,11 +153,84 @@ describe('collection training status api', () => {
}); });
expect(res.code).toBe(200); expect(res.code).toBe(200);
expect(res.data.queuedCounts.parse).toBe(1);
expect(res.data.trainingCounts.parse).toBe(0);
expect(res.data.queuedCounts.qa).toBe(0);
expect(res.data.trainingCounts.qa).toBe(1); expect(res.data.trainingCounts.qa).toBe(1);
expect(res.data.errorCounts.qa).toBe(0); expect(res.data.errorCounts.qa).toBe(0);
expect(res.data.errorCounts.chunk).toBe(1); expect(res.data.errorCounts.chunk).toBe(1);
}); });
it('should not include other dataset training records in collection queued counts', async () => {
const root = await getRootUser();
const [dataset, otherDataset] = await MongoDataset.create([
{
name: 'current',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
},
{
name: 'other',
teamId: root.teamId,
tmbId: root.tmbId,
vectorModel: 'test',
agentModel: 'test'
}
]);
const [collection, otherCollection] = await MongoDatasetCollection.create([
{
name: 'current',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id
},
{
name: 'other',
type: DatasetCollectionTypeEnum.file,
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: otherDataset._id
}
]);
await MongoDatasetTraining.create([
...Array.from({ length: 6 }).map(() => ({
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: otherDataset._id,
collectionId: otherCollection._id,
billId: 'test',
mode: TrainingModeEnum.chunk,
retryCount: 3,
lockTime: new Date('2000')
})),
{
teamId: root.teamId,
tmbId: root.tmbId,
datasetId: dataset._id,
collectionId: collection._id,
billId: 'test',
mode: TrainingModeEnum.parse,
retryCount: 3,
lockTime: new Date('2000')
}
]);
const res = await Call(trainingDetailHandler, {
auth: root,
query: {
collectionId: collection._id
}
});
expect(res.code).toBe(200);
expect(res.data.queuedCounts.parse).toBe(1);
expect(res.data.queuedCounts.chunk).toBe(0);
});
it('should keep deprecated scrollList compatible with the collection list item schema', async () => { it('should keep deprecated scrollList compatible with the collection list item schema', async () => {
const root = await getRootUser(); const root = await getRootUser();
const dataset = await MongoDataset.create({ const dataset = await MongoDataset.create({
......
projects/app/test/pageComponents/dataset/detail/CollectionCard/trainingStatesUtils.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import {
DatasetCollectionDataProcessModeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import type { GetCollectionTrainingDetailResponseType } from '@fastgpt/global/openapi/core/dataset/collection/api';
import {
getTrainingStepStatus,
isTrainingStepHighlighted,
TrainingStatus
} from '@/pageComponents/dataset/detail/CollectionCard/trainingStatesUtils';
const createTrainingDetail = (
overrides: Partial<GetCollectionTrainingDetailResponseType> = {}
): GetCollectionTrainingDetailResponseType => {
const counts = {
parse: 0,
qa: 0,
chunk: 0,
image: 0,
auto: 0,
imageParse: 0
};
return {
trainingType: DatasetCollectionDataProcessModeEnum.chunk,
advancedTraining: {
customPdfParse: false,
imageIndex: false,
autoIndexes: false
},
queuedCounts: { ...counts },
trainingCounts: { ...counts },
errorCounts: { ...counts },
trainedCount: 0,
...overrides
};
};
describe('trainingStatesUtils', () => {
it('should mark parsing step as running while content parsing is active', () => {
const trainingDetail = createTrainingDetail({
trainingCounts: {
parse: 1,
qa: 0,
chunk: 0,
image: 0,
auto: 0,
imageParse: 0
}
});
const modeOrder = [TrainingModeEnum.parse, TrainingModeEnum.chunk];
expect(
getTrainingStepStatus({
trainingDetail,
mode: TrainingModeEnum.parse,
modeOrder
})
).toBe(TrainingStatus.Running);
expect(
getTrainingStepStatus({
trainingDetail,
mode: TrainingModeEnum.chunk,
modeOrder
})
).toBe(TrainingStatus.NotStart);
});
it('should mark parsing step as queued while waiting to be picked by worker', () => {
const trainingDetail = createTrainingDetail({
queuedCounts: {
parse: 1,
qa: 0,
chunk: 0,
image: 0,
auto: 0,
imageParse: 0
}
});
const modeOrder = [TrainingModeEnum.parse, TrainingModeEnum.chunk];
expect(
getTrainingStepStatus({
trainingDetail,
mode: TrainingModeEnum.parse,
modeOrder
})
).toBe(TrainingStatus.Queued);
expect(
getTrainingStepStatus({
trainingDetail,
mode: TrainingModeEnum.chunk,
modeOrder
})
).toBe(TrainingStatus.NotStart);
});
it('should mark earlier steps ready after later steps start', () => {
const trainingDetail = createTrainingDetail({
trainingCounts: {
parse: 0,
qa: 0,
chunk: 1,
image: 0,
auto: 0,
imageParse: 0
},
trainedCount: 1
});
const modeOrder = [TrainingModeEnum.parse, TrainingModeEnum.chunk];
expect(
getTrainingStepStatus({
trainingDetail,
mode: TrainingModeEnum.parse,
modeOrder
})
).toBe(TrainingStatus.Ready);
expect(
getTrainingStepStatus({
trainingDetail,
mode: TrainingModeEnum.chunk,
modeOrder
})
).toBe(TrainingStatus.Running);
});
it('should keep multiple in-progress stages highlighted at the same time', () => {
const trainingDetail = createTrainingDetail({
trainingCounts: {
parse: 1,
qa: 0,
chunk: 2,
image: 0,
auto: 0,
imageParse: 0
}
});
const modeOrder = [TrainingModeEnum.parse, TrainingModeEnum.chunk];
const parseStatus = getTrainingStepStatus({
trainingDetail,
mode: TrainingModeEnum.parse,
modeOrder
});
const chunkStatus = getTrainingStepStatus({
trainingDetail,
mode: TrainingModeEnum.chunk,
modeOrder
});
expect(parseStatus).toBe(TrainingStatus.Running);
expect(chunkStatus).toBe(TrainingStatus.Running);
expect(isTrainingStepHighlighted(parseStatus)).toBe(true);
expect(isTrainingStepHighlighted(chunkStatus)).toBe(true);
});
it('should highlight completed steps and gray out only not-started steps', () => {
expect(isTrainingStepHighlighted(TrainingStatus.Ready)).toBe(true);
expect(isTrainingStepHighlighted(TrainingStatus.Queued)).toBe(true);
expect(isTrainingStepHighlighted(TrainingStatus.Running)).toBe(true);
expect(isTrainingStepHighlighted(TrainingStatus.Error)).toBe(true);
expect(isTrainingStepHighlighted(TrainingStatus.NotStart)).toBe(false);
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment