feat: add training retry time (#3187)

* feat: add training retry time

* remoce log

packages/service/core/dataset/training/schema.ts

@@ -51,6 +51,11 @@ const TrainingDataSchema = new Schema({
     type: Date,
     default: () => new Date('2000/1/1')
   },
+  retryCount: {
+    type: Number,
+    default: 5
+  },
+
   model: {
     // ai model
     type: String,
@@ -97,7 +102,7 @@ try {
   // lock training data(teamId); delete training data
   TrainingDataSchema.index({ teamId: 1, datasetId: 1 });
   // get training data and sort
-  TrainingDataSchema.index({ mode: 1, lockTime: 1, weight: -1 });
+  TrainingDataSchema.index({ mode: 1, retryCount: 1, lockTime: 1, weight: -1 });
   TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 * 60 }); // 7 days
 } catch (error) {
   console.log(error);

projects/app/src/pages/api/core/dataset/update.ts

@@ -9,7 +9,7 @@ import {
 } from '@fastgpt/global/support/permission/constant';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
-import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import { DatasetTypeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
 import { ClientSession } from 'mongoose';
 import { parseParentIdInMongo } from '@fastgpt/global/common/parentFolder/utils';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
@@ -21,6 +21,7 @@ import {
 import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
 import { TeamWritePermissionVal } from '@fastgpt/global/support/permission/user/constant';
 import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
 
 export type DatasetUpdateQuery = {};
 export type DatasetUpdateResponse = any;
@@ -84,6 +85,12 @@ async function handler(
 
   const isFolder = dataset.type === DatasetTypeEnum.folder;
 
+  updateTraining({
+    teamId: dataset.teamId,
+    datasetId: id,
+    agentModel: agentModel?.model
+  });
+
   const onUpdate = async (session?: ClientSession) => {
     await MongoDataset.findByIdAndUpdate(
       id,
@@ -137,3 +144,29 @@ async function handler(
   }
 }
 export default NextAPI(handler);
+
+async function updateTraining({
+  teamId,
+  datasetId,
+  agentModel
+}: {
+  teamId: string;
+  datasetId: string;
+  agentModel?: string;
+}) {
+  if (!agentModel) return;
+
+  await MongoDatasetTraining.updateMany(
+    {
+      teamId,
+      datasetId,
+      mode: { $in: [TrainingModeEnum.qa, TrainingModeEnum.auto] }
+    },
+    {
+      $set: {
+        model: agentModel,
+        retryCount: 5
+      }
+    }
+  );
+}

projects/app/src/service/events/generateQA.ts

@@ -39,11 +39,13 @@ export async function generateQA(): Promise<any> {
     try {
       const data = await MongoDatasetTraining.findOneAndUpdate(
         {
-          lockTime: { $lte: addMinutes(new Date(), -6) },
-          mode: TrainingModeEnum.qa
+          mode: TrainingModeEnum.qa,
+          retryCount: { $gte: 0 },
+          lockTime: { $lte: addMinutes(new Date(), -6) }
         },
         {
-          lockTime: new Date()
+          lockTime: new Date(),
+          $inc: { retryCount: -1 }
         }
       )
         .select({

projects/app/src/service/events/generateVector.ts

@@ -39,10 +39,12 @@ export async function generateVector(): Promise<any> {
       const data = await MongoDatasetTraining.findOneAndUpdate(
         {
           mode: TrainingModeEnum.chunk,
-          lockTime: { $lte: addMinutes(new Date(), -1) }
+          retryCount: { $gte: 0 },
+          lockTime: { $lte: addMinutes(new Date(), -6) }
         },
         {
-          lockTime: new Date()
+          lockTime: new Date(),
+          $inc: { retryCount: -1 }
         }
       ).select({
         _id: 1,