feat: 模型数据导入

src/api/model.ts

 import { GET, POST, DELETE, PUT } from './request';
-import type { ModelSchema } from '@/types/mongoSchema';
+import type { ModelSchema, ModelDataSchema } from '@/types/mongoSchema';
 import { ModelUpdateParams } from '@/types/model';
 import { TrainingItemType } from '../types/training';
 import { PagingData } from '@/types';
@@ -39,10 +39,15 @@ type GetModelDataListProps = RequestPaging & {
 export const getModelDataList = (props: GetModelDataListProps) =>
   GET(`/model/data/getModelData?${Obj2Query(props)}`);
 
-export const postModelData = (data: { modelId: string; data: { q: string; a: string }[] }) =>
-  POST(`/model/data/pushModelData`, data);
+export const postModelDataInput = (data: {
+  modelId: string;
+  data: { text: ModelDataSchema['text']; q: ModelDataSchema['q'] }[];
+}) => POST(`/model/data/pushModelDataInput`, data);
+
+export const postModelDataSelect = (modelId: string, dataIds: string[]) =>
+  POST(`/model/data/pushModelDataSelectData`, { modelId, dataIds });
 
-export const putModelDataById = (data: { modelId: string; answer: string }) =>
+export const putModelDataById = (data: { dataId: string; text: string }) =>
   PUT('/model/data/putModelData', data);
-export const DelOneModelData = (modelId: string) =>
-  DELETE(`/model/data/delModelDataById?modelId=${modelId}`);
+export const delOneModelData = (dataId: string) =>
+  DELETE(`/model/data/delModelDataById?dataId=${dataId}`);

src/constants/model.ts

@@ -75,10 +75,9 @@ export const formatModelStatus = {
   }
 };
 
-export const ModelDataStatusMap: Record<ModelDataType, string> = {
+export const ModelDataStatusMap = {
   0: '训练完成',
-  1: '等待训练',
-  2: '训练中'
+  1: '训练中'
 };
 
 export const defaultModel: ModelSchema = {

src/constants/redis.ts

+export const ModelDataIndex = 'model:data';
+export const VecModelDataIndex = 'vec:model:data';

src/hooks/usePaging.ts

@@ -75,6 +75,7 @@ export const usePaging = <T = any>({
     requesting,
     isLoadAll,
     nextPage,
-    initRequesting
+    initRequesting,
+    setData
   };
 };

src/pages/api/model/data/delModelDataById.ts

@@ -5,8 +5,8 @@ import { authToken } from '@/service/utils/tools';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    let { modelId } = req.query as {
-      modelId: string;
+    let { dataId } = req.query as {
+      dataId: string;
     };
     const { authorization } = req.headers;
 
@@ -14,7 +14,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       throw new Error('无权操作');
     }
 
-    if (!modelId) {
+    if (!dataId) {
       throw new Error('缺少参数');
     }
 
@@ -24,7 +24,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     await connectToDatabase();
 
     await ModelData.deleteOne({
-      modelId,
+      _id: dataId,
       userId
     });
 

src/pages/api/model/data/pushModelData.ts → src/pages/api/model/data/pushModelDataInput.ts

@@ -2,12 +2,13 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, ModelData, Model } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
+import { ModelDataSchema } from '@/types/mongoSchema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     const { modelId, data } = req.body as {
       modelId: string;
-      data: { q: string; a: string }[];
+      data: { text: ModelDataSchema['text']; q: ModelDataSchema['q'] }[];
     };
     const { authorization } = req.headers;
 

src/pages/api/model/data/pushModelDataSelectData.ts

+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, DataItem, ModelData } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import { customAlphabet } from 'nanoid';
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    let { dataIds, modelId } = req.body as { dataIds: string[]; modelId: string };
+
+    if (!dataIds) {
+      throw new Error('参数错误');
+    }
+    await connectToDatabase();
+
+    const { authorization } = req.headers;
+
+    const userId = await authToken(authorization);
+
+    const dataItems = (
+      await Promise.all(
+        dataIds.map((dataId) =>
+          DataItem.find<{ _id: string; result: { q: string }[]; text: string }>(
+            {
+              userId,
+              dataId
+            },
+            'result text'
+          )
+        )
+      )
+    ).flat();
+
+    // push data
+    await ModelData.insertMany(
+      dataItems.map((item) => ({
+        modelId: modelId,
+        userId,
+        text: item.text,
+        q: item.result.map((item) => ({
+          id: nanoid(),
+          text: item.q
+        }))
+      }))
+    );
+
+    jsonRes(res, {
+      data: dataItems
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/data/putModelData.ts

@@ -5,9 +5,9 @@ import { authToken } from '@/service/utils/tools';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    let { modelId, answer } = req.body as {
-      modelId: string;
-      answer: string;
+    let { dataId, text } = req.body as {
+      dataId: string;
+      text: string;
     };
     const { authorization } = req.headers;
 
@@ -15,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       throw new Error('无权操作');
     }
 
-    if (!modelId) {
+    if (!dataId) {
       throw new Error('缺少参数');
     }
 
@@ -26,11 +26,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     await ModelData.updateOne(
       {
-        modelId,
+        _id: dataId,
         userId
       },
       {
-        a: answer
+        text
       }
     );
 

src/pages/model/detail.tsx

@@ -255,7 +255,7 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
             <Training model={model} />
           </Card>
         )} */}
-        <Card p={4} height={'400px'} gridColumnStart={1} gridColumnEnd={3}>
+        <Card p={4} height={'500px'} gridColumnStart={1} gridColumnEnd={3}>
           {model._id && <ModelDataCard model={model} />}
         </Card>
       </Grid>

src/service/models/modelData.ts

@@ -13,22 +13,23 @@ const ModelDataSchema = new Schema({
     ref: 'user',
     required: true
   },
-  q: {
+  text: {
     type: String,
     required: true
   },
-  a: {
-    type: String,
-    default: ''
+  q: {
+    type: [
+      {
+        id: String, // 对应redis的key
+        text: String
+      }
+    ],
+    default: []
   },
   status: {
     type: Number,
-    enum: [0, 1, 2],
+    enum: [0, 1], // 1 训练ing
     default: 1
-  },
-  createTime: {
-    type: Date,
-    default: () => new Date()
   }
 });
 

src/service/mongo.ts

 import mongoose from 'mongoose';
 import { generateQA } from './events/generateQA';
 import { generateAbstract } from './events/generateAbstract';
-import { connectRedis } from './redis';
 
 /**
  * 连接 MongoDB 数据库
@@ -29,7 +28,6 @@ export async function connectToDatabase(): Promise<void> {
 
   generateQA();
   generateAbstract();
-  // connectRedis();
 }
 
 export * from './models/authCode';

src/service/redis.ts

 import { createClient, SchemaFieldTypes } from 'redis';
+import { ModelDataIndex } from '@/constants/redis';
+import { customAlphabet } from 'nanoid';
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 10);
 
 export const connectRedis = async () => {
   // 断开了，重连
@@ -6,12 +9,12 @@ export const connectRedis = async () => {
     await global.redisClient.disconnect();
   } else if (global.redisClient) {
     // 没断开，不再连接
-    return;
+    return global.redisClient;
   }
 
   try {
     global.redisClient = createClient({
-      url: 'redis://:121914yu@120.76.193.200:8100'
+      url: 'redis://default:121914yu@120.76.193.200:8100'
     });
 
     global.redisClient.on('error', (err) => {
@@ -31,31 +34,35 @@ export const connectRedis = async () => {
     await global.redisClient.select(0);
 
     // 创建索引
-    await global.redisClient.ft.create(
-      'vec:question',
-      {
-        '$.vector': SchemaFieldTypes.VECTOR,
-        '$.modelId': {
-          type: SchemaFieldTypes.TEXT,
-          AS: 'modelId'
+    try {
+      await global.redisClient.ft.create(
+        ModelDataIndex,
+        {
+          // '$.vector': SchemaFieldTypes.VECTOR,
+          '$.modelId': {
+            type: SchemaFieldTypes.TEXT,
+            AS: 'modelId'
+          },
+          '$.userId': {
+            type: SchemaFieldTypes.TEXT,
+            AS: 'userId'
+          },
+          '$.status': {
+            type: SchemaFieldTypes.NUMERIC,
+            AS: 'status'
+          }
         },
-        '$.userId': {
-          type: SchemaFieldTypes.TEXT,
-          AS: 'userId'
-        },
-        '$.status': {
-          type: SchemaFieldTypes.NUMERIC,
-          AS: 'status'
+        {
+          ON: 'JSON',
+          PREFIX: 'model:data'
         }
-      },
-      {
-        ON: 'JSON',
-        PREFIX: 'fastgpt:modeldata'
-      }
-    );
+      );
+    } catch (error) {
+      console.log('创建索引失败', error);
+    }
 
-    // await global.redisClient.json.set('fastgpt:modeldata:1', '$', {
-    //   vector: [],
+    // await global.redisClient.json.set('fastgpt:modeldata:2', '$', {
+    //   vector: [124, 214, 412, 4, 124, 1, 4, 1, 4, 3, 423],
     //   modelId: 'daf',
     //   userId: 'adfd',
     //   q: 'fasf',
@@ -63,12 +70,17 @@ export const connectRedis = async () => {
     //   status: 0,
     //   createTime: new Date()
     // });
-    // const value = await global.redisClient.get('fastgpt:modeldata:1');
+    // const value = await global.redisClient.json.get('fastgpt:modeldata:2');
 
     // console.log(value);
+    return global.redisClient;
   } catch (error) {
     console.log(error, '==');
     global.redisClient = null;
     return Promise.reject('redis 连接失败');
   }
 };
+
+export const getKey = (prefix = '') => {
+  return `${prefix}:${nanoid()}`;
+};

src/types/model.d.ts

@@ -8,3 +8,12 @@ export interface ModelUpdateParams {
   service: ModelSchema.service;
   security: ModelSchema.security;
 }
+
+export interface ModelDataItemType {
+  id: string;
+  status: 0 | 1; // 1代表向量生成完毕
+  q: string; // 提问词
+  a: string; // 原文
+  modelId: string;
+  userId: string;
+}

src/types/mongoSchema.d.ts

@@ -51,13 +51,17 @@ export interface ModelPopulate extends ModelSchema {
   userId: UserModelSchema;
 }
 
-export type ModelDataType = 0 | 1 | 2;
+export type ModelDataType = 0 | 1;
 export interface ModelDataSchema {
   _id: string;
-  q: string;
-  a: string;
+  modelId: string;
+  userId: string;
+  text: string;
+  q: {
+    id: string;
+    text: string;
+  }[];
   status: ModelDataType;
-  createTime: Date;
 }
 
 export interface TrainingSchema {

src/types/redis.d.ts

+export interface RedisModelDataItemType {
+  id: string;
+  value: {
+    vector: number[];
+    q: string; // 提问词
+    a: string; // 原文
+    modelId: string;
+    userId: string;
+  };
+}