feat: 模型数据导入

2b2c70e5 · archer · f32c557b · 2b2c70e5 · 2b2c70e5 · 2b2c70e5
Commit 2b2c70e5 authored Mar 30, 2023 by archer
16 changed files
--- a/src/api/model.ts
+++ b/src/api/model.ts
 import { GET, POST, DELETE, PUT } from './request';
-import type { ModelSchema } from '@/types/mongoSchema';
+import type { ModelSchema, ModelDataSchema } from '@/types/mongoSchema';
 import { ModelUpdateParams } from '@/types/model';
 import { TrainingItemType } from '../types/training';
 import { PagingData } from '@/types';
@@ -39,10 +39,15 @@ type GetModelDataListProps = RequestPaging & {
 export const getModelDataList = (props: GetModelDataListProps) =>
  GET(`/model/data/getModelData?${Obj2Query(props)}`);
-export const postModelData = (data: { modelId: string; data: { q: string; a: string }[] }) =>
+export const postModelDataInput = (data: {
-  POST(`/model/data/pushModelData`, data);
+  modelId: string;
+  data: { text: ModelDataSchema['text']; q: ModelDataSchema['q'] }[];
+}) => POST(`/model/data/pushModelDataInput`, data);
+export const postModelDataSelect = (modelId: string, dataIds: string[]) =>
+  POST(`/model/data/pushModelDataSelectData`, { modelId, dataIds });
-export const putModelDataById = (data: { modelId: string; answer: string }) =>
+export const putModelDataById = (data: { dataId: string; text: string }) =>
  PUT('/model/data/putModelData', data);
-export const DelOneModelData = (modelId: string) =>
+export const delOneModelData = (dataId: string) =>
-  DELETE(`/model/data/delModelDataById?modelId=${modelId}`);
+  DELETE(`/model/data/delModelDataById?dataId=${dataId}`);
--- a/src/constants/model.ts
+++ b/src/constants/model.ts
@@ -75,10 +75,9 @@ export const formatModelStatus = {
  }
 };
-export const ModelDataStatusMap: Record<ModelDataType, string> = {
+export const ModelDataStatusMap = {
  0: '训练完成',
-  1: '等待训练',
+  1: '训练中'
-  2: '训练中'
 };
 export const defaultModel: ModelSchema = {

--- a/src/constants/redis.ts
+++ b/src/constants/redis.ts
+export const ModelDataIndex = 'model:data';
+export const VecModelDataIndex = 'vec:model:data';
--- a/src/hooks/usePaging.ts
+++ b/src/hooks/usePaging.ts
@@ -75,6 +75,7 @@ export const usePaging = <T = any>({
    requesting,
    isLoadAll,
    nextPage,
-    initRequesting
+    initRequesting,
+    setData
  };
 };
--- a/src/pages/api/model/data/delModelDataById.ts
+++ b/src/pages/api/model/data/delModelDataById.ts
@@ -5,8 +5,8 @@ import { authToken } from '@/service/utils/tools';
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
-    let { modelId } = req.query as {
+    let { dataId } = req.query as {
-      modelId: string;
+      dataId: string;
    };
    const { authorization } = req.headers;
@@ -14,7 +14,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
      throw new Error('无权操作');
    }
-    if (!modelId) {
+    if (!dataId) {
      throw new Error('缺少参数');
    }
@@ -24,7 +24,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    await connectToDatabase();
    await ModelData.deleteOne({
-      modelId,
+      _id: dataId,
      userId
    });

--- a/src/pages/api/model/data/pushModelData.ts
+++ b/src/pages/api/model/data/pushModelData.ts
@@ -2,12 +2,13 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, ModelData, Model } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
+import { ModelDataSchema } from '@/types/mongoSchema';
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
    const { modelId, data } = req.body as {
      modelId: string;
-      data: { q: string; a: string }[];
+      data: { text: ModelDataSchema['text']; q: ModelDataSchema['q'] }[];
    };
    const { authorization } = req.headers;

--- a/src/pages/api/model/data/pushModelDataSelectData.ts
+++ b/src/pages/api/model/data/pushModelDataSelectData.ts
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, DataItem, ModelData } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import { customAlphabet } from 'nanoid';
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    let { dataIds, modelId } = req.body as { dataIds: string[]; modelId: string };
+    if (!dataIds) {
+      throw new Error('参数错误');
+    }
+    await connectToDatabase();
+    const { authorization } = req.headers;
+    const userId = await authToken(authorization);
+    const dataItems = (
+      await Promise.all(
+        dataIds.map((dataId) =>
+          DataItem.find<{ _id: string; result: { q: string }[]; text: string }>(
+            {
+              userId,
+              dataId
+            },
+            'result text'
+          )
+        )
+      )
+    ).flat();
+    // push data
+    await ModelData.insertMany(
+      dataItems.map((item) => ({
+        modelId: modelId,
+        userId,
+        text: item.text,
+        q: item.result.map((item) => ({
+          id: nanoid(),
+          text: item.q
+        }))
+      }))
+    );
+    jsonRes(res, {
+      data: dataItems
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}
--- a/src/pages/api/model/data/putModelData.ts
+++ b/src/pages/api/model/data/putModelData.ts
@@ -5,9 +5,9 @@ import { authToken } from '@/service/utils/tools';
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
-    let { modelId, answer } = req.body as {
+    let { dataId, text } = req.body as {
-      modelId: string;
+      dataId: string;
-      answer: string;
+      text: string;
    };
    const { authorization } = req.headers;
@@ -15,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
      throw new Error('无权操作');
    }
-    if (!modelId) {
+    if (!dataId) {
      throw new Error('缺少参数');
    }
@@ -26,11 +26,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    await ModelData.updateOne(
      {
-        modelId,
+        _id: dataId,
        userId
      },
      {
-        a: answer
+        text
      }
    );

--- a/src/pages/model/components/ModelDataCard.tsx
+++ b/src/pages/model/components/ModelDataCard.tsx
--- a/src/pages/model/detail.tsx
+++ b/src/pages/model/detail.tsx
@@ -255,7 +255,7 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
            <Training model={model} />
          </Card>
        )} */}
-        <Card p={4} height={'400px'} gridColumnStart={1} gridColumnEnd={3}>
+        <Card p={4} height={'500px'} gridColumnStart={1} gridColumnEnd={3}>
          {model._id && <ModelDataCard model={model} />}
        </Card>
      </Grid>

--- a/src/service/models/modelData.ts
+++ b/src/service/models/modelData.ts
@@ -13,22 +13,23 @@ const ModelDataSchema = new Schema({
    ref: 'user',
    required: true
  },
-  q: {
+  text: {
    type: String,
    required: true
  },
-  a: {
+  q: {
-    type: String,
+    type: [
-    default: ''
+      {
+        id: String, // 对应redis的key
+        text: String
+      }
+    ],
+    default: []
  },
  status: {
    type: Number,
-    enum: [0, 1, 2],
+    enum: [0, 1], // 1 训练ing
    default: 1
-  },
-  createTime: {
-    type: Date,
-    default: () => new Date()
  }
 });

--- a/src/service/mongo.ts
+++ b/src/service/mongo.ts
 import mongoose from 'mongoose';
 import { generateQA } from './events/generateQA';
 import { generateAbstract } from './events/generateAbstract';
-import { connectRedis } from './redis';
 /**
 * 连接 MongoDB 数据库
@@ -29,7 +28,6 @@ export async function connectToDatabase(): Promise<void> {
  generateQA();
  generateAbstract();
-  // connectRedis();
 }
 export * from './models/authCode';

--- a/src/service/redis.ts
+++ b/src/service/redis.ts
 import { createClient, SchemaFieldTypes } from 'redis';
+import { ModelDataIndex } from '@/constants/redis';
+import { customAlphabet } from 'nanoid';
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 10);
 export const connectRedis = async () => {
  // 断开了，重连
@@ -6,12 +9,12 @@ export const connectRedis = async () => {
    await global.redisClient.disconnect();
  } else if (global.redisClient) {
    // 没断开，不再连接
-    return;
+    return global.redisClient;
  }
  try {
    global.redisClient = createClient({
-      url: 'redis://:121914yu@120.76.193.200:8100'
+      url: 'redis://default:121914yu@120.76.193.200:8100'
    });
    global.redisClient.on('error', (err) => {
@@ -31,10 +34,11 @@ export const connectRedis = async () => {
    await global.redisClient.select(0);
    // 创建索引
+    try {
      await global.redisClient.ft.create(
-      'vec:question',
+        ModelDataIndex,
        {
-        '$.vector': SchemaFieldTypes.VECTOR,
+          // '$.vector': SchemaFieldTypes.VECTOR,
          '$.modelId': {
            type: SchemaFieldTypes.TEXT,
            AS: 'modelId'
@@ -50,12 +54,15 @@ export const connectRedis = async () => {
        },
        {
          ON: 'JSON',
-        PREFIX: 'fastgpt:modeldata'
+          PREFIX: 'model:data'
        }
      );
+    } catch (error) {
+      console.log('创建索引失败', error);
+    }
-    // await global.redisClient.json.set('fastgpt:modeldata:1', '$', {
+    // await global.redisClient.json.set('fastgpt:modeldata:2', '$', {
-    //   vector: [],
+    //   vector: [124, 214, 412, 4, 124, 1, 4, 1, 4, 3, 423],
    //   modelId: 'daf',
    //   userId: 'adfd',
    //   q: 'fasf',
@@ -63,12 +70,17 @@ export const connectRedis = async () => {
    //   status: 0,
    //   createTime: new Date()
    // });
-    // const value = await global.redisClient.get('fastgpt:modeldata:1');
+    // const value = await global.redisClient.json.get('fastgpt:modeldata:2');
    // console.log(value);
+    return global.redisClient;
  } catch (error) {
    console.log(error, '==');
    global.redisClient = null;
    return Promise.reject('redis 连接失败');
  }
 };
+export const getKey = (prefix = '') => {
+  return `${prefix}:${nanoid()}`;
+};
--- a/src/types/model.d.ts
+++ b/src/types/model.d.ts
@@ -8,3 +8,12 @@ export interface ModelUpdateParams {
  service: ModelSchema.service;
  security: ModelSchema.security;
 }
+export interface ModelDataItemType {
+  id: string;
+  status: 0 | 1; // 1代表向量生成完毕
+  q: string; // 提问词
+  a: string; // 原文
+  modelId: string;
+  userId: string;
+}
--- a/src/types/mongoSchema.d.ts
+++ b/src/types/mongoSchema.d.ts
@@ -51,13 +51,17 @@ export interface ModelPopulate extends ModelSchema {
  userId: UserModelSchema;
 }
-export type ModelDataType = 0 | 1 | 2;
+export type ModelDataType = 0 | 1;
 export interface ModelDataSchema {
  _id: string;
-  q: string;
+  modelId: string;
-  a: string;
+  userId: string;
+  text: string;
+  q: {
+    id: string;
+    text: string;
+  }[];
  status: ModelDataType;
-  createTime: Date;
 }
 export interface TrainingSchema {

--- a/src/types/redis.d.ts
+++ b/src/types/redis.d.ts
+export interface RedisModelDataItemType {
+  id: string;
+  value: {
+    vector: number[];
+    q: string; // 提问词
+    a: string; // 原文
+    modelId: string;
+    userId: string;
+  };
+}