Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 46eb96c7 by archer
Options

perf: 去除冗余代码

parent 0540c2e4
Showing with 36 additions and 852 deletions
.gitignore
...@@ -34,7 +34,6 @@ yarn-error.log* ...@@ -34,7 +34,6 @@ yarn-error.log*
# typescript # typescript
*.tsbuildinfo *.tsbuildinfo
next-env.d.ts next-env.d.ts
/public/trainData/
/.vscode/ /.vscode/
platform.json platform.json
testApi/ testApi/
\ No newline at end of file
src/api/model.ts
import { GET, POST, DELETE, PUT } from './request'; import { GET, POST, DELETE, PUT } from './request';
import type { ModelSchema, ModelDataSchema } from '@/types/mongoSchema'; import type { ModelSchema, ModelDataSchema } from '@/types/mongoSchema';
import { ModelUpdateParams } from '@/types/model'; import { ModelUpdateParams } from '@/types/model';
import { TrainingItemType } from '../types/training';
import { RequestPaging } from '../types/index'; import { RequestPaging } from '../types/index';
import { Obj2Query } from '@/utils/tools'; import { Obj2Query } from '@/utils/tools';
...@@ -32,21 +31,7 @@ export const getModelById = (id: string) => GET<ModelSchema>(`/model/detail?mode ...@@ -32,21 +31,7 @@ export const getModelById = (id: string) => GET<ModelSchema>(`/model/detail?mode
export const putModelById = (id: string, data: ModelUpdateParams) => export const putModelById = (id: string, data: ModelUpdateParams) =>
PUT(`/model/update?modelId=${id}`, data); PUT(`/model/update?modelId=${id}`, data);
export const postTrainModel = (id: string, form: FormData) =>
POST(`/model/train/train?modelId=${id}`, form, {
headers: {
'content-type': 'multipart/form-data'
}
});
export const putModelTrainingStatus = (id: string) =>
PUT(`/model/train/putTrainStatus?modelId=${id}`);
export const getModelTrainings = (id: string) =>
GET<TrainingItemType[]>(`/model/train/getTrainings?modelId=${id}`);
/* 模型 data */ /* 模型 data */
type GetModelDataListProps = RequestPaging & { type GetModelDataListProps = RequestPaging & {
modelId: string; modelId: string;
searchText: string; searchText: string;
......
src/api/request.ts
import axios, { Method, InternalAxiosRequestConfig, AxiosResponse } from 'axios'; import axios, { Method, InternalAxiosRequestConfig, AxiosResponse } from 'axios';
import { getToken, clearToken } from '@/utils/user'; import { getToken, clearToken } from '@/utils/user';
import { TOKEN_ERROR_CODE } from '@/constants/responseCode'; import { TOKEN_ERROR_CODE } from '@/service/errorCode';
interface ConfigType { interface ConfigType {
headers?: { [key: string]: string }; headers?: { [key: string]: string };
......
src/api/response/chat.d.ts
...@@ -6,7 +6,6 @@ export type InitChatResponse = { ...@@ -6,7 +6,6 @@ export type InitChatResponse = {
modelId: string; modelId: string;
name: string; name: string;
avatar: string; avatar: string;
intro: string;
chatModel: ModelSchema.service.chatModel; // 对话模型名 chatModel: ModelSchema.service.chatModel; // 对话模型名
modelName: ModelSchema.service.modelName; // 底层模型 modelName: ModelSchema.service.modelName; // 底层模型
history: ChatItemType[]; history: ChatItemType[];
......
src/constants/data.ts deleted 100644 → 0
import type { DataType } from '@/types/data';
export const DataTypeTextMap: Record<DataType, string> = {
QA: '问答拆分',
abstract: '摘要总结'
};
src/constants/model.ts
import type { ModelSchema } from '@/types/mongoSchema'; import type { ModelSchema } from '@/types/mongoSchema';
export enum ModelDataStatusEnum {
ready = 'ready',
waiting = 'waiting'
}
export const embeddingModel = 'text-embedding-ada-002'; export const embeddingModel = 'text-embedding-ada-002';
export enum ChatModelEnum { export enum ChatModelEnum {
'GPT35' = 'gpt-3.5-turbo', 'GPT35' = 'gpt-3.5-turbo',
...@@ -53,13 +48,6 @@ export const modelList: ModelConstantsData[] = [ ...@@ -53,13 +48,6 @@ export const modelList: ModelConstantsData[] = [
} }
]; ];
export enum TrainingStatusEnum {
pending = 'pending',
succeed = 'succeed',
errored = 'errored',
canceled = 'canceled'
}
export enum ModelStatusEnum { export enum ModelStatusEnum {
running = 'running', running = 'running',
training = 'training', training = 'training',
...@@ -86,6 +74,11 @@ export const formatModelStatus = { ...@@ -86,6 +74,11 @@ export const formatModelStatus = {
} }
}; };
export enum ModelDataStatusEnum {
ready = 'ready',
waiting = 'waiting'
}
export const ModelDataStatusMap: Record<`${ModelDataStatusEnum}`, string> = { export const ModelDataStatusMap: Record<`${ModelDataStatusEnum}`, string> = {
ready: '训练完成', ready: '训练完成',
waiting: '训练中' waiting: '训练中'
...@@ -126,15 +119,12 @@ export const defaultModel: ModelSchema = { ...@@ -126,15 +119,12 @@ export const defaultModel: ModelSchema = {
avatar: '', avatar: '',
status: ModelStatusEnum.pending, status: ModelStatusEnum.pending,
updateTime: Date.now(), updateTime: Date.now(),
trainingTimes: 0,
systemPrompt: '', systemPrompt: '',
intro: '',
temperature: 5, temperature: 5,
search: { search: {
mode: ModelVectorSearchModeEnum.hightSimilarity mode: ModelVectorSearchModeEnum.hightSimilarity
}, },
service: { service: {
trainId: '',
chatModel: ModelNameEnum.GPT35, chatModel: ModelNameEnum.GPT35,
modelName: ModelNameEnum.GPT35 modelName: ModelNameEnum.GPT35
}, },
......
src/constants/redis.ts deleted 100644 → 0
export const VecModelDataPrefix = 'model:data';
export const VecModelDataIdx = `idx:${VecModelDataPrefix}:hash`;
export enum ModelDataStatusEnum {
ready = 'ready',
waiting = 'waiting'
}
src/constants/responseCode.ts deleted 100644 → 0
export const ERROR_CODE: { [key: number]: string } = {
400: '请求失败',
401: '无权访问',
403: '紧张访问',
404: '请求不存在',
405: '请求方法错误',
406: '请求的格式错误',
410: '资源已删除',
422: '验证错误',
500: '服务器发生错误',
502: '网关错误',
503: '服务器暂时过载或维护',
504: '网关超时'
};
export const TOKEN_ERROR_CODE: { [key: number]: string } = {
506: '请先登录',
507: '请重新登录',
508: '登录已过期'
};
src/pages/api/chat/init.ts
...@@ -53,7 +53,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -53,7 +53,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
modelId: modelId, modelId: modelId,
name: model.name, name: model.name,
avatar: model.avatar, avatar: model.avatar,
intro: model.intro,
modelName: model.service.modelName, modelName: model.service.modelName,
chatModel: model.service.chatModel, chatModel: model.service.chatModel,
history history
......
src/pages/api/data/delData.ts deleted 100644 → 0
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Data, DataItem } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import type { DataListItem } from '@/types/data';
import type { PagingData } from '@/types';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { authorization } = req.headers;
if (!authorization) {
throw new Error('缺少登录凭证');
}
await authToken(authorization);
const { dataId } = req.query as { dataId: string };
if (!dataId) {
throw new Error('缺少参数');
}
await connectToDatabase();
await Data.findByIdAndUpdate(dataId, {
isDeleted: true
});
// 改变 dataItem 状态为 0
await DataItem.updateMany(
{
dataId
},
{
status: 0
}
);
jsonRes<PagingData<DataListItem>>(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/data/getDataItems.ts deleted 100644 → 0
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, DataItem } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
let {
dataId,
pageNum = 1,
pageSize = 10
} = req.query as { dataId: string; pageNum: string; pageSize: string };
pageNum = +pageNum;
pageSize = +pageSize;
if (!dataId) {
throw new Error('参数错误');
}
await connectToDatabase();
const { authorization } = req.headers;
await authToken(authorization);
const dataItems = await DataItem.find({
dataId
})
.sort({ _id: -1 }) // 按照创建时间倒序排列
.skip((pageNum - 1) * pageSize)
.limit(pageSize);
jsonRes(res, {
data: {
pageNum,
pageSize,
data: dataItems,
total: await DataItem.countDocuments({
dataId
})
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/data/getDataList.ts deleted 100644 → 0
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Data, DataItem } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import type { DataListItem } from '@/types/data';
import mongoose from 'mongoose';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { authorization } = req.headers;
if (!authorization) {
throw new Error('缺少登录凭证');
}
const userId = await authToken(authorization);
await connectToDatabase();
const datalist = await Data.aggregate<DataListItem>([
{
$match: {
userId: new mongoose.Types.ObjectId(userId),
isDeleted: false
}
},
{
$sort: { createTime: -1 } // 按照创建时间倒序排列
},
{
$lookup: {
from: 'dataitems',
localField: '_id',
foreignField: 'dataId',
as: 'items'
}
},
{
$addFields: {
totalData: {
$size: '$items' // 统计dataItem的总数
},
trainingData: {
$size: {
$filter: {
input: '$items',
as: 'item',
cond: { $ne: ['$$item.status', 0] } // 统计 status 不为0的数量
}
}
}
}
},
{
$project: {
items: 0 // 不返回 items 字段
}
}
]);
jsonRes(res, {
data: datalist
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/data/postData.ts deleted 100644 → 0
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Data } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import type { DataType } from '@/types/data';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
let { name, type } = req.body as { name: string; type: DataType };
if (!name || !type) {
throw new Error('参数错误');
}
await connectToDatabase();
const { authorization } = req.headers;
const userId = await authToken(authorization);
// 生成 data 集合
const data = await Data.create({
userId,
name,
type
});
jsonRes(res, {
data: data._id
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/data/putDataName.ts deleted 100644 → 0
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Data } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import type { DataListItem } from '@/types/data';
import type { PagingData } from '@/types';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { authorization } = req.headers;
if (!authorization) {
throw new Error('缺少登录凭证');
}
await authToken(authorization);
const { dataId, name } = req.query as { dataId: string; name: string };
if (!dataId || !name) {
throw new Error('缺少参数');
}
await connectToDatabase();
await Data.findByIdAndUpdate(dataId, {
name
});
jsonRes<PagingData<DataListItem>>(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/data/splitData.ts deleted 100644 → 0
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, DataItem, Data } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import { generateQA } from '@/service/events/generateQA';
import { generateAbstract } from '@/service/events/generateAbstract';
import { countChatTokens } from '@/utils/tools';
/* 拆分数据成QA */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { text, dataId } = req.body as { text: string; dataId: string };
if (!text || !dataId) {
throw new Error('参数错误');
}
await connectToDatabase();
const { authorization } = req.headers;
const userId = await authToken(authorization);
const DataRecord = await Data.findById(dataId);
if (!DataRecord) {
throw new Error('找不到数据集');
}
const replaceText = text.replace(/[\\n]+/g, ' ');
// 文本拆分成 chunk
let chunks = replaceText.match(/[^!?.。]+[!?.。]/g) || [];
const dataItems: any[] = [];
let splitText = '';
chunks.forEach((chunk) => {
splitText += chunk;
const tokens = countChatTokens({ messages: [{ role: 'system', content: splitText }] });
if (tokens >= 780) {
dataItems.push({
userId,
dataId,
type: DataRecord.type,
text: splitText,
status: 1
});
splitText = '';
}
});
// 批量插入数据
await DataItem.insertMany(dataItems);
try {
generateQA();
generateAbstract();
} catch (error) {
error;
}
jsonRes(res, {
data: { chunks, replaceText }
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/model/create.ts
...@@ -47,7 +47,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -47,7 +47,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
userId, userId,
status: ModelStatusEnum.running, status: ModelStatusEnum.running,
service: { service: {
trainId: '',
chatModel: Model2ChatModelMap[modelItem.model], // 聊天时用的模型 chatModel: Model2ChatModelMap[modelItem.model], // 聊天时用的模型
modelName: modelItem.model // 最底层的模型，不会变，用于计费等核心操作 modelName: modelItem.model // 最底层的模型，不会变，用于计费等核心操作
} }
......
src/pages/api/model/data/putModelData.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authToken } from '@/service/utils/tools'; import { authToken } from '@/service/utils/tools';
import { ModelDataStatusEnum } from '@/constants/redis'; import { ModelDataStatusEnum } from '@/constants/model';
import { generateVector } from '@/service/events/generateVector'; import { generateVector } from '@/service/events/generateVector';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
......
src/pages/api/model/update.ts
...@@ -8,7 +8,7 @@ import type { ModelUpdateParams } from '@/types/model'; ...@@ -8,7 +8,7 @@ import type { ModelUpdateParams } from '@/types/model';
/* 获取我的模型 */ /* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
const { name, search, service, security, systemPrompt, intro, temperature } = const { name, search, service, security, systemPrompt, temperature } =
req.body as ModelUpdateParams; req.body as ModelUpdateParams;
const { modelId } = req.query as { modelId: string }; const { modelId } = req.query as { modelId: string };
const { authorization } = req.headers; const { authorization } = req.headers;
...@@ -35,10 +35,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -35,10 +35,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
{ {
name, name,
systemPrompt, systemPrompt,
intro,
temperature, temperature,
search, search,
// service,
security security
} }
); );
......
src/pages/chat/index.tsx
...@@ -64,7 +64,6 @@ const Chat = ({ modelId, chatId }: { modelId: string; chatId: string }) => { ...@@ -64,7 +64,6 @@ const Chat = ({ modelId, chatId }: { modelId: string; chatId: string }) => {
modelId, modelId,
name: '', name: '',
avatar: '', avatar: '',
intro: '',
chatModel: '', chatModel: '',
modelName: '', modelName: '',
history: [] history: []
...@@ -517,7 +516,7 @@ const Chat = ({ modelId, chatId }: { modelId: string; chatId: string }) => { ...@@ -517,7 +516,7 @@ const Chat = ({ modelId, chatId }: { modelId: string; chatId: string }) => {
</Flex> </Flex>
</Box> </Box>
))} ))}
{chatData.history.length === 0 && <Empty intro={chatData.intro} />} {chatData.history.length === 0 && <Empty intro={''} />}
</Box> </Box>
{/* 发送区 */} {/* 发送区 */}
<Box m={media('20px auto', '0 auto')} w={'100%'} maxW={media('min(750px, 100%)', 'auto')}> <Box m={media('20px auto', '0 auto')} w={'100%'} maxW={media('min(750px, 100%)', 'auto')}>
......
src/pages/model/detail/components/Training.tsx deleted 100644 → 0
import React, { useEffect, useCallback, useState } from 'react';
import { Box, TableContainer, Table, Thead, Tbody, Tr, Th, Td } from '@chakra-ui/react';
import type { ModelSchema } from '@/types/mongoSchema';
import { getModelTrainings } from '@/api/model';
import type { TrainingItemType } from '@/types/training';
const Training = ({ model }: { model: ModelSchema }) => {
const columns: {
title: string;
key: keyof TrainingItemType;
dataIndex: string;
}[] = [
{
title: '训练ID',
key: 'tuneId',
dataIndex: 'tuneId'
},
{
title: '状态',
key: 'status',
dataIndex: 'status'
}
];
const [records, setRecords] = useState<TrainingItemType[]>([]);
const loadTrainingRecords = useCallback(async (id: string) => {
try {
const res = await getModelTrainings(id);
setRecords(res);
} catch (error) {
console.log('error->', error);
}
}, []);
useEffect(() => {
model._id && loadTrainingRecords(model._id);
}, [loadTrainingRecords, model]);
return (
<>
<Box fontWeight={'bold'} fontSize={'lg'}>
训练记录: {model.trainingTimes}次
</Box>
<TableContainer mt={4}>
<Table variant={'simple'}>
<Thead>
<Tr>
{columns.map((item) => (
<Th key={item.key}>{item.title}</Th>
))}
</Tr>
</Thead>
<Tbody>
{records.map((item) => (
<Tr key={item._id}>
{columns.map((col) => (
// @ts-ignore
<Td key={col.key}>{item[col.dataIndex]}</Td>
))}
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
</>
);
};
export default Training;
src/pages/model/detail/index.tsx
import React, { useCallback, useState, useMemo, useEffect } from 'react'; import React, { useCallback, useState, useMemo, useEffect } from 'react';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { getModelById, delModelById, putModelTrainingStatus, putModelById } from '@/api/model'; import { getModelById, delModelById, putModelById } from '@/api/model';
import type { ModelSchema } from '@/types/mongoSchema'; import type { ModelSchema } from '@/types/mongoSchema';
import { Card, Box, Flex, Button, Tag, Grid } from '@chakra-ui/react'; import { Card, Box, Flex, Button, Tag, Grid } from '@chakra-ui/react';
import { useToast } from '@/hooks/useToast'; import { useToast } from '@/hooks/useToast';
...@@ -76,29 +76,6 @@ const ModelDetail = ({ modelId }: { modelId: string }) => { ...@@ -76,29 +76,6 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
setLoading(false); setLoading(false);
}, [setLoading, router, modelId]); }, [setLoading, router, modelId]);
/* 点击更新模型状态 */
const handleClickUpdateStatus = useCallback(async () => {
if (!model || model.status !== ModelStatusEnum.training) return;
setLoading(true);
try {
const res = await putModelTrainingStatus(model._id);
typeof res === 'string' &&
toast({
title: res,
status: 'info'
});
loadModel();
} catch (error: any) {
console.log('error->', error);
toast({
title: error.message || '更新失败',
status: 'error'
});
}
setLoading(false);
}, [model, setLoading, loadModel, toast]);
// 提交保存模型修改 // 提交保存模型修改
const saveSubmitSuccess = useCallback( const saveSubmitSuccess = useCallback(
async (data: ModelSchema) => { async (data: ModelSchema) => {
...@@ -107,7 +84,6 @@ const ModelDetail = ({ modelId }: { modelId: string }) => { ...@@ -107,7 +84,6 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
await putModelById(data._id, { await putModelById(data._id, {
name: data.name, name: data.name,
systemPrompt: data.systemPrompt, systemPrompt: data.systemPrompt,
intro: data.intro,
temperature: data.temperature, temperature: data.temperature,
search: data.search, search: data.search,
service: data.service, service: data.service,
...@@ -168,13 +144,7 @@ const ModelDetail = ({ modelId }: { modelId: string }) => { ...@@ -168,13 +144,7 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
<Box fontSize={'xl'} fontWeight={'bold'}> <Box fontSize={'xl'} fontWeight={'bold'}>
{model.name} {model.name}
</Box> </Box>
<Tag <Tag ml={2} variant="solid" colorScheme={formatModelStatus[model.status].colorTheme}>
ml={2}
variant="solid"
colorScheme={formatModelStatus[model.status].colorTheme}
cursor={model.status === ModelStatusEnum.training ? 'pointer' : 'default'}
onClick={handleClickUpdateStatus}
>
{formatModelStatus[model.status].text} {formatModelStatus[model.status].text}
</Tag> </Tag>
<Box flex={1} /> <Box flex={1} />
......
src/service/errorCode.ts
export const ERROR_CODE: { [key: number]: string } = {
400: '请求失败',
401: '无权访问',
403: '紧张访问',
404: '请求不存在',
405: '请求方法错误',
406: '请求的格式错误',
410: '资源已删除',
422: '验证错误',
500: '服务器发生错误',
502: '网关错误',
503: '服务器暂时过载或维护',
504: '网关超时'
};
export const TOKEN_ERROR_CODE: { [key: number]: string } = {
506: '请先登录',
507: '请重新登录',
508: '登录已过期'
};
export const openaiError: Record<string, string> = { export const openaiError: Record<string, string> = {
context_length_exceeded: '内容超长了，请重置对话', context_length_exceeded: '内容超长了，请重置对话',
Unauthorized: 'API-KEY 不合法', Unauthorized: 'API-KEY 不合法',
......
src/service/events/generateAbstract.ts deleted 100644 → 0
import { DataItem } from '@/service/mongo';
import { getOpenAIApi } from '@/service/utils/auth';
import { axiosConfig } from '@/service/utils/tools';
import { getOpenApiKey } from '../utils/openai';
import type { ChatCompletionRequestMessage } from 'openai';
import { DataItemSchema } from '@/types/mongoSchema';
import { ChatModelEnum } from '@/constants/model';
import { pushSplitDataBill } from '@/service/events/pushBill';
export async function generateAbstract(next = false): Promise<any> {
if (process.env.queueTask !== '1') {
fetch(process.env.parentUrl || '');
return;
}
if (global.generatingAbstract && !next) return;
global.generatingAbstract = true;
const systemPrompt: ChatCompletionRequestMessage = {
role: 'system',
content: `总结助手,我会向你发送一段长文本,请从文本中归纳总结5至15条信息,如果是英文,请增加一条中文的总结,并按以下格式输出: A1:\nA2:\nA3:\n`
};
let dataItem: DataItemSchema | null = null;
try {
// 找出一个需要生成的 dataItem
dataItem = await DataItem.findOne({
status: { $ne: 0 },
times: { $gt: 0 },
type: 'abstract'
});
if (!dataItem) {
console.log('没有需要生成 【摘要】 的数据');
global.generatingAbstract = false;
return;
}
// 更新状态为生成中
await DataItem.findByIdAndUpdate(dataItem._id, {
status: 2
});
// 获取 openapi Key
let userApiKey, systemKey;
try {
const key = await getOpenApiKey(dataItem.userId);
userApiKey = key.userApiKey;
systemKey = key.systemKey;
} catch (error: any) {
if (error?.code === 501) {
// 余额不够了, 把用户所有记录改成闲置
await DataItem.updateMany({
userId: dataItem.userId,
status: 0
});
}
throw new Error('获取 openai key 失败');
}
console.log('正在生成一组摘要, ID:', dataItem._id);
const startTime = Date.now();
// 获取 openai 请求实例
const chatAPI = getOpenAIApi(userApiKey || systemKey);
// 请求 chatgpt 获取摘要
const abstractResponse = await chatAPI.createChatCompletion(
{
model: ChatModelEnum.GPT35,
temperature: 0.8,
n: 1,
messages: [
systemPrompt,
{
role: 'user',
content: dataItem?.text || ''
}
]
},
{
timeout: 180000,
...axiosConfig
}
);
// 提取摘要内容
const rawContent: string = abstractResponse?.data.choices[0].message?.content || '';
// 从 content 中提取摘要内容
const splitContents = splitText(rawContent);
// 插入数据库，并修改状态
await DataItem.findByIdAndUpdate(dataItem._id, {
status: 0,
$push: {
rawResponse: rawContent,
result: {
$each: splitContents
}
}
});
console.log(
`生成摘要成功，time: ${(Date.now() - startTime) / 1000}s`,
`摘要匹配数量: ${splitContents.length}`
);
// 计费
pushSplitDataBill({
isPay: !userApiKey && splitContents.length > 0,
userId: dataItem.userId,
type: 'abstract',
text: systemPrompt.content + dataItem.text + rawContent,
tokenLen: 0
});
} catch (error: any) {
console.log('error: 生成摘要错误', dataItem?._id);
console.log('response:', error);
if (dataItem?._id) {
await DataItem.findByIdAndUpdate(dataItem._id, {
status: dataItem.times > 1 ? 1 : 0, // 还有重试次数则可以继续进行
$inc: {
// 剩余尝试次数-1
times: -1
}
});
}
}
generateAbstract(true);
}
/**
* 检查文本是否按格式返回
*/
function splitText(text: string) {
const regex = /A\d+:(\s*)(.*?)\s*(?=A\d+:|$)/gs;
const matches = text.matchAll(regex); // 获取所有匹配到的结果
const result = []; // 存储最终的结果
for (const match of matches) {
if (match[2]) {
result.push({
abstract: match[2] as string
});
}
}
if (result.length === 0) {
result.push({
abstract: text
});
}
return result;
}
src/service/events/pushBill.ts
...@@ -7,7 +7,6 @@ import { ...@@ -7,7 +7,6 @@ import {
embeddingModel embeddingModel
} from '@/constants/model'; } from '@/constants/model';
import { BillTypeEnum } from '@/constants/user'; import { BillTypeEnum } from '@/constants/user';
import type { DataType } from '@/types/data';
import { countChatTokens } from '@/utils/tools'; import { countChatTokens } from '@/utils/tools';
export const pushChatBill = async ({ export const pushChatBill = async ({
...@@ -81,7 +80,7 @@ export const pushSplitDataBill = async ({ ...@@ -81,7 +80,7 @@ export const pushSplitDataBill = async ({
userId: string; userId: string;
tokenLen: number; tokenLen: number;
text: string; text: string;
type: DataType; type: `${BillTypeEnum}`;
}) => { }) => {
await connectToDatabase(); await connectToDatabase();
......
src/service/models/data.ts deleted 100644 → 0
import { Schema, model, models, Model } from 'mongoose';
import { DataSchema as Datatype } from '@/types/mongoSchema';
import { DataTypeTextMap } from '@/constants/data';
const DataSchema = new Schema({
userId: {
type: Schema.Types.ObjectId,
ref: 'user',
required: true
},
name: {
type: String,
required: true
},
createTime: {
type: Date,
default: () => new Date()
},
type: {
type: String,
required: true,
enum: Object.keys(DataTypeTextMap)
},
isDeleted: {
type: Boolean,
default: false
}
});
export const Data: Model<Datatype> = models['data'] || model('data', DataSchema);
src/service/models/dataItem.ts deleted 100644 → 0
import type { DataItemSchema as DataItemType } from '@/types/mongoSchema';
import { Schema, model, models, Model } from 'mongoose';
import { DataTypeTextMap } from '@/constants/data';
const DataItemSchema = new Schema({
userId: {
type: Schema.Types.ObjectId,
ref: 'user',
required: true
},
dataId: {
type: Schema.Types.ObjectId,
ref: 'data',
required: true
},
type: {
type: String,
required: true,
enum: Object.keys(DataTypeTextMap)
},
times: {
// 剩余重试次数
type: Number,
default: 3
},
text: {
// 文本内容
type: String,
required: true
},
rawResponse: {
// 原始拆分结果
type: [String],
default: []
},
result: {
type: [
{
q: {
type: String,
default: ''
},
a: {
type: String,
default: ''
},
abstract: {
// 摘要
type: String,
default: ''
},
abstractVector: {
// 摘要对应的向量
type: [Number],
default: []
}
}
],
default: []
},
status: {
// 0-闲置，1-待生成，2-生成中
type: Number,
default: 1
}
});
export const DataItem: Model<DataItemType> =
models['dataItem'] || model('dataItem', DataItemSchema);
src/service/models/model.ts
...@@ -21,11 +21,6 @@ const ModelSchema = new Schema({ ...@@ -21,11 +21,6 @@ const ModelSchema = new Schema({
type: String, type: String,
default: '' default: ''
}, },
intro: {
// 模型介绍
type: String,
default: ''
},
status: { status: {
type: String, type: String,
required: true, required: true,
...@@ -35,10 +30,6 @@ const ModelSchema = new Schema({ ...@@ -35,10 +30,6 @@ const ModelSchema = new Schema({
type: Date, type: Date,
default: () => new Date() default: () => new Date()
}, },
trainingTimes: {
type: Number,
default: 0
},
temperature: { temperature: {
type: Number, type: Number,
min: 0, min: 0,
...@@ -53,11 +44,6 @@ const ModelSchema = new Schema({ ...@@ -53,11 +44,6 @@ const ModelSchema = new Schema({
} }
}, },
service: { service: {
trainId: {
// 训练时需要的 ID， 不能训练的模型没有这个值。
type: String,
required: false
},
chatModel: { chatModel: {
// 聊天时使用的模型 // 聊天时使用的模型
type: String, type: String,
......
src/service/models/training.ts deleted 100644 → 0
import { Schema, model, models, Model } from 'mongoose';
import { TrainingSchema as TrainingType } from '@/types/mongoSchema';
const TrainingSChema = new Schema({
serviceName: {
// 模型厂商名
type: String,
required: true
},
tuneId: {
// 微调进程 ID
type: String,
required: true
},
modelId: {
// 关联模型的 ID
type: Schema.Types.ObjectId,
ref: 'model',
required: true
},
status: {
// 状态值
type: String,
required: true,
enum: ['pending', 'succeed', 'errored', 'canceled']
}
});
export const Training: Model<TrainingType> =
models['training'] || model('training', TrainingSChema);
src/service/mongo.ts
...@@ -45,11 +45,8 @@ export * from './models/authCode'; ...@@ -45,11 +45,8 @@ export * from './models/authCode';
export * from './models/chat'; export * from './models/chat';
export * from './models/model'; export * from './models/model';
export * from './models/user'; export * from './models/user';
export * from './models/training';
export * from './models/bill'; export * from './models/bill';
export * from './models/pay'; export * from './models/pay';
export * from './models/data';
export * from './models/dataItem';
export * from './models/splitData'; export * from './models/splitData';
export * from './models/openapi'; export * from './models/openapi';
export * from './models/promotionRecord'; export * from './models/promotionRecord';
src/service/utils/tools.ts
...@@ -87,11 +87,9 @@ export const authOpenApiKey = async (req: NextApiRequest) => { ...@@ -87,11 +87,9 @@ export const authOpenApiKey = async (req: NextApiRequest) => {
/* openai axios config */ /* openai axios config */
export const axiosConfig = { export const axiosConfig = {
httpsAgent: global.httpsAgent, httpsAgent: global.httpsAgent,
headers: process.env.OPENAI_BASE_URL_AUTH headers: {
? { auth: process.env.OPENAI_BASE_URL_AUTH || ''
auth: process.env.OPENAI_BASE_URL_AUTH }
}
: {}
}; };
/* delete invalid symbol */ /* delete invalid symbol */
......
src/types/data.d.ts deleted 100644 → 0
import type { DataSchema } from './mongoSchema';
export type DataType = 'QA' | 'abstract';
export interface DataListItem extends DataSchema {
trainingData: number;
totalData: number;
}
src/types/model.d.ts
...@@ -3,7 +3,6 @@ import type { ModelSchema } from './mongoSchema'; ...@@ -3,7 +3,6 @@ import type { ModelSchema } from './mongoSchema';
export interface ModelUpdateParams { export interface ModelUpdateParams {
name: string; name: string;
systemPrompt: string; systemPrompt: string;
intro: string;
temperature: number; temperature: number;
search: ModelSchema['search']; search: ModelSchema['search'];
service: ModelSchema['service']; service: ModelSchema['service'];
......
src/types/mongoSchema.d.ts
import type { ChatItemType } from './chat'; import type { ChatItemType } from './chat';
import { import {
ModelStatusEnum, ModelStatusEnum,
TrainingStatusEnum,
ModelNameEnum, ModelNameEnum,
ModelVectorSearchModeEnum, ModelVectorSearchModeEnum,
ChatModelEnum ChatModelEnum
...@@ -35,17 +34,14 @@ export interface ModelSchema { ...@@ -35,17 +34,14 @@ export interface ModelSchema {
name: string; name: string;
avatar: string; avatar: string;
systemPrompt: string; systemPrompt: string;
intro: string;
userId: string; userId: string;
status: `${ModelStatusEnum}`; status: `${ModelStatusEnum}`;
updateTime: number; updateTime: number;
trainingTimes: number;
temperature: number; temperature: number;
search: { search: {
mode: `${ModelVectorSearchModeEnum}`; mode: `${ModelVectorSearchModeEnum}`;
}; };
service: { service: {
trainId: string; // 训练的模型，训练后就是训练的模型id
chatModel: `${ChatModelEnum}`; // 聊天时用的模型，训练后就是训练的模型 chatModel: `${ChatModelEnum}`; // 聊天时用的模型，训练后就是训练的模型
modelName: `${ModelNameEnum}`; // 底层模型名称，不会变 modelName: `${ModelNameEnum}`; // 底层模型名称，不会变
}; };
...@@ -81,17 +77,6 @@ export interface ModelSplitDataSchema { ...@@ -81,17 +77,6 @@ export interface ModelSplitDataSchema {
textList: string[]; textList: string[];
} }
export interface TrainingSchema {
_id: string;
tuneId: string;
modelId: string;
status: `${TrainingStatusEnum}`;
}
export interface TrainingPopulate extends TrainingSchema {
modelId: ModelSchema;
}
export interface ChatSchema { export interface ChatSchema {
_id: string; _id: string;
userId: string; userId: string;
...@@ -126,34 +111,6 @@ export interface PaySchema { ...@@ -126,34 +111,6 @@ export interface PaySchema {
status: 'SUCCESS' | 'REFUND' | 'NOTPAY' | 'CLOSED'; status: 'SUCCESS' | 'REFUND' | 'NOTPAY' | 'CLOSED';
} }
export interface DataSchema {
_id: string;
userId: string;
name: string;
createTime: Date;
type: DataType;
}
export interface DataItemSchema {
_id: string;
userId: string;
dataId: string;
type: DataType;
times: number;
text: string;
rawResponse: string[];
result: {
q?: string;
a?: string;
abstract?: string;
}[];
status: 0 | 1 | 2;
}
export interface DataItemPopulate extends DataItemSchema {
userId: UserModelSchema;
}
export interface OpenApiSchema { export interface OpenApiSchema {
_id: string; _id: string;
userId: string; userId: string;
......
src/types/training.d.ts deleted 100644 → 0
import { TrainingStatusEnum } from '@/constants/model';
export type TrainingItemType = {
_id: string;
serviceName: string;
tuneId: string;
modelId: string;
status: `${TrainingStatusEnum}`;
};
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment