Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 3db69077 by archer
Options

feat: 数据集合管理

parent 8a9f1ed2
Showing with 244 additions and 13 deletions
src/api/data.ts
...@@ -11,3 +11,8 @@ export const postData = (name: string) => POST<string>(`/data/postData?name=${na ...@@ -11,3 +11,8 @@ export const postData = (name: string) => POST<string>(`/data/postData?name=${na
export const postSplitData = (dataId: string, text: string) => export const postSplitData = (dataId: string, text: string) =>
POST(`/data/splitData`, { dataId, text }); POST(`/data/splitData`, { dataId, text });
export const updateDataName = (dataId: string, name: string) =>
PUT(`/data/putDataName?dataId=${dataId}&name=${name}`);
export const delData = (dataId: string) => DELETE(`/data/delData?dataId=${dataId}`);
src/hooks/useRequest.tsx 0 → 100644
import { useToast } from '@/hooks/useToast';
import { useMutation } from '@tanstack/react-query';
import type { UseMutationOptions } from '@tanstack/react-query';
interface Props extends UseMutationOptions<any, any, any, any> {
successToast?: string;
errorToast?: string;
}
export const useRequest = ({ successToast, errorToast, onSuccess, onError, ...props }: Props) => {
const { toast } = useToast();
const mutation = useMutation<unknown, unknown, any, unknown>({
...props,
onSuccess(res, variables: void, context: unknown) {
onSuccess?.(res, variables, context);
successToast &&
toast({
title: successToast,
status: 'success'
});
},
onError(err: any, variables: void, context: unknown) {
onError?.(err, variables, context);
errorToast &&
toast({
title: typeof err === 'string' ? err : err?.message || errorToast,
status: 'error'
});
}
});
return mutation;
};
src/pages/api/data/delData.ts 0 → 100644
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Data } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import type { DataListItem } from '@/types/data';
import type { PagingData } from '@/types';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { authorization } = req.headers;
if (!authorization) {
throw new Error('缺少登录凭证');
}
await authToken(authorization);
const { dataId } = req.query as { dataId: string };
if (!dataId) {
throw new Error('缺少参数');
}
await connectToDatabase();
await Data.findByIdAndUpdate(dataId, {
isDeleted: true
});
jsonRes<PagingData<DataListItem>>(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/data/getDataList.ts
...@@ -28,7 +28,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -28,7 +28,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const datalist = await Data.aggregate<DataListItem>([ const datalist = await Data.aggregate<DataListItem>([
{ {
$match: { $match: {
userId: new mongoose.Types.ObjectId(userId) userId: new mongoose.Types.ObjectId(userId),
isDeleted: false
} }
}, },
{ {
......
src/pages/api/data/putDataName.ts 0 → 100644
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Data } from '@/service/mongo';
import { authToken } from '@/service/utils/tools';
import type { DataListItem } from '@/types/data';
import type { PagingData } from '@/types';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { authorization } = req.headers;
if (!authorization) {
throw new Error('缺少登录凭证');
}
await authToken(authorization);
const { dataId, name } = req.query as { dataId: string; name: string };
if (!dataId || !name) {
throw new Error('缺少参数');
}
await connectToDatabase();
await Data.findByIdAndUpdate(dataId, {
name
});
jsonRes<PagingData<DataListItem>>(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/timer/clearAuthCode.ts
...@@ -5,6 +5,9 @@ import { AuthCode } from '@/service/models/authCode'; ...@@ -5,6 +5,9 @@ import { AuthCode } from '@/service/models/authCode';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.headers.auth !== 'archer') {
throw new Error('凭证错误');
}
try { try {
await connectToDatabase(); await connectToDatabase();
......
src/pages/api/timer/clearChatWindow.ts
...@@ -4,6 +4,9 @@ import { connectToDatabase, Chat } from '@/service/mongo'; ...@@ -4,6 +4,9 @@ import { connectToDatabase, Chat } from '@/service/mongo';
/* 定时删除那些不活跃的内容 */ /* 定时删除那些不活跃的内容 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.headers.auth !== 'archer') {
throw new Error('凭证错误');
}
try { try {
await connectToDatabase(); await connectToDatabase();
......
src/pages/api/timer/initBill.ts
...@@ -7,6 +7,9 @@ import type { BillSchema } from '@/types/mongoSchema'; ...@@ -7,6 +7,9 @@ import type { BillSchema } from '@/types/mongoSchema';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
if (req.headers.auth !== 'archer') {
throw new Error('凭证错误');
}
await connectToDatabase(); await connectToDatabase();
await Bill.updateMany( await Bill.updateMany(
......
src/pages/api/timer/initDataItemTime.ts 0 → 100644
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, DataItem, Data } from '@/service/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
if (req.headers.auth !== 'archer') {
throw new Error('凭证错误');
}
await connectToDatabase();
// await DataItem.updateMany(
// {},
// {
// times: 2
// }
// );
await Data.updateMany(
{},
{
isDeleted: false
}
);
jsonRes(res, {
data: {}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/timer/updateTraining.ts
...@@ -11,6 +11,9 @@ import { sendTrainSucceed } from '@/service/utils/sendEmail'; ...@@ -11,6 +11,9 @@ import { sendTrainSucceed } from '@/service/utils/sendEmail';
import { httpsAgent } from '@/service/utils/tools'; import { httpsAgent } from '@/service/utils/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.headers.auth !== 'archer') {
throw new Error('凭证错误');
}
try { try {
await connectToDatabase(); await connectToDatabase();
......
src/pages/data/defail.tsx 0 → 100644
import React from 'react';
const DataDetail = ({ dataId }: { dataId: string }) => {
return <div>DataDetail</div>;
};
export default DataDetail;
export async function getServerSideProps(context: any) {
const dataId = context.query?.dataId || '';
return {
props: { dataId }
};
}
src/pages/data/list.tsx
...@@ -11,14 +11,17 @@ import { ...@@ -11,14 +11,17 @@ import {
Th, Th,
Td, Td,
TableContainer, TableContainer,
useDisclosure useDisclosure,
Input
} from '@chakra-ui/react'; } from '@chakra-ui/react';
import { getDataList } from '@/api/data'; import { getDataList, updateDataName, delData } from '@/api/data';
import { usePaging } from '@/hooks/usePaging'; import { usePaging } from '@/hooks/usePaging';
import type { DataListItem } from '@/types/data'; import type { DataListItem } from '@/types/data';
import ScrollData from '@/components/ScrollData'; import ScrollData from '@/components/ScrollData';
import dayjs from 'dayjs'; import dayjs from 'dayjs';
import dynamic from 'next/dynamic'; import dynamic from 'next/dynamic';
import { useConfirm } from '@/hooks/useConfirm';
import { useRequest } from '@/hooks/useRequest';
const CreateDataModal = dynamic(() => import('./components/CreateDataModal')); const CreateDataModal = dynamic(() => import('./components/CreateDataModal'));
const ImportDataModal = dynamic(() => import('./components/ImportDataModal')); const ImportDataModal = dynamic(() => import('./components/ImportDataModal'));
...@@ -34,6 +37,9 @@ const DataList = () => { ...@@ -34,6 +37,9 @@ const DataList = () => {
pageSize: 20 pageSize: 20
}); });
const [ImportDataId, setImportDataId] = useState<string>(); const [ImportDataId, setImportDataId] = useState<string>();
const { openConfirm, ConfirmChild } = useConfirm({
content: '删除数据集，将删除里面的所有内容，请确认！'
});
const { const {
isOpen: isOpenCreateDataModal, isOpen: isOpenCreateDataModal,
...@@ -41,6 +47,15 @@ const DataList = () => { ...@@ -41,6 +47,15 @@ const DataList = () => {
onClose: onCloseCreateDataModal onClose: onCloseCreateDataModal
} = useDisclosure(); } = useDisclosure();
const { mutate: handleDelData, isLoading: isDeleting } = useRequest({
mutationFn: (dataId: string) => delData(dataId),
successToast: '删除数据集成功',
errorToast: '删除数据集异常',
onSuccess() {
getData(1, true);
}
});
return ( return (
<Box display={['block', 'flex']} flexDirection={'column'} h={'100%'}> <Box display={['block', 'flex']} flexDirection={'column'} h={'100%'}>
<Card px={6} py={4}> <Card px={6} py={4}>
...@@ -74,7 +89,17 @@ const DataList = () => { ...@@ -74,7 +89,17 @@ const DataList = () => {
<Tbody> <Tbody>
{dataList.map((item, i) => ( {dataList.map((item, i) => (
<Tr key={item._id}> <Tr key={item._id}>
<Td>{item.name}</Td> <Td>
<Input
placeholder="请输入数据集名称"
defaultValue={item.name}
size={'sm'}
onBlur={(e) => {
if (!e.target.value) return;
updateDataName(item._id, e.target.value);
}}
/>
</Td>
<Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td> <Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td>
<Td> <Td>
{item.trainingData} / {item.totalData} {item.trainingData} / {item.totalData}
...@@ -83,12 +108,31 @@ const DataList = () => { ...@@ -83,12 +108,31 @@ const DataList = () => {
<Button <Button
size={'sm'} size={'sm'}
variant={'outline'} variant={'outline'}
colorScheme={'gray'}
mr={2}
onClick={() => setImportDataId(item._id)}
>
详细
</Button>
<Button
size={'sm'}
variant={'outline'}
mr={2} mr={2}
onClick={() => setImportDataId(item._id)} onClick={() => setImportDataId(item._id)}
> >
导入 导入
</Button> </Button>
<Button size={'sm'}>导出</Button> <Button mr={2} size={'sm'}>
导出
</Button>
<Button
size={'sm'}
colorScheme={'red'}
isLoading={isDeleting}
onClick={openConfirm(() => handleDelData(item._id))}
>
删除
</Button>
</Td> </Td>
</Tr> </Tr>
))} ))}
...@@ -104,6 +148,7 @@ const DataList = () => { ...@@ -104,6 +148,7 @@ const DataList = () => {
{isOpenCreateDataModal && ( {isOpenCreateDataModal && (
<CreateDataModal onClose={onCloseCreateDataModal} onSuccess={() => getData(1, true)} /> <CreateDataModal onClose={onCloseCreateDataModal} onSuccess={() => getData(1, true)} />
)} )}
<ConfirmChild />
</Box> </Box>
); );
}; };
......
src/service/events/generateQA.ts
...@@ -35,10 +35,7 @@ export async function generateQA(next = false): Promise<any> { ...@@ -35,10 +35,7 @@ export async function generateQA(next = false): Promise<any> {
// 减少一次重试次数, 并更新状态为生成中 // 减少一次重试次数, 并更新状态为生成中
await DataItem.findByIdAndUpdate(dataItem._id, { await DataItem.findByIdAndUpdate(dataItem._id, {
status: 2, status: 2
$inc: {
time: -1
}
}); });
// 获取 openapi Key // 获取 openapi Key
...@@ -86,22 +83,31 @@ export async function generateQA(next = false): Promise<any> { ...@@ -86,22 +83,31 @@ export async function generateQA(next = false): Promise<any> {
const splitResponse = splitText(content || ''); const splitResponse = splitText(content || '');
// 插入数据库，并修改状态 // 插入数据库，并修改状态
await DataItem.findByIdAndUpdate(dataItem._id, { await DataItem.findByIdAndUpdate(dataItem._id, {
status: dataItem.temperature >= 100 ? 0 : 1, status: dataItem.temperature >= 80 ? 0 : 1, // 需要生成 5 组内容。0,0.2,0.4,0.6,0.8
temperature: dataItem.temperature >= 100 ? dataItem.temperature : dataItem.temperature + 25, temperature: dataItem.temperature >= 80 ? dataItem.temperature : dataItem.temperature + 20,
$push: { $push: {
result: { result: {
$each: splitResponse $each: splitResponse
} }
} }
}); });
console.log('生成成功，time:', `${(Date.now() - startTime) / 1000}s`); console.log(
'生成成功，time:',
`${(Date.now() - startTime) / 1000}s`,
'result length: ',
splitResponse.length
);
} catch (error: any) { } catch (error: any) {
console.log('error: 生成QA错误', dataItem?._id); console.log('error: 生成QA错误', dataItem?._id);
console.log('response:', error?.response); console.log('response:', error?.response);
// 重置状态 // 重置状态
if (dataItem?._id) { if (dataItem?._id) {
await DataItem.findByIdAndUpdate(dataItem._id, { await DataItem.findByIdAndUpdate(dataItem._id, {
status: dataItem.times > 0 ? 1 : 0 // 还有重试次数则可以继续进行 status: dataItem.times > 0 ? 1 : 0, // 还有重试次数则可以继续进行
$inc: {
// 剩余尝试次数-1
times: -1
}
}); });
} }
} }
......
src/service/models/data.ts
...@@ -13,6 +13,10 @@ const DataSchema = new Schema({ ...@@ -13,6 +13,10 @@ const DataSchema = new Schema({
createTime: { createTime: {
type: Date, type: Date,
default: () => new Date() default: () => new Date()
},
isDeleted: {
type: Boolean,
default: false
} }
}); });
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment