Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 29c95d24 by archer
Options

perf: model data code

parent e0b1a783
Showing with 67 additions and 110 deletions
next.config.js
/** @type {import('next').NextConfig} */ /** @type {import('next').NextConfig} */
const path = require('path');
const isDev = process.env.NODE_ENV === 'development';
const nextConfig = { const nextConfig = {
output: 'standalone', output: 'standalone',
reactStrictMode: false, reactStrictMode: true,
compress: true, compress: true,
webpack(config) { webpack(config) {
config.module.rules = config.module.rules.concat([ config.module.rules = config.module.rules.concat([
......
src/api/model.ts
import { GET, POST, DELETE, PUT } from './request'; import { GET, POST, DELETE, PUT } from './request';
import type { ModelSchema, ModelDataSchema, ModelSplitDataSchema } from '@/types/mongoSchema'; import type { ModelSchema, ModelDataSchema } from '@/types/mongoSchema';
import { ModelUpdateParams } from '@/types/model'; import { ModelUpdateParams } from '@/types/model';
import { TrainingItemType } from '../types/training'; import { TrainingItemType } from '../types/training';
import { RequestPaging } from '../types/index'; import { RequestPaging } from '../types/index';
......
src/constants/model.ts
import type { ModelDataType, ModelSchema } from '@/types/mongoSchema'; import type { ModelSchema } from '@/types/mongoSchema';
export enum ModelDataStatusEnum { export enum ModelDataStatusEnum {
ready = 'ready', ready = 'ready',
...@@ -120,7 +120,7 @@ export const ModelVectorSearchModeMap: Record< ...@@ -120,7 +120,7 @@ export const ModelVectorSearchModeMap: Record<
export const defaultModel: ModelSchema = { export const defaultModel: ModelSchema = {
_id: '', _id: '',
userId: '', userId: '',
name: '', name: 'modelName',
avatar: '', avatar: '',
status: ModelStatusEnum.pending, status: ModelStatusEnum.pending,
updateTime: Date.now(), updateTime: Date.now(),
......
src/pages/_error.tsx 0 → 100644
function Error({ statusCode }: { statusCode: number }) {
return (
<p>
{statusCode ? `An error ${statusCode} occurred on server` : 'An error occurred on client'}
</p>
);
}
Error.getInitialProps = ({ res, err }: { res: any; err: any }) => {
const statusCode = res ? res.statusCode : err ? err.statusCode : 404;
console.log(err);
return { statusCode };
};
export default Error;
src/pages/model/detail/components/ModelDataCard.tsx
import React, { useCallback, useState } from 'react'; import React, { useCallback, useState, useRef } from 'react';
import { import {
Box, Box,
TableContainer, TableContainer,
...@@ -19,7 +19,6 @@ import { ...@@ -19,7 +19,6 @@ import {
Input Input
} from '@chakra-ui/react'; } from '@chakra-ui/react';
import type { BoxProps } from '@chakra-ui/react'; import type { BoxProps } from '@chakra-ui/react';
import type { ModelSchema } from '@/types/mongoSchema';
import type { ModelDataItemType } from '@/types/model'; import type { ModelDataItemType } from '@/types/model';
import { ModelDataStatusMap } from '@/constants/model'; import { ModelDataStatusMap } from '@/constants/model';
import { usePagination } from '@/hooks/usePagination'; import { usePagination } from '@/hooks/usePagination';
...@@ -34,19 +33,23 @@ import { useLoading } from '@/hooks/useLoading'; ...@@ -34,19 +33,23 @@ import { useLoading } from '@/hooks/useLoading';
import { fileDownload } from '@/utils/file'; import { fileDownload } from '@/utils/file';
import dynamic from 'next/dynamic'; import dynamic from 'next/dynamic';
import { useMutation, useQuery } from '@tanstack/react-query'; import { useMutation, useQuery } from '@tanstack/react-query';
import type { FormData as InputDataType } from './InputDataModal';
import Papa from 'papaparse'; import Papa from 'papaparse';
import InputModal, { FormData as InputDataType } from './InputDataModal';
const InputModel = dynamic(() => import('./InputDataModal')); const SelectFileModal = dynamic(() => import('./SelectFileModal'));
const SelectFileModel = dynamic(() => import('./SelectFileModal'));
const SelectUrlModel = dynamic(() => import('./SelectUrlModal'));
const SelectCsvModal = dynamic(() => import('./SelectCsvModal')); const SelectCsvModal = dynamic(() => import('./SelectCsvModal'));
let lastSearch = ''; const ModelDataCard = ({ modelId }: { modelId: string }) => {
const ModelDataCard = ({ model }: { model: ModelSchema }) => {
const { Loading, setIsLoading } = useLoading(); const { Loading, setIsLoading } = useLoading();
const lastSearch = useRef('');
const [searchText, setSearchText] = useState(''); const [searchText, setSearchText] = useState('');
const tdStyles = useRef<BoxProps>({
fontSize: 'xs',
maxW: '500px',
whiteSpace: 'pre-wrap',
maxH: '250px',
overflowY: 'auto'
});
const { const {
data: modelDataList, data: modelDataList,
isLoading, isLoading,
...@@ -58,7 +61,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { ...@@ -58,7 +61,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
api: getModelDataList, api: getModelDataList,
pageSize: 10, pageSize: 10,
params: { params: {
modelId: model._id, modelId,
searchText searchText
} }
}); });
...@@ -71,18 +74,19 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { ...@@ -71,18 +74,19 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
onClose: onCloseSelectFileModal onClose: onCloseSelectFileModal
} = useDisclosure(); } = useDisclosure();
const { const {
isOpen: isOpenSelectUrlModal,
onOpen: onOpenSelectUrlModal,
onClose: onCloseSelectUrlModal
} = useDisclosure();
const {
isOpen: isOpenSelectCsvModal, isOpen: isOpenSelectCsvModal,
onOpen: onOpenSelectCsvModal, onOpen: onOpenSelectCsvModal,
onClose: onCloseSelectCsvModal onClose: onCloseSelectCsvModal
} = useDisclosure(); } = useDisclosure();
const { data: splitDataLen, refetch } = useQuery(['getModelSplitDataList'], () => const { data: splitDataLen = 0, refetch } = useQuery(
getModelSplitDataListLen(model._id) ['getModelSplitDataList'],
() => getModelSplitDataListLen(modelId),
{
onError(err) {
console.log(err);
}
}
); );
const refetchData = useCallback( const refetchData = useCallback(
...@@ -94,8 +98,8 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { ...@@ -94,8 +98,8 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
); );
// 获取所有的数据，并导出 json // 获取所有的数据，并导出 json
const { mutate: onclickExport, isLoading: isLoadingExport } = useMutation({ const { mutate: onclickExport, isLoading: isLoadingExport = false } = useMutation({
mutationFn: () => getExportDataList(model._id), mutationFn: () => getExportDataList(modelId),
onSuccess(res) { onSuccess(res) {
try { try {
setIsLoading(true); setIsLoading(true);
...@@ -112,17 +116,12 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { ...@@ -112,17 +116,12 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
error; error;
} }
setIsLoading(false); setIsLoading(false);
},
onError(err) {
console.log(err);
} }
}); });
const tdStyles: BoxProps = {
fontSize: 'xs',
maxW: '500px',
whiteSpace: 'pre-wrap',
maxH: '250px',
overflowY: 'auto'
};
return ( return (
<> <>
<Flex> <Flex>
...@@ -150,7 +149,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { ...@@ -150,7 +149,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
> >
导出 导出
</Button> </Button>
<Menu> <Menu autoSelect={false}>
<MenuButton as={Button} size={'sm'}> <MenuButton as={Button} size={'sm'}>
导入 导入
</MenuButton> </MenuButton>
...@@ -166,17 +165,13 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { ...@@ -166,17 +165,13 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
手动输入 手动输入
</MenuItem> </MenuItem>
<MenuItem onClick={onOpenSelectFileModal}>文本/文件拆分</MenuItem> <MenuItem onClick={onOpenSelectFileModal}>文本/文件拆分</MenuItem>
{/* <MenuItem onClick={onOpenSelectUrlModal}>网站内容拆分</MenuItem> */}
<MenuItem onClick={onOpenSelectCsvModal}>csv 问答对导入</MenuItem> <MenuItem onClick={onOpenSelectCsvModal}>csv 问答对导入</MenuItem>
</MenuList> </MenuList>
</Menu> </Menu>
</Flex> </Flex>
<Flex mt={4}> <Flex mt={4}>
{/* 拆分数据提示 */} {splitDataLen > 0 && <Box fontSize={'xs'}>{splitDataLen}条数据正在拆分...</Box>}
{!!(splitDataLen && splitDataLen > 0) && ( <Box flex={1} />
<Box fontSize={'xs'}>{splitDataLen}条数据正在拆分...</Box>
)}
<Box flex={1}></Box>
<Input <Input
maxW={'240px'} maxW={'240px'}
size={'sm'} size={'sm'}
...@@ -184,15 +179,15 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { ...@@ -184,15 +179,15 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
placeholder="搜索相关问题和答案，回车确认" placeholder="搜索相关问题和答案，回车确认"
onChange={(e) => setSearchText(e.target.value)} onChange={(e) => setSearchText(e.target.value)}
onBlur={() => { onBlur={() => {
if (searchText === lastSearch) return; if (searchText === lastSearch.current) return;
getData(1); getData(1);
lastSearch = searchText; lastSearch.current = searchText;
}} }}
onKeyDown={(e) => { onKeyDown={(e) => {
if (searchText === lastSearch) return; if (searchText === lastSearch.current) return;
if (e.key === 'Enter') { if (e.key === 'Enter') {
getData(1); getData(1);
lastSearch = searchText; lastSearch.current = searchText;
} }
}} }}
/> />
...@@ -203,7 +198,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { ...@@ -203,7 +198,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
<Table variant={'simple'} w={'100%'}> <Table variant={'simple'} w={'100%'}>
<Thead> <Thead>
<Tr> <Tr>
<Th>匹配内容(问题)</Th> <Th>{'匹配内容(问题)'}</Th>
<Th>对应答案</Th> <Th>对应答案</Th>
<Th>状态</Th> <Th>状态</Th>
<Th>操作</Th> <Th>操作</Th>
...@@ -213,10 +208,10 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { ...@@ -213,10 +208,10 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
{modelDataList.map((item) => ( {modelDataList.map((item) => (
<Tr key={item.id}> <Tr key={item.id}>
<Td> <Td>
<Box {...tdStyles}>{item.q}</Box> <Box {...tdStyles.current}>{item.q}</Box>
</Td> </Td>
<Td> <Td>
<Box {...tdStyles}>{item.a || '-'}</Box> <Box {...tdStyles.current}>{item.a || '-'}</Box>
</Td> </Td>
<Td>{ModelDataStatusMap[item.status]}</Td> <Td>{ModelDataStatusMap[item.status]}</Td>
<Td> <Td>
...@@ -258,33 +253,22 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => { ...@@ -258,33 +253,22 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
<Loading loading={isLoading} fixed={false} /> <Loading loading={isLoading} fixed={false} />
{editInputData !== undefined && ( {editInputData !== undefined && (
<InputModel <InputModal
modelId={model._id} modelId={modelId}
defaultValues={editInputData} defaultValues={editInputData}
onClose={() => setEditInputData(undefined)} onClose={() => setEditInputData(undefined)}
onSuccess={refetchData} onSuccess={refetchData}
/> />
)} )}
{isOpenSelectFileModal && ( {isOpenSelectFileModal && (
<SelectFileModel <SelectFileModal
modelId={model._id} modelId={modelId}
onClose={onCloseSelectFileModal} onClose={onCloseSelectFileModal}
onSuccess={refetchData} onSuccess={refetchData}
/> />
)} )}
{isOpenSelectUrlModal && (
<SelectUrlModel
modelId={model._id}
onClose={onCloseSelectUrlModal}
onSuccess={refetchData}
/>
)}
{isOpenSelectCsvModal && ( {isOpenSelectCsvModal && (
<SelectCsvModal <SelectCsvModal modelId={modelId} onClose={onCloseSelectCsvModal} onSuccess={refetchData} />
modelId={model._id}
onClose={onCloseSelectCsvModal}
onSuccess={refetchData}
/>
)} )}
</> </>
); );
......
src/pages/model/detail/index.tsx
import React, { useCallback, useState, useRef, useMemo, useEffect } from 'react'; import React, { useCallback, useState, useMemo, useEffect } from 'react';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { getModelById, delModelById, putModelTrainingStatus, putModelById } from '@/api/model'; import { getModelById, delModelById, putModelTrainingStatus, putModelById } from '@/api/model';
import type { ModelSchema } from '@/types/mongoSchema'; import type { ModelSchema } from '@/types/mongoSchema';
...@@ -8,16 +8,16 @@ import { useForm } from 'react-hook-form'; ...@@ -8,16 +8,16 @@ import { useForm } from 'react-hook-form';
import { formatModelStatus, ModelStatusEnum, modelList, defaultModel } from '@/constants/model'; import { formatModelStatus, ModelStatusEnum, modelList, defaultModel } from '@/constants/model';
import { useGlobalStore } from '@/store/global'; import { useGlobalStore } from '@/store/global';
import { useScreen } from '@/hooks/useScreen'; import { useScreen } from '@/hooks/useScreen';
import ModelEditForm from './components/ModelEditForm';
import { useQuery } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query';
import dynamic from 'next/dynamic'; import dynamic from 'next/dynamic';
const ModelEditForm = dynamic(() => import('./components/ModelEditForm'));
const ModelDataCard = dynamic(() => import('./components/ModelDataCard')); const ModelDataCard = dynamic(() => import('./components/ModelDataCard'));
const ModelDetail = ({ modelId }: { modelId: string }) => { const ModelDetail = ({ modelId }: { modelId: string }) => {
const { toast } = useToast(); const { toast } = useToast();
const router = useRouter(); const router = useRouter();
const { isPc, media } = useScreen(); const { isPc } = useScreen();
const { setLoading } = useGlobalStore(); const { setLoading } = useGlobalStore();
const [model, setModel] = useState<ModelSchema>(defaultModel); const [model, setModel] = useState<ModelSchema>(defaultModel);
...@@ -76,36 +76,6 @@ const ModelDetail = ({ modelId }: { modelId: string }) => { ...@@ -76,36 +76,6 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
setLoading(false); setLoading(false);
}, [setLoading, router, modelId]); }, [setLoading, router, modelId]);
/* 上传数据集,触发微调 */
// const startTraining = useCallback(
// async (e: React.ChangeEvent<HTMLInputElement>) => {
// if (!modelId || !e.target.files || e.target.files?.length === 0) return;
// setLoading(true);
// try {
// const file = e.target.files[0];
// const formData = new FormData();
// formData.append('file', file);
// await postTrainModel(modelId, formData);
// toast({
// title: '开始训练...',
// status: 'success'
// });
// // 重新获取模型
// loadModel();
// } catch (err: any) {
// toast({
// title: err?.message || '上传文件失败',
// status: 'error'
// });
// console.log('error->', err);
// }
// setLoading(false);
// },
// [setLoading, loadModel, modelId, toast]
// );
/* 点击更新模型状态 */ /* 点击更新模型状态 */
const handleClickUpdateStatus = useCallback(async () => { const handleClickUpdateStatus = useCallback(async () => {
if (!model || model.status !== ModelStatusEnum.training) return; if (!model || model.status !== ModelStatusEnum.training) return;
...@@ -236,21 +206,12 @@ const ModelDetail = ({ modelId }: { modelId: string }) => { ...@@ -236,21 +206,12 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
</> </>
)} )}
</Card> </Card>
<Grid mt={5} gridTemplateColumns={media('1fr 1fr', '1fr')} gridGap={5}> <Grid mt={5} gridTemplateColumns={['1fr', '1fr 1fr']} gridGap={5}>
<ModelEditForm formHooks={formHooks} handleDelModel={handleDelModel} canTrain={canTrain} /> <ModelEditForm formHooks={formHooks} handleDelModel={handleDelModel} canTrain={canTrain} />
{canTrain && model._id && ( {canTrain && !!model._id && (
<Card <Card p={4} gridColumnStart={[1, 1]} gridColumnEnd={[2, 3]}>
p={4} <ModelDataCard modelId={model._id} />
{...media(
{
gridColumnStart: 1,
gridColumnEnd: 3
},
{}
)}
>
<ModelDataCard model={model} />
</Card> </Card>
)} )}
</Grid> </Grid>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment