feat: 训练数据管理

package.json

@@ -27,6 +27,7 @@
     "hyperdown": "^2.4.29",
     "hyperdown": "^2.4.29",
     "immer": "^9.0.19",
     "immer": "^9.0.19",
     "jsonwebtoken": "^9.0.0",
     "jsonwebtoken": "^9.0.0",
+    "mammoth": "^1.5.1",
     "mongoose": "^6.10.0",
     "mongoose": "^6.10.0",
     "nanoid": "^4.0.1",
     "nanoid": "^4.0.1",
     "next": "13.1.6",
     "next": "13.1.6",

src/api/data.ts

+import { GET, POST, DELETE, PUT } from './request';
+import { RequestPaging } from '../types/index';
+import { Obj2Query } from '@/utils/tools';
+import type { DataListItem } from '@/types/data';
+import type { PagingData } from '../types/index';
+
+export const getDataList = (data: RequestPaging) =>
+  GET<PagingData<DataListItem>>(`/data/getDataList?${Obj2Query(data)}`);
+
+export const postData = (name: string) => POST<string>(`/data/postData?name=${name}`);
+
+export const postSplitData = (dataId: string, text: string) =>
+  POST(`/data/splitData`, { dataId, text });

src/components/Layout/index.tsx

@@ -26,12 +26,12 @@ const navbarList = [
     link: '/model/list',
     link: '/model/list',
     activeLink: ['/model/list', '/model/detail']
     activeLink: ['/model/list', '/model/detail']
   },
   },
-  // {
-  //   label: '数据',
-  //   icon: 'icon-datafull',
-  //   link: '/training/dataList',
-  //   activeLink: ['/training/dataList']
-  // },
+  {
+    label: '数据',
+    icon: 'icon-datafull',
+    link: '/data/list',
+    activeLink: ['/data/list']
+  },
   {
   {
     label: '账号',
     label: '账号',
     icon: 'icon-yonghu-yuan',
     icon: 'icon-yonghu-yuan',
@@ -62,8 +62,8 @@ const Layout = ({ children }: { children: JSX.Element }) => {
               <Box h={'100%'} position={'fixed'} left={0} top={0} w={'80px'}>
               <Box h={'100%'} position={'fixed'} left={0} top={0} w={'80px'}>
                 <Navbar navbarList={navbarList} />
                 <Navbar navbarList={navbarList} />
               </Box>
               </Box>
-              <Box ml={'80px'} p={7}>
-                <Box maxW={'1100px'} m={'auto'}>
+              <Box ml={'80px'} p={7} h={'100%'}>
+                <Box maxW={'1100px'} m={'auto'} h={'100%'}>
                   <Auth>{children}</Auth>
                   <Auth>{children}</Auth>
                 </Box>
                 </Box>
               </Box>
               </Box>

src/components/ScrollData/index.tsx

+import React from 'react';
+import type { BoxProps } from '@chakra-ui/react';
+import { Box } from '@chakra-ui/react';
+
+interface Props extends BoxProps {
+  nextPage: () => void;
+  children: React.ReactNode;
+}
+
+const ScrollData = ({ children, nextPage, ...props }: Props) => {
+  return (
+    <Box {...props} overflow={'auto'}>
+      {children}
+    </Box>
+  );
+};
+
+export default ScrollData;

src/hooks/useConfirm.tsx

@@ -19,9 +19,10 @@ export const useConfirm = ({ title = '提示', content }: { title?: string; cont
   return {
   return {
     openConfirm: useCallback(
     openConfirm: useCallback(
       (confirm?: any, cancel?: any) => {
       (confirm?: any, cancel?: any) => {
-        onOpen();
         confirmCb.current = confirm;
         confirmCb.current = confirm;
         cancelCb.current = cancel;
         cancelCb.current = cancel;
+
+        return onOpen;
       },
       },
       [onOpen]
       [onOpen]
     ),
     ),

src/hooks/usePaging.ts

 import { useState, useCallback } from 'react';
 import { useState, useCallback } from 'react';
 import type { PagingData } from '../types/index';
 import type { PagingData } from '../types/index';
 import { useQuery } from '@tanstack/react-query';
 import { useQuery } from '@tanstack/react-query';
+import { useToast } from './useToast';
 
 
 export const usePaging = <T = any>({
 export const usePaging = <T = any>({
   api,
   api,
   pageSize = 10,
   pageSize = 10,
-  params
+  params = {}
 }: {
 }: {
   api: (data: any) => Promise<PagingData<T>>;
   api: (data: any) => Promise<PagingData<T>>;
   pageSize?: number;
   pageSize?: number;
   params?: Record<string, any>;
   params?: Record<string, any>;
 }) => {
 }) => {
+  const { toast } = useToast();
   const [data, setData] = useState<T[]>([]);
   const [data, setData] = useState<T[]>([]);
   const [pageNum, setPageNum] = useState(1);
   const [pageNum, setPageNum] = useState(1);
   const [total, setTotal] = useState(0);
   const [total, setTotal] = useState(0);
@@ -18,36 +20,40 @@ export const usePaging = <T = any>({
   const [requesting, setRequesting] = useState(false);
   const [requesting, setRequesting] = useState(false);
 
 
   const getData = useCallback(
   const getData = useCallback(
-    async (init = false) => {
+    async (num: number, init = false) => {
       if (requesting) return;
       if (requesting) return;
       if (!init && isLoadAll) return;
       if (!init && isLoadAll) return;
       setRequesting(true);
       setRequesting(true);
 
 
       try {
       try {
         const res = await api({
         const res = await api({
-          pageNum,
+          pageNum: num,
           pageSize,
           pageSize,
-          ...(params ? params : {})
+          ...params
         });
         });
         setData((state) => {
         setData((state) => {
           const data = init ? res.data : state.concat(res.data);
           const data = init ? res.data : state.concat(res.data);
           if (data.length >= res.total) {
           if (data.length >= res.total) {
             setIsLoadAll(true);
             setIsLoadAll(true);
           }
           }
+          setTotal(res.total);
           return data;
           return data;
         });
         });
-        setTotal(res.total);
-      } catch (error) {
+      } catch (error: any) {
+        toast({
+          title: error?.message || '获取数据异常',
+          status: 'error'
+        });
         console.log(error);
         console.log(error);
       }
       }
 
 
       setRequesting(false);
       setRequesting(false);
       return null;
       return null;
     },
     },
-    [api, isLoadAll, pageNum, pageSize, params, requesting]
+    [api, isLoadAll, pageSize, params, requesting, toast]
   );
   );
 
 
-  useQuery(['init', pageNum], () => getData(pageNum === 1));
+  useQuery(['init', pageNum], () => getData(pageNum, pageNum === 1));
 
 
   return {
   return {
     pageNum,
     pageNum,
@@ -55,6 +61,7 @@ export const usePaging = <T = any>({
     setPageNum,
     setPageNum,
     total,
     total,
     data,
     data,
-    getData
+    getData,
+    requesting
   };
   };
 };
 };

src/hooks/useSelectFile.tsx

+import React, { useRef, useCallback } from 'react';
+import { Box } from '@chakra-ui/react';
+
+export const useSelectFile = (props?: { fileType?: string; multiple?: boolean }) => {
+  const { fileType = '*', multiple = false } = props || {};
+  const SelectFileDom = useRef<HTMLInputElement>(null);
+
+  const File = useCallback(
+    ({ onSelect }: { onSelect: (e: File[]) => void }) => (
+      <Box position={'absolute'} w={0} h={0} overflow={'hidden'}>
+        <input
+          ref={SelectFileDom}
+          type="file"
+          accept={fileType}
+          multiple={multiple}
+          onChange={(e) => {
+            if (!e.target.files || e.target.files?.length === 0) return;
+            onSelect(Array.from(e.target.files));
+          }}
+        />
+      </Box>
+    ),
+    [fileType, multiple]
+  );
+
+  const onOpen = useCallback(() => {
+    SelectFileDom.current && SelectFileDom.current.click();
+  }, []);
+
+  return {
+    File,
+    onOpen
+  };
+};

src/hooks/useTabs.tsx

+import React, { useState, useCallback, useRef } from 'react';
+
+export const useTabs = ({
+  tabs = []
+}: {
+  tabs: {
+    id: string;
+    label: string;
+  }[];
+}) => {
+  const [activeTab, setActiveTab] = useState(tabs[0].id);
+
+  return {
+    tabs,
+    activeTab,
+    setActiveTab
+  };
+};

src/pages/_app.tsx

@@ -38,8 +38,9 @@ export default function App({ Component, pageProps }: AppProps) {
         />
         />
         <link rel="icon" href="/favicon.ico" />
         <link rel="icon" href="/favicon.ico" />
       </Head>
       </Head>
-      <Script src="/iconfont.js" strategy="afterInteractive"></Script>
-      <Script src="/qrcode.min.js" strategy="afterInteractive"></Script>
+      <Script src="/js/iconfont.js" strategy="afterInteractive"></Script>
+      <Script src="/js/qrcode.min.js" strategy="afterInteractive"></Script>
+      <Script src="/js/pdf.js" strategy="afterInteractive"></Script>
       <QueryClientProvider client={queryClient}>
       <QueryClientProvider client={queryClient}>
         <ChakraProvider theme={theme}>
         <ChakraProvider theme={theme}>
           <ColorModeScript initialColorMode={theme.config.initialColorMode} />
           <ColorModeScript initialColorMode={theme.config.initialColorMode} />

src/pages/api/data/getDataList.ts

+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, Data, DataItem } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import type { DataSchema } from '@/types/mongoSchema';
+import type { DataListItem } from '@/types/data';
+import type { PagingData } from '@/types';
+import mongoose from 'mongoose';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { authorization } = req.headers;
+    let { pageNum = 1, pageSize = 10 } = req.query as { pageNum: string; pageSize: string };
+
+    pageNum = +pageNum;
+    pageSize = +pageSize;
+
+    if (!authorization) {
+      throw new Error('缺少登录凭证');
+    }
+
+    const userId = await authToken(authorization);
+
+    await connectToDatabase();
+
+    // 根据 id 获取用户账单
+    const datalist = await Data.aggregate<DataListItem>([
+      {
+        $match: {
+          userId: new mongoose.Types.ObjectId(userId)
+        }
+      },
+      {
+        $sort: { createTime: -1 } // 按照创建时间倒序排列
+      },
+      {
+        $skip: (pageNum - 1) * pageSize // 跳过前面的数据
+      },
+      {
+        $limit: pageSize // 取出指定数量的数据
+      },
+      {
+        $lookup: {
+          from: 'dataitems',
+          localField: '_id',
+          foreignField: 'dataId',
+          as: 'items'
+        }
+      },
+      {
+        $addFields: {
+          totalData: {
+            $size: '$items' // 统计dataItem的总数
+          },
+          trainingData: {
+            $size: {
+              $filter: {
+                input: '$items',
+                as: 'item',
+                cond: { $eq: ['$$item.status', 1] } // 统计status为1的数量
+              }
+            }
+          }
+        }
+      },
+      {
+        $project: {
+          items: 0 // 不返回 items 字段
+        }
+      }
+    ]);
+
+    jsonRes<PagingData<DataListItem>>(res, {
+      data: {
+        pageNum,
+        pageSize,
+        data: datalist,
+        total: 1
+      }
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/data/postData.ts

+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, Data } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    let { name } = req.query as { name: string };
+    if (!name) {
+      throw new Error('参数错误');
+    }
+    await connectToDatabase();
+
+    const { authorization } = req.headers;
+
+    const userId = await authToken(authorization);
+
+    // 生成 data 集合
+    const data = await Data.create({
+      userId,
+      name
+    });
+
+    jsonRes(res, {
+      data: data._id
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/data/splitData.ts

@@ -4,11 +4,10 @@ import { connectToDatabase, Data, DataItem } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
 import { authToken } from '@/service/utils/tools';
 import { generateQA } from '@/service/events/generateQA';
 import { generateQA } from '@/service/events/generateQA';
 
 
-/* 定时删除那些不活跃的内容 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
   try {
-    let { text, name } = req.body as { text: string; name: string };
-    if (!text || !name) {
+    let { text, dataId } = req.body as { text: string; dataId: string };
+    if (!text || !dataId) {
       throw new Error('参数错误');
       throw new Error('参数错误');
     }
     }
     text = text.replace(/\n+/g, '\n');
     text = text.replace(/\n+/g, '\n');
@@ -18,28 +17,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
 
     const userId = await authToken(authorization);
     const userId = await authToken(authorization);
 
 
-    // 生成 data 父级
-    const data = await Data.create({
-      userId,
-      name
-    });
-
     const dataItems: any[] = [];
     const dataItems: any[] = [];
 
 
     // 格式化文本长度
     // 格式化文本长度
     for (let i = 0; i <= text.length / 1000; i++) {
     for (let i = 0; i <= text.length / 1000; i++) {
-      const dataItem = {
+      dataItems.push({
+        temperature: 0,
         userId,
         userId,
-        dataId: data._id,
+        dataId,
         text: text.slice(i * 1000, (i + 1) * 1000),
         text: text.slice(i * 1000, (i + 1) * 1000),
         status: 1
         status: 1
-      };
-
-      [0, 0.2, 0.4, 0.6, 0.8, 1.0].forEach((temperature) => {
-        dataItems.push({
-          temperature,
-          ...dataItem
-        });
       });
       });
     }
     }
 
 
@@ -58,8 +45,3 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     });
     });
   }
   }
 }
 }
-
-/**
- * 检查文本是否按格式返回
- */
-function splitText(text: string) {}

src/pages/data/components/CreateDataModal.tsx

+import React, { useState } from 'react';
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalFooter,
+  ModalBody,
+  ModalCloseButton,
+  Button,
+  Input
+} from '@chakra-ui/react';
+import { postData } from '@/api/data';
+import { useMutation } from '@tanstack/react-query';
+
+const CreateDataModal = ({
+  onClose,
+  onSuccess
+}: {
+  onClose: () => void;
+  onSuccess: () => void;
+}) => {
+  const [inputVal, setInputVal] = useState('');
+
+  const { isLoading, mutate } = useMutation({
+    mutationFn: (name: string) => postData(name),
+    onSuccess() {
+      onSuccess();
+      onClose();
+    }
+  });
+
+  return (
+    <Modal isOpen={true} onClose={onClose}>
+      <ModalOverlay />
+      <ModalContent>
+        <ModalHeader>创建数据集</ModalHeader>
+        <ModalCloseButton />
+
+        <ModalBody display={'flex'}>
+          <Input
+            value={inputVal}
+            onChange={(e) => setInputVal(e.target.value)}
+            placeholder={'数据集名称'}
+          ></Input>
+        </ModalBody>
+        <ModalFooter>
+          <Button colorScheme={'gray'} onClick={onClose}>
+            取消
+          </Button>
+          <Button
+            ml={3}
+            isDisabled={inputVal === ''}
+            isLoading={isLoading}
+            onClick={() => mutate(inputVal)}
+          >
+            确认
+          </Button>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};
+
+export default CreateDataModal;

src/pages/data/components/ImportDataModal.tsx

+import React, { useState, useCallback } from 'react';
+import {
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalFooter,
+  ModalBody,
+  ModalCloseButton,
+  Button,
+  Input,
+  Box,
+  Flex,
+  Textarea
+} from '@chakra-ui/react';
+import { useTabs } from '@/hooks/useTabs';
+import { useConfirm } from '@/hooks/useConfirm';
+import { useSelectFile } from '@/hooks/useSelectFile';
+import { readTxtContent, readPdfContent, readDocContent } from '@/utils/tools';
+import { postSplitData } from '@/api/data';
+import { useMutation } from '@tanstack/react-query';
+import { useToast } from '@/hooks/useToast';
+import { useLoading } from '@/hooks/useLoading';
+
+const ImportDataModal = ({ dataId, onClose }: { dataId: string; onClose: () => void }) => {
+  const { openConfirm, ConfirmChild } = useConfirm({
+    content: '确认提交生成任务？该任务无法终止！'
+  });
+  const { toast } = useToast();
+  const { setIsLoading, Loading } = useLoading();
+  const { File, onOpen } = useSelectFile({ fileType: '.txt,.doc,.docx,.pdf', multiple: true });
+  const { tabs, activeTab, setActiveTab } = useTabs({
+    tabs: [
+      { id: 'text', label: '文本' },
+      { id: 'doc', label: '文件' },
+      { id: 'url', label: '链接' }
+    ]
+  });
+
+  const [textInput, setTextInput] = useState('');
+  const [fileText, setFileText] = useState('');
+
+  const { mutate: handleClickSubmit, isLoading } = useMutation({
+    mutationFn: async () => {
+      let text = '';
+      if (activeTab === 'text') {
+        text = textInput;
+      } else if (activeTab === 'doc') {
+        text = fileText;
+      } else if (activeTab === 'url') {
+      }
+      if (!text) return;
+      return postSplitData(dataId, text);
+    },
+    onSuccess() {
+      toast({
+        title: '任务提交成功',
+        status: 'success'
+      });
+      onClose();
+    },
+    onError(err: any) {
+      toast({
+        title: err?.message || '提交任务异常',
+        status: 'error'
+      });
+    }
+  });
+
+  const onSelectFile = useCallback(
+    async (e: File[]) => {
+      setIsLoading(true);
+      try {
+        const fileTexts = (
+          await Promise.all(
+            e.map((file) => {
+              // @ts-ignore
+              const extension = file?.name?.split('.').pop().toLowerCase();
+              if (extension === 'txt') {
+                return readTxtContent(file);
+              } else if (extension === 'pdf') {
+                return readPdfContent(file);
+              } else if (extension === 'docx' || extension === 'doc') {
+                return readDocContent(file);
+              }
+              return '';
+            })
+          )
+        ).join('\n');
+        setFileText(fileTexts);
+      } catch (error: any) {
+        console.log(error);
+        toast({
+          title: typeof error === 'string' ? error : '解析文件失败',
+          status: 'error'
+        });
+      }
+      setIsLoading(false);
+    },
+    [setIsLoading, toast]
+  );
+
+  return (
+    <Modal isOpen={true} onClose={onClose}>
+      <ModalOverlay />
+      <ModalContent position={'relative'} maxW={['90vw', '800px']}>
+        <ModalHeader>导入数据，生成QA</ModalHeader>
+        <ModalCloseButton />
+
+        <ModalBody display={'flex'}>
+          <Box>
+            {tabs.map((item) => (
+              <Button
+                key={item.id}
+                display={'block'}
+                variant={activeTab === item.id ? 'solid' : 'outline'}
+                _notLast={{
+                  mb: 3
+                }}
+                onClick={() => setActiveTab(item.id)}
+              >
+                {item.label}
+              </Button>
+            ))}
+          </Box>
+
+          <Box flex={'1 0 0'} w={0} ml={3} minH={'200px'}>
+            {activeTab === 'text' && (
+              <Textarea
+                h={'100%'}
+                maxLength={-1}
+                value={textInput}
+                placeholder={'请粘贴或输入需要处理的文本'}
+                onChange={(e) => setTextInput(e.target.value)}
+              />
+            )}
+            {activeTab === 'doc' && (
+              <Flex
+                flexDirection={'column'}
+                h={'100%'}
+                alignItems={'center'}
+                justifyContent={'center'}
+                border={'1px solid '}
+                borderColor={'blackAlpha.200'}
+                borderRadius={'md'}
+              >
+                <Button onClick={onOpen}>选择文件</Button>
+                {fileText && <Box mt={2}>一共 {fileText.length} 个字</Box>}
+              </Flex>
+            )}
+          </Box>
+        </ModalBody>
+        <ModalFooter>
+          <Button colorScheme={'gray'} onClick={onClose}>
+            取消
+          </Button>
+          <Button
+            ml={3}
+            isLoading={isLoading}
+            isDisabled={!textInput && !fileText}
+            onClick={openConfirm(handleClickSubmit)}
+          >
+            确认
+          </Button>
+        </ModalFooter>
+        <Loading />
+      </ModalContent>
+
+      <ConfirmChild />
+      <File onSelect={onSelectFile} />
+    </Modal>
+  );
+};
+
+export default ImportDataModal;

src/pages/data/list.tsx

+import React, { useState } from 'react';
+import {
+  Card,
+  Box,
+  Flex,
+  Button,
+  Table,
+  Thead,
+  Tbody,
+  Tr,
+  Th,
+  Td,
+  TableContainer,
+  useDisclosure
+} from '@chakra-ui/react';
+import { getDataList } from '@/api/data';
+import { usePaging } from '@/hooks/usePaging';
+import type { DataListItem } from '@/types/data';
+import ScrollData from '@/components/ScrollData';
+import dayjs from 'dayjs';
+import dynamic from 'next/dynamic';
+
+const CreateDataModal = dynamic(() => import('./components/CreateDataModal'));
+const ImportDataModal = dynamic(() => import('./components/ImportDataModal'));
+
+const DataList = () => {
+  const {
+    setPageNum,
+    pageNum,
+    data: dataList,
+    getData
+  } = usePaging<DataListItem>({
+    api: getDataList,
+    pageSize: 20
+  });
+  const [ImportDataId, setImportDataId] = useState<string>();
+
+  const {
+    isOpen: isOpenCreateDataModal,
+    onOpen: onOpenCreateDataModal,
+    onClose: onCloseCreateDataModal
+  } = useDisclosure();
+
+  return (
+    <Box display={['block', 'flex']} flexDirection={'column'} h={'100%'}>
+      <Card px={6} py={4}>
+        <Flex>
+          <Box flex={1} mr={1}>
+            <Box fontSize={'xl'} fontWeight={'bold'}>
+              对话数据管理
+            </Box>
+            <Box fontSize={'xs'} color={'blackAlpha.600'}>
+              允许你将任意文本数据拆分成 QA 的形式。你可以使用这些 QA 去微调你的对话模型。
+            </Box>
+          </Box>
+          <Button variant={'outline'} onClick={onOpenCreateDataModal}>
+            创建数据集
+          </Button>
+        </Flex>
+      </Card>
+      {/* 数据表 */}
+      <Card mt={3} flex={'1 0 0'} h={['auto', '0']} px={6} py={4}>
+        <ScrollData h={'100%'} nextPage={() => setPageNum(pageNum + 1)}>
+          <TableContainer>
+            <Table>
+              <Thead>
+                <Tr>
+                  <Th>集合名</Th>
+                  <Th>创建时间</Th>
+                  <Th>训练中 / 总数据</Th>
+                  <Th></Th>
+                </Tr>
+              </Thead>
+              <Tbody>
+                {dataList.map((item, i) => (
+                  <Tr key={item._id}>
+                    <Td>{item.name}</Td>
+                    <Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td>
+                    <Td>
+                      {item.trainingData} / {item.totalData}
+                    </Td>
+                    <Td>
+                      <Button
+                        size={'sm'}
+                        variant={'outline'}
+                        mr={2}
+                        onClick={() => setImportDataId(item._id)}
+                      >
+                        导入
+                      </Button>
+                      <Button size={'sm'}>导出</Button>
+                    </Td>
+                  </Tr>
+                ))}
+              </Tbody>
+            </Table>
+          </TableContainer>
+        </ScrollData>
+      </Card>
+
+      {ImportDataId && (
+        <ImportDataModal dataId={ImportDataId} onClose={() => setImportDataId(undefined)} />
+      )}
+      {isOpenCreateDataModal && (
+        <CreateDataModal onClose={onCloseCreateDataModal} onSuccess={() => getData(1, true)} />
+      )}
+    </Box>
+  );
+};
+
+export default DataList;

src/pages/model/detail.tsx

@@ -291,15 +291,7 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
           </Box>
           </Box>
           <Flex mt={5} alignItems={'center'}>
           <Flex mt={5} alignItems={'center'}>
             <Box flex={'0 0 80px'}>删除模型:</Box>
             <Box flex={'0 0 80px'}>删除模型:</Box>
-            <Button
-              colorScheme={'red'}
-              size={'sm'}
-              onClick={() => {
-                openConfirm(() => {
-                  handleDelModel();
-                });
-              }}
-            >
+            <Button colorScheme={'red'} size={'sm'} onClick={openConfirm(handleDelModel)}>
               删除模型
               删除模型
             </Button>
             </Button>
           </Flex>
           </Flex>

src/pages/training/dataList.tsx

-import React from 'react';
-import { Card, Box, Flex, Button } from '@chakra-ui/react';
-
-const TrainDataList = () => {
-  return (
-    <>
-      <Card px={6} py={4}>
-        <Flex alignItems={'center'}>
-          <Box fontSize={'xl'} fontWeight={'bold'} flex={1}>
-            训练数据管理
-          </Box>
-          <Button variant={'outline'} mr={6}>
-            导入数据
-          </Button>
-          <Button>插入一条数据</Button>
-        </Flex>
-      </Card>
-      {/* 数据表 */}
-    </>
-  );
-};
-
-export default TrainDataList;

src/service/events/generateQA.ts

@@ -23,7 +23,7 @@ export async function generateQA(next = false): Promise<any> {
   try {
   try {
     // 找出一个需要生成的 dataItem
     // 找出一个需要生成的 dataItem
     dataItem = await DataItem.findOne({
     dataItem = await DataItem.findOne({
-      status: 1,
+      status: { $ne: 0 },
       times: { $gt: 0 }
       times: { $gt: 0 }
     });
     });
 
 
@@ -56,7 +56,8 @@ export async function generateQA(next = false): Promise<any> {
       throw new Error('获取 openai key 失败');
       throw new Error('获取 openai key 失败');
     }
     }
 
 
-    console.log('正在生成一个QA', dataItem._id);
+    console.log('正在生成一个QA, ID:', dataItem._id, 'temperature: ', dataItem.temperature / 100);
+
     const startTime = Date.now();
     const startTime = Date.now();
 
 
     // 获取 openai 请求实例
     // 获取 openai 请求实例
@@ -65,7 +66,7 @@ export async function generateQA(next = false): Promise<any> {
     const response = await chatAPI.createChatCompletion(
     const response = await chatAPI.createChatCompletion(
       {
       {
         model: ChatModelNameEnum.GPT35,
         model: ChatModelNameEnum.GPT35,
-        temperature: dataItem.temperature,
+        temperature: dataItem.temperature / 100,
         n: 1,
         n: 1,
         messages: [
         messages: [
           systemPrompt,
           systemPrompt,
@@ -76,28 +77,27 @@ export async function generateQA(next = false): Promise<any> {
         ]
         ]
       },
       },
       {
       {
-        timeout: 60000,
+        timeout: 120000,
         httpsAgent
         httpsAgent
       }
       }
     );
     );
     const content = response.data.choices[0].message?.content;
     const content = response.data.choices[0].message?.content;
     // 从 content 中提取 QA
     // 从 content 中提取 QA
     const splitResponse = splitText(content || '');
     const splitResponse = splitText(content || '');
-    if (splitResponse.length > 0) {
-      // 插入数据库，并修改状态
-      await DataItem.findByIdAndUpdate(dataItem._id, {
-        status: 0,
-        $push: {
-          result: {
-            $each: splitResponse
-          }
+    // 插入数据库，并修改状态
+    await DataItem.findByIdAndUpdate(dataItem._id, {
+      status: dataItem.temperature >= 100 ? 0 : 1,
+      temperature: dataItem.temperature >= 100 ? dataItem.temperature : dataItem.temperature + 25,
+      $push: {
+        result: {
+          $each: splitResponse
         }
         }
-      });
-      console.log('生成成功，time:', `${(Date.now() - startTime) / 1000}s`);
-    }
+      }
+    });
+    console.log('生成成功，time:', `${(Date.now() - startTime) / 1000}s`);
   } catch (error: any) {
   } catch (error: any) {
     console.log('error: 生成QA错误', dataItem?._id);
     console.log('error: 生成QA错误', dataItem?._id);
-    console.log('statusText:', error?.response?.statusText);
+    console.log('response:', error?.response);
     // 重置状态
     // 重置状态
     if (dataItem?._id) {
     if (dataItem?._id) {
       await DataItem.findByIdAndUpdate(dataItem._id, {
       await DataItem.findByIdAndUpdate(dataItem._id, {

src/service/models/bill.ts

 import { Schema, model, models } from 'mongoose';
 import { Schema, model, models } from 'mongoose';
-import { ModelList } from '@/constants/model';
+import { modelList } from '@/constants/model';
 
 
 const BillSchema = new Schema({
 const BillSchema = new Schema({
   userId: {
   userId: {
@@ -14,7 +14,7 @@ const BillSchema = new Schema({
   },
   },
   modelName: {
   modelName: {
     type: String,
     type: String,
-    enum: ModelList.map((item) => item.model),
+    enum: modelList.map((item) => item.model),
     required: true
     required: true
   },
   },
   chatId: {
   chatId: {

src/types/data.d.ts

+import type { DataSchema } from './mongoSchema';
+
+export interface DataListItem extends DataSchema {
+  trainingData: number;
+  totalData: number;
+}

src/types/index.d.ts

@@ -4,6 +4,9 @@ declare global {
   var mongodb: Mongoose | string | null;
   var mongodb: Mongoose | string | null;
   var generatingQA: boolean;
   var generatingQA: boolean;
   var QRCode: any;
   var QRCode: any;
+  interface Window {
+    ['pdfjs-dist/build/pdf']: any;
+  }
 }
 }
 
 
 export type PagingData<T> = {
 export type PagingData<T> = {

src/utils/tools.ts

 import crypto from 'crypto';
 import crypto from 'crypto';
 import { useToast } from '@/hooks/useToast';
 import { useToast } from '@/hooks/useToast';
+import mammoth from 'mammoth';
 
 
 /**
 /**
  * copy text data
  * copy text data
@@ -50,17 +51,76 @@ export const Obj2Query = (obj: Record<string, string | number>) => {
 };
 };
 
 
 /**
 /**
- * 读取文件内容
+ * 读取 txt 文件内容
  */
  */
-export const loadLocalFileContent = (file: File) => {
+export const readTxtContent = (file: File) => {
   return new Promise((resolve: (_: string) => void, reject) => {
   return new Promise((resolve: (_: string) => void, reject) => {
     const reader = new FileReader();
     const reader = new FileReader();
     reader.onload = () => {
     reader.onload = () => {
       resolve(reader.result as string);
       resolve(reader.result as string);
     };
     };
     reader.onerror = (err) => {
     reader.onerror = (err) => {
-      reject(err);
+      console.log('error txt read:', err);
+      reject('读取 txt 文件失败');
     };
     };
     reader.readAsText(file);
     reader.readAsText(file);
   });
   });
 };
 };
+
+/**
+ * 读取 pdf 内容
+ */
+export const readPdfContent = (file: File) =>
+  new Promise<string>((resolve, reject) => {
+    const pdfjsLib = window['pdfjs-dist/build/pdf'];
+    pdfjsLib.workerSrc = '/js/pdf.worker.js';
+
+    const readPDFPage = async (doc: any, pageNo: number) => {
+      const page = await doc.getPage(pageNo);
+      const tokenizedText = await page.getTextContent();
+      const pageText = tokenizedText.items.map((token: any) => token.str).join('');
+      return pageText.replaceAll(/\s+/g, '\n');
+    };
+
+    let reader = new FileReader();
+    reader.readAsArrayBuffer(file);
+    reader.onload = async (event) => {
+      if (!event?.target?.result) return reject('解析 PDF 失败');
+      try {
+        const doc = await pdfjsLib.getDocument(event.target.result).promise;
+        const pageTextPromises = [];
+        for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) {
+          pageTextPromises.push(readPDFPage(doc, pageNo));
+        }
+        const pageTexts = await Promise.all(pageTextPromises);
+        resolve(pageTexts.join('\n'));
+      } catch (err) {
+        console.log(err, 'pdfjs error');
+        reject('解析 PDF 失败');
+      }
+    };
+    reader.onerror = (err) => {
+      console.log(err, 'reader error');
+      reject('解析 PDF 失败');
+    };
+  });
+
+/**
+ * 读取doc
+ */
+export const readDocContent = (file: File) =>
+  new Promise<string>((resolve, reject) => {
+    const reader = new FileReader();
+    reader.readAsArrayBuffer(file);
+    reader.onload = ({ target }) => {
+      if (!target?.result) return reject('读取 doc 文件失败');
+      return mammoth.extractRawText({ arrayBuffer: target.result as ArrayBuffer }).then((res) => {
+        resolve(res.value);
+      });
+    };
+    reader.onerror = (err) => {
+      console.log('error doc read:', err);
+
+      reject('读取 doc 文件失败');
+    };
+  });