Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 79177660 by Archer Committed by GitHub
Options

Dataset folder manager (#274) 

* feat: retry send

* perf: qa default value

* feat: dataset folder

* feat: kb folder delete and path

* fix: ts

* perf: script load

* feat: fileCard and dataCard

* feat: search file

* feat: max token

* feat: select dataset

* fix: preview chunk

* perf: source update

* export data limit file_id

* docs

* fix: export limit
parent a1a63260
Showing with 1770 additions and 592 deletions
README.md
......@@ -48,7 +48,7 @@ FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开
- [x] 支持 url 读取、 CSV 批量导入
- [x] 支持知识库单独设置向量模型
- [x] 源文件存储
- [x] 文件学习 Agent
- [ ] 文件学习 Agent
3. 多种效果测试渠道
- [x] 知识库单点搜索测试
- [x] 对话时反馈引用并可修改与删除
......
client/data/config.json
......@@ -9,7 +9,7 @@
"show_doc": true,
"systemTitle": "FastGPT",
"authorText": "Made by FastGPT Team.",
"gitLoginKey": "",
"exportLimitMinutes": 0,
"scripts": []
},
"SystemParams": {
......@@ -61,4 +61,4 @@
"maxToken": 16000,
"price": 0
}
}
\ No newline at end of file
}
client/public/docs/versionIntro.md
### Fast GPT V4.3
### Fast GPT V4.4
1. 新增 - 知识库源文件存储，可以从引用窗口点击文件名，查看源文件。
2. 新增 - 用户反馈和管理员标注预期答案，以不断提高模型回复准确率。 该功能为测试版，未来交互可能会有变化，欢迎大家提出宝贵意见。
3. 优化 - [使用文档](https://doc.fastgpt.run/docs/intro/)
4. [点击查看高级编排介绍文档](https://doc.fastgpt.run/docs/workflow)
5. [点击查看商业版](https://fael3z0zfze.feishu.cn/docx/F155dbirfo8vDDx2WgWc6extnwf)
1. 新增 - 知识库目录结构
2. 优化 - [使用文档](https://doc.fastgpt.run/docs/intro/)
3. [点击查看高级编排介绍文档](https://doc.fastgpt.run/docs/workflow)
4. [点击查看商业版](https://fael3z0zfze.feishu.cn/docx/F155dbirfo8vDDx2WgWc6extnwf)
client/public/imgs/files/file.svg 0 → 100644
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694327751771" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4992" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M0 0h1024v1024H0V0z" fill="#202425" opacity=".01" p-id="4993"></path><path d="M136.533333 68.266667a68.266667 68.266667 0 0 0-68.266666 68.266666v428.305067a17.066667 17.066667 0 0 0 28.842666 12.356267l237.738667-226.440534a34.133333 34.133333 0 0 1 42.496-3.6864l268.288 178.858667a17.066667 17.066667 0 0 0 22.766933-3.447467L951.978667 171.4176A17.066667 17.066667 0 0 0 955.733333 160.699733V136.533333a68.266667 68.266667 0 0 0-68.266666-68.266666H136.533333z m819.2 255.3856a17.066667 17.066667 0 0 0-30.344533-10.717867l-221.866667 274.705067a17.066667 17.066667 0 0 0-3.7888 10.717866v340.309334a17.066667 17.066667 0 0 0 17.066667 17.066666h170.666667a68.266667 68.266667 0 0 0 68.266666-68.266666V323.652267zM614.4 955.733333a17.066667 17.066667 0 0 0 17.066667-17.066666v-330.990934a17.066667 17.066667 0 0 0-7.611734-14.199466l-204.8-136.533334a17.066667 17.066667 0 0 0-26.5216 14.199467V938.666667a17.066667 17.066667 0 0 0 17.066667 17.066666h204.8z m-307.2 0a17.066667 17.066667 0 0 0 17.066667-17.066666v-443.733334a17.066667 17.066667 0 0 0-28.842667-12.356266l-221.866667 211.285333a17.066667 17.066667 0 0 0-5.290666 12.3904V887.466667a68.266667 68.266667 0 0 0 68.266666 68.266666h170.666667z" fill="#FFAA44" p-id="4994"></path><path d="M73.557333 693.8624a17.066667 17.066667 0 0 0-5.290666 12.3904V887.466667a68.266667 68.266667 0 0 0 68.266666 68.266666h170.666667a17.066667 17.066667 0 0 0 17.066667-17.066666v-443.733334a17.066667 17.066667 0 0 0-28.842667-12.356266l-221.866667 211.285333zM392.533333 938.666667a17.066667 17.066667 0 0 0 17.066667 17.066666h204.8a17.066667 17.066667 0 0 0 17.066667-17.066666v-330.990934a17.066667 17.066667 0 0 0-7.611734-14.199466l-204.8-136.533334a17.066667 17.066667 0 0 0-26.5216 14.199467V938.666667z m307.2 0a17.066667 17.066667 0 0 0 17.066667 17.066666h170.666667a68.266667 68.266667 0 0 0 68.266666-68.266666V323.6864a17.066667 17.066667 0 0 0-30.344533-10.752l-221.866667 274.705067a17.066667 17.066667 0 0 0-3.7888 10.717866v340.309334z" fill="#11AA66" p-id="4995"></path></svg>
\ No newline at end of file
client/public/imgs/files/folder.svg 0 → 100644
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694141197423" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4891" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M855.04 385.024q19.456 2.048 38.912 10.24t33.792 23.04 21.504 37.376 2.048 54.272q-2.048 8.192-8.192 40.448t-14.336 74.24-18.432 86.528-19.456 76.288q-5.12 18.432-14.848 37.888t-25.088 35.328-36.864 26.112-51.2 10.24l-567.296 0q-21.504 0-44.544-9.216t-42.496-26.112-31.744-40.96-12.288-53.76l0-439.296q0-62.464 33.792-97.792t95.232-35.328l503.808 0q22.528 0 46.592 8.704t43.52 24.064 31.744 35.84 12.288 44.032l0 11.264-53.248 0q-40.96 0-95.744-0.512t-116.736-0.512-115.712-0.512-92.672-0.512l-47.104 0q-26.624 0-41.472 16.896t-23.04 44.544q-8.192 29.696-18.432 62.976t-18.432 61.952q-10.24 33.792-20.48 65.536-2.048 8.192-2.048 13.312 0 17.408 11.776 29.184t29.184 11.776q31.744 0 43.008-39.936l54.272-198.656q133.12 1.024 243.712 1.024l286.72 0z" fill="#FFCC66" p-id="4892"></path></svg>
\ No newline at end of file
client/public/locales/en/common.json
......@@ -2,6 +2,10 @@
"App": "App",
"Cancel": "No",
"Confirm": "Yes",
"Create New": "Create",
"Dataset": "Dataset",
"Folder": "Folder",
"Name": "Name",
"Running": "Running",
"Select value is empty": "Select value is empty",
"UnKnow": "UnKnow",
......@@ -63,10 +67,8 @@
"online": "Online Chat",
"share": "Share",
"test": "Test Chat "
}
},
"commom": {
"Password inconsistency": "Password inconsistency"
},
"retry": "Retry"
},
"common": {
"Add": "Add",
......@@ -76,14 +78,22 @@
"Copy Successful": "Copy Successful",
"Course": "",
"Delete": "Delete",
"Delete Failed": "Delete Failed",
"Delete Success": "Delete Successful",
"Delete Warning": "Warning",
"Filed is repeat": "Filed is repeated",
"Filed is repeated": "",
"Input": "Input",
"Output": "Output",
"Password inconsistency": "Password inconsistency",
"Rename": "Rename",
"Search": "Search",
"Status": "Status",
"export": ""
},
"dataset": {
"Confirm to delete the data": "Confirm to delete the data?",
"Export": "Export",
"Queue Desc": "This data refers to the current amount of training for the entire system. FastGPT uses queued training, and if you have too much data to train, you may need to wait for a while",
"System Data Queue": "Data Queue"
},
......@@ -93,9 +103,11 @@
"Create File": "Create File",
"Create file": "Create file",
"Drag and drop": "Drag and drop files here",
"Embedding": "Embedding",
"Fetch Url": "Fetch Url",
"If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format",
"Parse": "{{name}} Parsing...",
"Ready": "Ready",
"Release the mouse to upload the file": "Release the mouse to upload the file",
"Select a maximum of 10 files": "Select a maximum of 10 files",
"Uploading": "Uploading: {{name}}, Progress: {{percent}}%",
......@@ -149,6 +161,24 @@
"desc": "AI knowledge base question and answer platform based on LLM large model",
"slogan": "Let the AI know more about you"
},
"kb": {
"Chunk Length": "Chunk Length",
"Confirm to delete the file": "Are you sure to delete the file and all its data?",
"Create Folder": "Create Folder",
"Delete Dataset Error": "Delete dataset failed",
"Edit Folder": "Edit Folder",
"File Size": "File Size",
"Filename": "Filename",
"Files": "{{total}} Files",
"Folder Name": "Input folder name",
"My Dataset": "My Dataset",
"Other Data": "Other Data",
"Select Dataset": "Select Dataset",
"Select Folder": "Enter folder",
"Upload Time": "Upload Time",
"deleteDatasetTips": "Are you sure to delete the knowledge base? Data cannot be recovered after deletion, please confirm!",
"deleteFolderTips": "Are you sure to delete this folder and all the knowledge bases it contains? Data cannot be recovered after deletion, please confirm!"
},
"navbar": {
"Account": "Account",
"Apps": "Apps",
......
client/public/locales/zh/common.json
......@@ -2,6 +2,10 @@
"App": "应用",
"Cancel": "取消",
"Confirm": "确认",
"Create New": "新建",
"Dataset": "知识库",
"Folder": "文件夹",
"Name": "名称",
"Running": "运行中",
"Select value is empty": "选择的内容为空",
"UnKnow": "未知",
......@@ -63,10 +67,8 @@
"online": "在线使用",
"share": "外部链接调用",
"test": "测试"
}
},
"commom": {
"Password inconsistency": "两次密码不一致"
},
"retry": "重新生成"
},
"common": {
"Add": "添加",
......@@ -76,14 +78,22 @@
"Copy Successful": "复制成功",
"Course": "",
"Delete": "删除",
"Delete Failed": "删除失败",
"Delete Success": "删除成功",
"Delete Warning": "删除警告",
"Filed is repeat": "",
"Filed is repeated": "字段重复了",
"Input": "输入",
"Output": "输出",
"Password inconsistency": "两次密码不一致",
"Rename": "重命名",
"Search": "搜索",
"Status": "状态",
"export": ""
},
"dataset": {
"Confirm to delete the data": "确认删除该数据？",
"Export": "导出",
"Queue Desc": "该数据是指整个系统当前待训练的数量。{{title}} 采用排队训练的方式，如果待训练的数据过多，可能需要等待一段时间",
"System Data Queue": "排队长度"
},
......@@ -93,9 +103,11 @@
"Create File": "创建新文件",
"Create file": "创建文件",
"Drag and drop": "拖拽文件至此",
"Embedding": "索引中",
"Fetch Url": "链接读取",
"If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式",
"Parse": "{{name}} 解析中...",
"Ready": "可用",
"Release the mouse to upload the file": "松开鼠标上传文件",
"Select a maximum of 10 files": "最多选择10个文件",
"Uploading": "正在上传 {{name}}，进度: {{percent}}%",
......@@ -149,6 +161,24 @@
"desc": "基于 LLM 大模型的 AI 知识库问答平台",
"slogan": "让 AI 更懂你的知识"
},
"kb": {
"Chunk Length": "数据总量",
"Confirm to delete the file": "确认删除该文件及其所有数据？",
"Create Folder": "创建文件夹",
"Delete Dataset Error": "删除知识库异常",
"Edit Folder": "编辑文件夹",
"File Size": "文件大小",
"Filename": "文件名",
"Files": "文件: {{total}}个",
"Folder Name": "输入文件夹名称",
"My Dataset": "我的知识库",
"Other Data": "其他数据",
"Select Dataset": "选择该知识库",
"Select Folder": "进入文件夹",
"Upload Time": "上传时间",
"deleteDatasetTips": "确认删除该知识库？删除后数据无法恢复，请确认！",
"deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！"
},
"navbar": {
"Account": "账号",
"Apps": "应用",
......
client/src/api/plugins/kb.ts
import { GET, POST, PUT, DELETE } from '../request';
import type { DatasetItemType, KbItemType, KbListItemType } from '@/types/plugin';
import { RequestPaging } from '@/types/index';
import type {
DatasetItemType,
FileInfo,
KbFileItemType,
KbItemType,
KbListItemType,
KbPathItemType
} from '@/types/plugin';
import { TrainingModeEnum } from '@/constants/plugin';
import {
Props as PushDataProps,
......@@ -10,13 +16,17 @@ import {
Props as SearchTestProps,
Response as SearchTestResponse
} from '@/pages/api/openapi/kb/searchTest';
import { Response as KbDataItemType } from '@/pages/api/plugins/kb/data/getDataById';
import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData';
import type { KbUpdateParams, CreateKbParams } from '../request/kb';
import type { KbUpdateParams, CreateKbParams, GetKbDataListProps } from '../request/kb';
import { QuoteItemType } from '@/types/chat';
/* knowledge base */
export const getKbList = () => GET<KbListItemType[]>(`/plugins/kb/list`);
export const getKbList = (parentId?: string) =>
GET<KbListItemType[]>(`/plugins/kb/list`, { parentId });
export const getAllDataset = () => GET<KbListItemType[]>(`/plugins/kb/allDataset`);
export const getKbPaths = (parentId?: string) =>
GET<KbPathItemType[]>('/plugins/kb/paths', { parentId });
export const getKbById = (id: string) => GET<KbItemType>(`/plugins/kb/detail?id=${id}`);
......@@ -26,25 +36,27 @@ export const putKbById = (data: KbUpdateParams) => PUT(`/plugins/kb/update`, dat
export const delKbById = (id: string) => DELETE(`/plugins/kb/delete?id=${id}`);
/* kb file */
export const getKbFiles = (data: { kbId: string; searchText: string }) =>
GET<KbFileItemType[]>(`/plugins/kb/file/list`, data);
export const deleteKbFileById = (params: { fileId: string; kbId: string }) =>
DELETE(`/plugins/kb/file/delFileByFileId`, params);
export const getFileInfoById = (fileId: string) =>
GET<FileInfo>(`/plugins/kb/file/getFileInfo`, { fileId });
export const delEmptyFiles = (kbId: string) =>
DELETE(`/plugins/kb/file/deleteEmptyFiles`, { kbId });
/* kb data */
type GetKbDataListProps = RequestPaging & {
kbId: string;
searchText: string;
};
export const getKbDataList = (data: GetKbDataListProps) =>
POST(`/plugins/kb/data/getDataList`, data);
/**
* 获取导出数据（不分页）
*/
export const getExportDataList = (kbId: string) =>
GET<[string, string, string][]>(
`/plugins/kb/data/exportModelData`,
{ kbId },
{
timeout: 600000
}
);
export const getExportDataList = (data: { kbId: string; fileId: string }) =>
GET<[string, string, string][]>(`/plugins/kb/data/exportModelData`, data, {
timeout: 600000
});
/**
* 获取模型正在拆分数据的数量
......
client/src/api/request/kb.d.ts
import { KbTypeEnum } from '@/constants/kb';
import type { RequestPaging } from '@/types';
export type KbUpdateParams = {
id: string;
name: string;
tags: string;
avatar: string;
tags?: string;
name?: string;
avatar?: string;
};
export type CreateKbParams = {
parentId?: string;
name: string;
tags: string[];
avatar: string;
vectorModel: string;
vectorModel?: string;
type: `${KbTypeEnum}`;
};
export type GetKbDataListProps = RequestPaging & {
kbId: string;
searchText: string;
fileId: string;
};
client/src/components/ChatBox/QuoteModal.tsx
......@@ -19,7 +19,7 @@ const QuoteModal = ({
rawSearch = [],
onClose
}: {
onUpdateQuote: (quoteId: string, sourceText: string) => Promise<void>;
onUpdateQuote: (quoteId: string, sourceText?: string) => Promise<void>;
rawSearch: SearchType[];
onClose: () => void;
}) => {
......@@ -129,7 +129,7 @@ const QuoteModal = ({
{editDataItem && (
<InputDataModal
onClose={() => setEditDataItem(undefined)}
onSuccess={() => onUpdateQuote(editDataItem.id, '手动修改')}
onSuccess={() => onUpdateQuote(editDataItem.id)}
onDelete={() => onUpdateQuote(editDataItem.id, '已删除')}
kbId={editDataItem.kb_id}
defaultValues={{
......
client/src/components/ChatBox/ResponseTags.tsx
......@@ -44,7 +44,7 @@ const ResponseTags = ({
};
}, [responseData]);
const updateQuote = useCallback(async (quoteId: string, sourceText: string) => {}, []);
const updateQuote = useCallback(async (quoteId: string, sourceText?: string) => {}, []);
const TagStyles: BoxProps = {
mr: 2,
......
client/src/components/ChatBox/SelectDataset.tsx
......@@ -12,7 +12,7 @@ import {
import MyModal from '../MyModal';
import { useTranslation } from 'next-i18next';
import { useQuery } from '@tanstack/react-query';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
import { useToast } from '@/hooks/useToast';
import Avatar from '../Avatar';
import MyIcon from '@/components/Icon';
......@@ -29,10 +29,10 @@ const SelectDataset = ({
const theme = useTheme();
const { isPc } = useGlobalStore();
const { toast } = useToast();
const { myKbList, loadKbList } = useUserStore();
const { myKbList, loadKbList } = useDatasetStore();
const [selectedId, setSelectedId] = useState<string>();
useQuery(['loadKbList'], loadKbList);
useQuery(['loadKbList'], () => loadKbList());
return (
<MyModal isOpen={true} onClose={onClose} w={'100%'} maxW={['90vw', '900px']} isCentered={!isPc}>
......
client/src/components/ChatBox/WholeResponseModal.tsx
import React, { useMemo } from 'react';
import { Box, ModalBody, useTheme, ModalHeader, Flex } from '@chakra-ui/react';
import { Box, ModalBody, useTheme, Flex } from '@chakra-ui/react';
import type { ChatHistoryItemResType } from '@/types/chat';
import { useTranslation } from 'react-i18next';
......
client/src/components/ChatBox/index.tsx
......@@ -55,6 +55,7 @@ const SelectDataset = dynamic(() => import('./SelectDataset'));
const InputDataModal = dynamic(() => import('@/pages/kb/detail/components/InputDataModal'));
import styles from './index.module.scss';
import Script from 'next/script';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24);
......@@ -293,7 +294,7 @@ const ChatBox = (
* user confirm send prompt
*/
const sendPrompt = useCallback(
async (variables: Record<string, any> = {}, inputVal = '') => {
async (variables: Record<string, any> = {}, inputVal = '', history = chatHistory) => {
if (!onStartChat) return;
if (isChatting) {
toast({
......@@ -314,7 +315,7 @@ const ChatBox = (
}
const newChatList: ChatSiteItemType[] = [
...chatHistory,
...history,
{
dataId: nanoid(),
obj: 'Human',
......@@ -407,6 +408,22 @@ const ChatBox = (
]
);
// retry input
const retryInput = useCallback(
async (index: number) => {
if (!onDelMessage) return;
const delHistory = chatHistory.slice(index);
setChatHistory((state) => (index === 0 ? [] : state.slice(0, index)));
await Promise.all(
delHistory.map((item, i) => onDelMessage({ contentId: item.dataId, index: index + i }))
);
sendPrompt(variables, delHistory[0].value, chatHistory.slice(0, index));
},
[chatHistory, onDelMessage, sendPrompt, variables]
);
// output data
useImperativeHandle(ref, () => ({
getChatHistory: () => chatHistory,
......@@ -470,7 +487,7 @@ const ChatBox = (
);
const statusBoxData = useMemo(() => {
const colorMap = {
loading: '#67c13b',
loading: 'myGray.700',
running: '#67c13b',
finish: 'myBlue.600'
};
......@@ -484,6 +501,7 @@ const ChatBox = (
};
}, [chatHistory, isChatting, t]);
// page change and abort request
useEffect(() => {
return () => {
controller.current?.abort('leave');
......@@ -492,16 +510,7 @@ const ChatBox = (
};
}, [router.query]);
useEffect(() => {
event.on('guideClick', ({ text }: { text: string }) => {
if (!text) return;
handleSubmit((data) => sendPrompt(data, text))();
});
return () => {
event.off('guideClick');
};
}, [handleSubmit, sendPrompt]);
// page destroy and abort request
useEffect(() => {
const listen = () => {
cancelBroadcast();
......@@ -513,8 +522,22 @@ const ChatBox = (
};
}, []);
// add guide text listener
useEffect(() => {
event.on('guideClick', ({ text }: { text: string }) => {
if (!text) return;
handleSubmit((data) => sendPrompt(data, text))();
});
return () => {
event.off('guideClick');
};
}, [handleSubmit, sendPrompt]);
return (
<Flex flexDirection={'column'} h={'100%'}>
<Script src="/js/html2pdf.bundle.min.js" strategy="lazyOnload"></Script>
<Box ref={ChatBoxRef} flex={'1 0 0'} h={0} w={'100%'} overflow={'overlay'} px={[4, 0]} pb={3}>
<Box id="chat-container" maxW={['100%', '92%']} h={'100%'} mx={'auto'}>
{showEmpty && <Empty />}
......@@ -616,7 +639,7 @@ const ChatBox = (
justifyContent={'flex-end'}
mr={3}
>
<MyTooltip label={'复制'}>
<MyTooltip label={t('common.Copy')}>
<MyIcon
{...controlIconStyle}
name={'copy'}
......@@ -624,8 +647,18 @@ const ChatBox = (
onClick={() => onclickCopy(item.value)}
/>
</MyTooltip>
{!!onDelMessage && (
<MyTooltip label={t('chat.retry')}>
<MyIcon
{...controlIconStyle}
name={'retryLight'}
_hover={{ color: 'green.500' }}
onClick={() => retryInput(index)}
/>
</MyTooltip>
)}
{onDelMessage && (
<MyTooltip label={'删除'}>
<MyTooltip label={t('common.Delete')}>
<MyIcon
{...controlIconStyle}
mr={0}
......
client/src/components/Icon/icons/fill/plus.svg 0 → 100644
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694331723034"
class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5978"
xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64">
<path
d="M512 70.283C267.486 70.283 69.268 268.046 69.268 512S267.486 953.717 512 953.717 954.732 755.954 954.732 512 756.514 70.283 512 70.283m223.045 488.321H558.603v176.442c0 25.738-20.866 46.604-46.604 46.604s-46.604-20.866-46.604-46.604V558.604H288.953c-25.738 0-46.604-20.866-46.604-46.604s20.866-46.604 46.604-46.604h176.442V288.954c0-25.738 20.866-46.604 46.604-46.604s46.604 20.866 46.604 46.604v176.442h176.442c25.738 0 46.604 20.866 46.604 46.604s-20.866 46.604-46.604 46.604z"
p-id="5979"></path>
</svg>
\ No newline at end of file
client/src/components/Icon/icons/light/retry.svg 0 → 100644
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694067364830"
class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5118"
xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64">
<path
d="M727.950222 274.773333l-55.296-9.329777a38.741333 38.741333 0 0 0-12.856889 76.344888l193.308445 32.597334c1.991111 0.113778 1.991111 0.113778 2.844444 0 2.275556 0.227556 4.266667 0.113778 7.850667-0.284445l0.682667-0.056889a28.216889 28.216889 0 0 0 5.632-0.967111c1.080889 0 1.080889 0 3.185777-0.568889a15.530667 15.530667 0 0 0 4.039111-2.332444l1.137778-0.796444 0.796445-0.398223a28.444444 28.444444 0 0 0 4.152889-2.730666 37.091556 37.091556 0 0 0 6.542222-6.826667l0.796444-0.967111c1.080889-1.422222 1.080889-1.422222 2.161778-3.128889a37.432889 37.432889 0 0 0 3.697778-9.557333c0.568889-1.194667 0.568889-1.194667 1.137778-3.128889 0.113778-1.763556 0.113778-1.763556 0-2.503111v0.910222a36.579556 36.579556 0 0 0-0.341334-10.24l-0.113778-0.967111a22.755556 22.755556 0 0 0-0.682666-3.982222c0-1.080889 0-1.080889-0.568889-3.128889l-68.494222-183.751111a38.798222 38.798222 0 0 0-49.777778-22.755556 38.798222 38.798222 0 0 0-22.755556 49.777778l16.270223 43.804444A397.880889 397.880889 0 0 0 512 113.777778C292.408889 113.777778 113.777778 292.408889 113.777778 512s178.631111 398.222222 398.222222 398.222222 398.222222-178.631111 398.222222-398.222222a38.684444 38.684444 0 1 0-77.368889 0c0 176.924444-143.928889 320.853333-320.853333 320.853333S191.146667 688.924444 191.146667 512 335.075556 191.146667 512 191.146667c80.099556 0 157.070222 29.980444 215.950222 83.626666z"
p-id="5119"></path>
</svg>
\ No newline at end of file
client/src/components/Icon/icons/light/rightArrow.svg 0 → 100644
<svg width="24" height="24" viewBox="0 0 24 24">
<path d="M8.3 5.7a1 1 0 011.4-1.4l7.71 7.7-7.7 7.7a1 1 0 11-1.42-1.4l6.3-6.3-6.3-6.3z" fill-rule="nonzero"></path>
</svg>
\ No newline at end of file
client/src/components/Icon/icons/light/search.svg 0 → 100644
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694224177076"
class="icon" viewBox="0 0 1026 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="3984"
xmlns:xlink="http://www.w3.org/1999/xlink" width="64.125" height="64">
<path
d="M989.365124 873.455175c21.85764 24.973422 33.760499 46.831061 35.714294 65.567202 1.948078 18.730424-5.271103 37.076377-21.661831 55.026425-18.736141 21.075836-39.416072 31.030616-62.055515 29.858625-22.633727-1.171991-44.491366-10.344968-65.567202-27.513213L679.093265 806.715982c-35.128298 22.633727-72.786383 40.197876-112.989976 52.68673-40.197876 12.488855-82.545355 18.730424-127.036721 18.730424-60.882095 0-117.863745-11.512672-170.940663-34.536586-53.078347-23.029631-99.523509-54.446147-139.331197-94.252406-39.811976-39.811976-71.228492-86.25285-94.252406-139.329768C11.512672 556.93603 0 499.950092 0 439.066568c0-60.883524 11.512672-117.863745 34.542303-170.940663 23.023914-53.078347 54.44043-99.523509 94.252406-139.331197 39.807688-39.811976 86.25285-71.228492 139.331197-94.252406 53.076918-23.029631 110.058568-34.542303 170.940663-34.542303 60.883524 0 117.869462 11.512672 170.94638 34.542303 53.078347 23.023914 99.517792 54.44043 139.329768 94.252406 39.807688 39.807688 71.222775 86.25285 94.252406 139.331197 23.023914 53.076918 34.536586 110.057139 34.536586 170.940663 0 46.054974-6.633185 89.764536-19.903844 131.134403s-32.002511 79.619664-56.198417 114.742246l38.639985 38.639985c18.730424 18.730424 38.439889 38.249797 59.124108 58.543829s39.61188 39.416072 56.784413 57.371837C973.754771 857.448917 984.680017 868.771497 989.365124 873.455175L989.365124 873.455175zM443.751675 731.779995c40.588063 0 78.83786-7.609369 114.742246-22.829535 35.904385-15.224454 67.13081-36.105911 93.66641-62.641511 26.541317-26.541317 47.422774-57.762025 62.641511-93.66641 15.218737-35.910102 22.835252-74.154183 22.835252-114.747963 0-40.589492-7.615086-78.832143-22.835252-114.742246-15.218737-35.905815-36.100194-67.125093-62.641511-93.667839-26.5356-26.5356-57.762025-47.415628-93.66641-62.641511-35.904385-15.218737-74.154183-22.828106-114.742246-22.828106-40.589492 0-78.83929 7.609369-114.743675 22.828106-35.904385 15.225883-67.129381 36.105911-93.66641 62.641511-26.541317 26.542747-47.422774 57.762025-62.641511 93.667839-15.218737 35.910102-22.829535 74.152753-22.829535 114.742246 0 40.59378 7.610798 78.83786 22.829535 114.747963 15.218737 35.904385 36.100194 67.125093 62.641511 93.66641 26.53703 26.5356 57.762025 47.417057 93.66641 62.641511C364.912385 724.170627 403.162183 731.779995 443.751675 731.779995L443.751675 731.779995zM443.751675 731.779995"
p-id="3985"></path>
</svg>
\ No newline at end of file
client/src/components/Icon/index.tsx
......@@ -79,7 +79,11 @@ const map = {
promotionLight: require('./icons/light/promotion.svg').default,
logsLight: require('./icons/light/logs.svg').default,
badLight: require('./icons/light/bad.svg').default,
markLight: require('./icons/light/mark.svg').default
markLight: require('./icons/light/mark.svg').default,
retryLight: require('./icons/light/retry.svg').default,
rightArrowLight: require('./icons/light/rightArrow.svg').default,
searchLight: require('./icons/light/search.svg').default,
plusFill: require('./icons/fill/plus.svg').default
};
export type IconName = keyof typeof map;
......
client/src/components/MyInput/index.tsx 0 → 100644
import React from 'react';
import { Flex, Input, InputProps } from '@chakra-ui/react';
interface Props extends InputProps {
leftIcon?: React.ReactNode;
}
const MyInput = ({ leftIcon, ...props }: Props) => {
return (
<Flex position={'relative'} alignItems={'center'}>
<Input w={'100%'} pl={leftIcon ? '30px' : 3} {...props} />
{leftIcon && (
<Flex
alignItems={'center'}
position={'absolute'}
left={3}
w={'20px'}
zIndex={10}
transform={'translateY(1.5px)'}
>
{leftIcon}
</Flex>
)}
</Flex>
);
};
export default MyInput;
client/src/components/MyMenu/index.tsx 0 → 100644
import React from 'react';
import { Menu, MenuList, MenuItem } from '@chakra-ui/react';
interface Props {
width: number;
offset?: [number, number];
Button: React.ReactNode;
menuList: {
isActive?: boolean;
child: React.ReactNode;
onClick: () => void;
}[];
}
const MyMenu = ({ width, offset = [0, 10], Button, menuList }: Props) => {
const menuItemStyles = {
borderRadius: 'sm',
py: 3,
display: 'flex',
alignItems: 'center',
_hover: {
backgroundColor: 'myWhite.600',
color: 'hover.blue'
}
};
return (
<Menu offset={offset} autoSelect={false} isLazy>
{Button}
<MenuList
minW={`${width}px !important`}
p={'6px'}
border={'1px solid #fff'}
boxShadow={'0px 2px 4px rgba(161, 167, 179, 0.25), 0px 0px 1px rgba(121, 141, 159, 0.25);'}
>
{menuList.map((item, i) => (
<MenuItem
key={i}
{...menuItemStyles}
onClick={item.onClick}
color={item.isActive ? 'hover.blue' : ''}
>
{item.child}
</MenuItem>
))}
</MenuList>
</Menu>
);
};
export default MyMenu;
client/src/constants/common.ts
......@@ -10,7 +10,8 @@ export const fileImgs = [
{ suffix: 'csv', src: '/imgs/files/csv.svg' },
{ suffix: '(doc|docs)', src: '/imgs/files/doc.svg' },
{ suffix: 'txt', src: '/imgs/files/txt.svg' },
{ suffix: 'md', src: '/imgs/files/markdown.svg' }
{ suffix: 'md', src: '/imgs/files/markdown.svg' },
{ suffix: '.', src: '/imgs/files/file.svg' }
];
export enum TrackEventName {
......
client/src/constants/kb.ts
......@@ -14,3 +14,24 @@ export const defaultKbDetail: KbItemType = {
maxToken: 3000
}
};
export enum KbTypeEnum {
folder = 'folder',
dataset = 'dataset'
}
export enum FileStatusEnum {
embedding = 'embedding',
ready = 'ready'
}
export const KbTypeMap = {
[KbTypeEnum.folder]: {
name: 'folder'
},
[KbTypeEnum.dataset]: {
name: 'dataset'
}
};
export const FolderAvatarSrc = '/imgs/files/folder.svg';
export const OtherFileId = 'other';
client/src/hooks/useConfirm.tsx
import { useCallback, useRef } from 'react';
import { useCallback, useRef, useState } from 'react';
import {
AlertDialog,
AlertDialogBody,
......@@ -11,21 +11,25 @@ import {
} from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
export const useConfirm = (props: { title?: string; content: string }) => {
export const useConfirm = (props: { title?: string | null; content?: string | null }) => {
const { t } = useTranslation();
const { title = t('Warning'), content } = props;
const [customContent, setCustomContent] = useState(content);
const { isOpen, onOpen, onClose } = useDisclosure();
const cancelRef = useRef(null);
const confirmCb = useRef<any>();
const cancelCb = useRef<any>();
return {
openConfirm: useCallback(
(confirm?: any, cancel?: any) => {
(confirm?: any, cancel?: any, customContent?: string) => {
confirmCb.current = confirm;
cancelCb.current = cancel;
customContent && setCustomContent(customContent);
return onOpen;
},
[onOpen]
......@@ -44,7 +48,7 @@ export const useConfirm = (props: { title?: string; content: string }) => {
{title}
</AlertDialogHeader>
<AlertDialogBody>{content}</AlertDialogBody>
<AlertDialogBody>{customContent}</AlertDialogBody>
<AlertDialogFooter>
<Button
......@@ -70,7 +74,7 @@ export const useConfirm = (props: { title?: string; content: string }) => {
</AlertDialogOverlay>
</AlertDialog>
),
[content, isOpen, onClose, title]
[customContent, isOpen, onClose, t, title]
)
};
};
client/src/hooks/useEditInfo.tsx
import React, { useCallback, useRef } from 'react';
import React, { useCallback, useRef, useState } from 'react';
import { ModalFooter, ModalBody, Input, useDisclosure, Button } from '@chakra-ui/react';
import MyModal from '@/components/MyModal';
......
client/src/pages/_app.tsx
......@@ -41,16 +41,14 @@ function App({ Component, pageProps }: AppProps) {
const { setLastRoute } = useGlobalStore();
const [scripts, setScripts] = useState<FeConfigsType['scripts']>([]);
const [googleClientVerKey, setGoogleVerKey] = useState<string>();
useEffect(() => {
// get init data
(async () => {
const {
feConfigs: { scripts, googleClientVerKey }
feConfigs: { scripts }
} = await clientInitData();
setScripts(scripts || []);
setGoogleVerKey(googleClientVerKey);
})();
// add window error track
window.onerror = function (msg, url) {
......@@ -94,20 +92,10 @@ function App({ Component, pageProps }: AppProps) {
/>
<link rel="icon" href="/favicon.ico" />
</Head>
<Script src="/js/qrcode.min.js" strategy="lazyOnload"></Script>
<Script src="/js/pdf.js" strategy="lazyOnload"></Script>
<Script src="/js/html2pdf.bundle.min.js" strategy="lazyOnload"></Script>
{scripts?.map((item, i) => (
<Script key={i} strategy="lazyOnload" {...item}></Script>
))}
{googleClientVerKey && (
<>
<Script
src={`https://www.recaptcha.net/recaptcha/api.js?render=${googleClientVerKey}`}
strategy="lazyOnload"
></Script>
</>
)}
<QueryClientProvider client={queryClient}>
<ChakraProvider theme={theme}>
<ColorModeScript initialColorMode={theme.config.initialColorMode} />
......
client/src/pages/account/components/UpdatePswModal.tsx
......@@ -25,7 +25,7 @@ const UpdatePswModal = ({ onClose }: { onClose: () => void }) => {
const { mutate: onSubmit, isLoading } = useRequest({
mutationFn: (data: FormType) => {
if (data.newPsw !== data.confirmPsw) {
return Promise.reject(t('commom.Password inconsistency'));
return Promise.reject(t('common.Password inconsistency'));
}
return updatePasswordByOld(data);
},
......
client/src/pages/account/index.tsx
......@@ -13,6 +13,7 @@ import UserInfo from './components/Info';
import { serviceSideProps } from '@/utils/i18n';
import { feConfigs } from '@/store/static';
import { useTranslation } from 'react-i18next';
import Script from 'next/script';
const Promotion = dynamic(() => import('./components/Promotion'));
const BillTable = dynamic(() => import('./components/BillTable'));
......@@ -97,51 +98,54 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
);
return (
<PageContainer>
<Flex flexDirection={['column', 'row']} h={'100%'} pt={[4, 0]}>
{isPc ? (
<Flex
flexDirection={'column'}
p={4}
h={'100%'}
flex={'0 0 200px'}
borderRight={theme.borders.base}
>
<SideTabs
flex={1}
mx={'auto'}
mt={2}
w={'100%'}
list={tabList.current}
activeId={currentTab}
onChange={setCurrentTab}
/>
</Flex>
) : (
<Box mb={3}>
<Tabs
m={'auto'}
size={isPc ? 'md' : 'sm'}
list={tabList.current.map((item) => ({
id: item.id,
label: item.label
}))}
activeId={currentTab}
onChange={setCurrentTab}
/>
</Box>
)}
<>
<Script src="/js/qrcode.min.js" strategy="lazyOnload"></Script>
<PageContainer>
<Flex flexDirection={['column', 'row']} h={'100%'} pt={[4, 0]}>
{isPc ? (
<Flex
flexDirection={'column'}
p={4}
h={'100%'}
flex={'0 0 200px'}
borderRight={theme.borders.base}
>
<SideTabs
flex={1}
mx={'auto'}
mt={2}
w={'100%'}
list={tabList.current}
activeId={currentTab}
onChange={setCurrentTab}
/>
</Flex>
) : (
<Box mb={3}>
<Tabs
m={'auto'}
size={isPc ? 'md' : 'sm'}
list={tabList.current.map((item) => ({
id: item.id,
label: item.label
}))}
activeId={currentTab}
onChange={setCurrentTab}
/>
</Box>
)}
<Box flex={'1 0 0'} h={'100%'} pb={[4, 0]}>
{currentTab === TabEnum.info && <UserInfo />}
{currentTab === TabEnum.promotion && <Promotion />}
{currentTab === TabEnum.bill && <BillTable />}
{currentTab === TabEnum.pay && <PayRecordTable />}
{currentTab === TabEnum.inform && <InformTable />}
</Box>
</Flex>
<ConfirmModal />
</PageContainer>
<Box flex={'1 0 0'} h={'100%'} pb={[4, 0]}>
{currentTab === TabEnum.info && <UserInfo />}
{currentTab === TabEnum.promotion && <Promotion />}
{currentTab === TabEnum.bill && <BillTable />}
{currentTab === TabEnum.pay && <PayRecordTable />}
{currentTab === TabEnum.inform && <InformTable />}
</Box>
</Flex>
<ConfirmModal />
</PageContainer>
</>
);
};
......
client/src/pages/api/admin/initv44.ts 0 → 100644
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase, KB } from '@/service/mongo';
import { KbTypeEnum, KbTypeMap } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authRoot: true });
await KB.updateMany(
{
type: { $exists: false }
},
{
$set: {
type: KbTypeEnum.dataset,
parentId: null
}
}
);
jsonRes(res, {});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}
client/src/pages/api/openapi/kb/pushData.ts
......@@ -88,7 +88,7 @@ export async function pushDataToKb({
]);
const modeMaxToken = {
[TrainingModeEnum.index]: vectorModel.maxToken,
[TrainingModeEnum.index]: vectorModel.maxToken * 1.5,
[TrainingModeEnum.qa]: global.qaModel.maxToken * 0.8
};
......@@ -146,7 +146,6 @@ export async function pushDataToKb({
}
} catch (error) {
console.log(error);
error;
}
return Promise.resolve(data);
})
......
client/src/pages/api/openapi/kb/updateData.ts
......@@ -50,7 +50,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
await PgClient.update(PgTrainingTableName, {
where: [['id', dataId], 'AND', ['user_id', userId]],
values: [
{ key: 'source', value: '手动修改' },
{ key: 'a', value: a.replace(/'/g, '"') },
...(q
? [
......
client/src/pages/api/openapi/plugin/vector.ts
......@@ -69,7 +69,7 @@ export async function getVector({
.then(async (res) => {
if (!res.data?.data?.[0]?.embedding) {
// @ts-ignore
return Promise.reject(res.data?.error?.message || 'Embedding API Error');
return Promise.reject(res.data?.err?.message || 'Embedding API Error');
}
return {
tokenLen: res.data.usage.total_tokens || 0,
......
client/src/pages/api/plugins/kb/allDataset.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { getVectorModel } from '@/service/utils/data';
import { KbListItemType } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const kbList = await KB.find({
userId,
type: 'dataset'
});
const data = kbList.map((item) => ({
...item.toJSON(),
vectorModel: getVectorModel(item.vectorModel)
}));
jsonRes<KbListItemType[]>(res, {
data
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
client/src/pages/api/plugins/kb/create.ts
......@@ -6,11 +6,7 @@ import type { CreateKbParams } from '@/api/request/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { name, tags, avatar, vectorModel } = req.body as CreateKbParams;
if (!name || !vectorModel) {
throw new Error('缺少参数');
}
const { name, tags, avatar, vectorModel, parentId, type } = req.body as CreateKbParams;
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
......@@ -22,7 +18,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
userId,
tags,
vectorModel,
avatar
avatar,
parentId: parentId || null,
type
});
jsonRes(res, { data: _id });
......
client/src/pages/api/plugins/kb/data/exportModelData.ts
......@@ -4,11 +4,13 @@ import { connectToDatabase, User } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { OtherFileId } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
let { kbId } = req.query as {
let { kbId, fileId } = req.query as {
kbId: string;
fileId: string;
};
if (!kbId) {
......@@ -20,7 +22,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const thirtyMinutesAgo = new Date(Date.now() - 30 * 60 * 1000);
const thirtyMinutesAgo = new Date(
Date.now() - (global.feConfigs?.exportLimitMinutes || 0) * 60 * 1000
);
// auth export times
const authTimes = await User.findOne(
......@@ -35,21 +39,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
);
if (!authTimes) {
throw new Error('上次导出未到半小时，每半小时仅可导出一次。');
const minutes = `${global.feConfigs?.exportLimitMinutes || 0} 分钟`;
throw new Error(`上次导出未到 ${minutes}，每 ${minutes}仅可导出一次。`);
}
// 统计数据
const count = await PgClient.count(PgTrainingTableName, {
where: [['kb_id', kbId], 'AND', ['user_id', userId]]
});
const where: any = [['kb_id', kbId], 'AND', ['user_id', userId]];
// 从 pg 中获取所有数据
const pgData = await PgClient.select<{ q: string; a: string; source: string }>(
PgTrainingTableName,
{
where: [['kb_id', kbId], 'AND', ['user_id', userId]],
where,
fields: ['q', 'a', 'source'],
order: [{ field: 'id', mode: 'DESC' }],
limit: count
limit: 1000000
}
);
......@@ -78,7 +80,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
export const config = {
api: {
bodyParser: {
sizeLimit: '100mb'
sizeLimit: '200mb'
}
}
};
client/src/pages/api/plugins/kb/data/getDataList.ts
......@@ -5,6 +5,7 @@ import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import type { KbDataItemType } from '@/types/plugin';
import { PgTrainingTableName } from '@/constants/plugin';
import { OtherFileId } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
......@@ -12,12 +13,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
kbId,
pageNum = 1,
pageSize = 10,
searchText = ''
searchText = '',
fileId = ''
} = req.body as {
kbId: string;
pageNum: number;
pageSize: number;
searchText: string;
fileId: string;
};
if (!kbId) {
throw new Error('缺少参数');
......@@ -33,6 +36,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
['user_id', userId],
'AND',
['kb_id', kbId],
...(fileId
? fileId === OtherFileId
? ["AND (file_id IS NULL OR file_id = '')"]
: ['AND', ['file_id', fileId]]
: []),
...(searchText
? [
'AND',
......
client/src/pages/api/plugins/kb/delete.ts
......@@ -3,12 +3,12 @@ import { jsonRes } from '@/service/response';
import { connectToDatabase, KB, App, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { Types } from 'mongoose';
import { PgTrainingTableName } from '@/constants/plugin';
import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id } = req.query as {
id: string;
};
......@@ -20,26 +20,30 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const deletedIds = [id, ...(await findAllChildrenIds(id))];
// delete training data
await TrainingData.deleteMany({
userId,
kbId: id
kbId: { $in: deletedIds }
});
// delete all pg data
await PgClient.delete(PgTrainingTableName, {
where: [['user_id', userId], 'AND', ['kb_id', id]]
where: [
['user_id', userId],
'AND',
`kb_id IN (${deletedIds.map((id) => `'${id}'`).join(',')})`
]
});
// delete related files
const gridFs = new GridFSStorage('dataset', userId);
await gridFs.deleteFilesByKbId(id);
await Promise.all(deletedIds.map((id) => gridFs.deleteFilesByKbId(id)));
// delete kb data
await KB.findOneAndDelete({
_id: id,
await KB.deleteMany({
_id: { $in: deletedIds },
userId
});
......@@ -51,3 +55,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
}
}
async function findAllChildrenIds(id: string) {
// find children
const children = await KB.find({ parentId: id });
let allChildrenIds = children.map((child) => String(child._id));
for (const child of children) {
const grandChildrenIds = await findAllChildrenIds(child._id);
allChildrenIds = allChildrenIds.concat(grandChildrenIds);
}
return allChildrenIds;
}
client/src/pages/api/plugins/kb/file/delFileByFileId.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { Types } from 'mongoose';
import { OtherFileId } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { fileId, kbId } = req.query as { fileId: string; kbId: string };
if (!fileId || !kbId) {
throw new Error('fileId and kbId is required');
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
if (fileId === OtherFileId) {
await PgClient.delete(PgTrainingTableName, {
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
"AND (file_id IS NULL OR file_id = '')"
]
});
} else {
const gridFs = new GridFSStorage('dataset', userId);
const bucket = gridFs.GridFSBucket();
await gridFs.findAndAuthFile(fileId);
// delete all pg data
await PgClient.delete(PgTrainingTableName, {
where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['file_id', fileId]]
});
// delete file
await bucket.delete(new Types.ObjectId(fileId));
}
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { Types } from 'mongoose';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { kbId } = req.query as { kbId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const gridFs = new GridFSStorage('dataset', userId);
const bucket = gridFs.GridFSBucket();
const files = await bucket
// 1 hours expired
.find({
uploadDate: { $lte: new Date(Date.now() - 60 * 1000) },
['metadata.kbId']: kbId,
['metadata.userId']: userId
})
.sort({ _id: -1 })
.toArray();
const data = await Promise.all(
files.map(async (file) => {
return {
id: file._id,
chunkLength: await PgClient.count(PgTrainingTableName, {
fields: ['id'],
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
'AND',
['file_id', String(file._id)]
]
})
};
})
);
await Promise.all(
data
.filter((item) => item.chunkLength === 0)
.map((file) => bucket.delete(new Types.ObjectId(file.id)))
);
jsonRes(res);
} catch (err) {
jsonRes(res);
}
}
client/src/pages/api/plugins/kb/file/getFileInfo.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { OtherFileId } from '@/constants/kb';
import type { FileInfo } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { fileId } = req.query as { kbId: string; fileId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
if (fileId === OtherFileId) {
return jsonRes<FileInfo>(res, {
data: {
id: OtherFileId,
size: 0,
filename: 'kb.Other Data',
uploadDate: new Date(),
encoding: '',
contentType: ''
}
});
}
const gridFs = new GridFSStorage('dataset', userId);
const file = await gridFs.findAndAuthFile(fileId);
jsonRes<FileInfo>(res, {
data: file
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
client/src/pages/api/plugins/kb/file/list.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { KbFileItemType } from '@/types/plugin';
import { FileStatusEnum, OtherFileId } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
let { kbId, searchText } = req.query as { kbId: string; searchText: string };
searchText = searchText.replace(/'/g, '');
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const gridFs = new GridFSStorage('dataset', userId);
const bucket = gridFs.GridFSBucket();
const files = await bucket
.find({ ['metadata.kbId']: kbId, ...(searchText && { filename: { $regex: searchText } }) })
.sort({ _id: -1 })
.toArray();
async function GetOtherData() {
return {
id: OtherFileId,
size: 0,
filename: 'kb.Other Data',
uploadTime: new Date(),
status: (await TrainingData.findOne({ userId, kbId, file_id: '' }))
? FileStatusEnum.embedding
: FileStatusEnum.ready,
chunkLength: await PgClient.count(PgTrainingTableName, {
fields: ['id'],
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
"AND (file_id IS NULL OR file_id = '')"
]
})
};
}
const data = await Promise.all([
GetOtherData(),
...files.map(async (file) => {
return {
id: String(file._id),
size: file.length,
filename: file.filename,
uploadTime: file.uploadDate,
status: (await TrainingData.findOne({ userId, kbId, file_id: file._id }))
? FileStatusEnum.embedding
: FileStatusEnum.ready,
chunkLength: await PgClient.count(PgTrainingTableName, {
fields: ['id'],
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
'AND',
['file_id', String(file._id)]
]
})
};
})
]);
jsonRes<KbFileItemType[]>(res, {
data: data.flat().filter((item) => item.chunkLength > 0)
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
client/src/pages/api/plugins/kb/list.ts
......@@ -2,29 +2,25 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { KbListItemType } from '@/types/plugin';
import { getVectorModel } from '@/service/utils/data';
import { KbListItemType } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { parentId } = req.query as { parentId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const kbList = await KB.find(
{
userId
},
'_id avatar name tags vectorModel'
).sort({ updateTime: -1 });
const kbList = await KB.find({
userId,
parentId: parentId || null
}).sort({ updateTime: -1 });
const data = await Promise.all(
kbList.map(async (item) => ({
_id: item._id,
avatar: item.avatar,
name: item.name,
tags: item.tags,
...item.toJSON(),
vectorModel: getVectorModel(item.vectorModel)
}))
);
......
client/src/pages/api/plugins/kb/paths.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { KbPathItemType } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { parentId } = req.query as { parentId: string };
jsonRes<KbPathItemType[]>(res, {
data: await getParents(parentId)
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
async function getParents(parentId?: string): Promise<KbPathItemType[]> {
if (!parentId) {
return [];
}
const parent = await KB.findById(parentId, 'name parentId');
if (!parent) return [];
const paths = await getParents(parent.parentId);
paths.push({ parentId, parentName: parent.name });
return paths;
}
client/src/pages/api/plugins/kb/update.ts
......@@ -6,7 +6,7 @@ import type { KbUpdateParams } from '@/api/request/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { id, name, tags, avatar } = req.body as KbUpdateParams;
const { id, name, avatar, tags = '' } = req.body as KbUpdateParams;
if (!id || !name) {
throw new Error('缺少参数');
......@@ -23,8 +23,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
userId
},
{
avatar,
name,
...(name && { name }),
...(avatar && { avatar }),
tags: tags.split(' ').filter((item) => item)
}
);
......
client/src/pages/api/system/getInitData.ts
import type { FeConfigsType } from '@/types';
import type { FeConfigsType, SystemEnvType } from '@/types';
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { readFileSync } from 'fs';
......@@ -29,12 +29,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
}
const defaultSystemEnv = {
const defaultSystemEnv: SystemEnvType = {
vectorMaxProcess: 15,
qaMaxProcess: 15,
pgIvfflatProbe: 20
};
const defaultFeConfigs = {
const defaultFeConfigs: FeConfigsType = {
show_emptyChat: true,
show_register: false,
show_appStore: false,
......@@ -44,7 +44,7 @@ const defaultFeConfigs = {
show_doc: true,
systemTitle: 'FastGPT',
authorText: 'Made by FastGPT Team.',
gitLoginKey: '',
exportLimitMinutes: 0,
scripts: []
};
const defaultChatModels = [
......@@ -99,8 +99,10 @@ export async function getInitConfig() {
const res = JSON.parse(readFileSync(filename, 'utf-8'));
console.log(res);
global.systemEnv = res.SystemParams || defaultSystemEnv;
global.feConfigs = res.FeConfig || defaultFeConfigs;
global.systemEnv = res.SystemParams
? { ...defaultSystemEnv, ...res.SystemParams }
: defaultSystemEnv;
global.feConfigs = res.FeConfig ? { ...defaultFeConfigs, ...res.FeConfig } : defaultFeConfigs;
global.chatModels = res.ChatModels || defaultChatModels;
global.qaModel = res.QAModel || defaultQAModel;
global.vectorModels = res.VectorModels || defaultVectorModels;
......
client/src/pages/app/detail/components/AdEdit/components/Nodes/NodeKbSearch.tsx
import React, { useMemo, useState } from 'react';
import React, { useMemo } from 'react';
import { NodeProps } from 'reactflow';
import { FlowModuleItemType } from '@/types/flow';
import { Flex, Box, Button, useTheme, useDisclosure, Grid } from '@chakra-ui/react';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
import { useQuery } from '@tanstack/react-query';
import NodeCard from '../modules/NodeCard';
import Divider from '../modules/Divider';
......@@ -21,7 +21,7 @@ const KBSelect = ({
onChange: (e: SelectedKbType) => void;
}) => {
const theme = useTheme();
const { myKbList, loadKbList } = useUserStore();
const { datasets, loadAllDatasets } = useDatasetStore();
const {
isOpen: isOpenKbSelect,
onOpen: onOpenKbSelect,
......@@ -29,11 +29,11 @@ const KBSelect = ({
} = useDisclosure();
const showKbList = useMemo(
() => myKbList.filter((item) => activeKbs.find((kb) => kb.kbId === item._id)),
[myKbList, activeKbs]
() => datasets.filter((item) => activeKbs.find((kb) => kb.kbId === item._id)),
[datasets, activeKbs]
);
useQuery(['initkb'], loadKbList);
useQuery(['loadAllDatasets'], loadAllDatasets);
return (
<>
......@@ -58,12 +58,7 @@ const KBSelect = ({
))}
</Grid>
{isOpenKbSelect && (
<KBSelectModal
kbList={myKbList}
activeKbs={activeKbs}
onChange={onChange}
onClose={onCloseKbSelect}
/>
<KBSelectModal activeKbs={activeKbs} onChange={onChange} onClose={onCloseKbSelect} />
)}
</>
);
......
client/src/pages/app/detail/components/BasicEdit/index.tsx
......@@ -56,18 +56,21 @@ import MyIcon from '@/components/Icon';
import ChatBox, { type ComponentRef, type StartChatFnProps } from '@/components/ChatBox';
import { addVariable } from '../VariableEditModal';
import { KBSelectModal, KbParamsModal } from '../KBSelectModal';
import { KbParamsModal } from '../KBSelectModal';
import { AppTypeEnum } from '@/constants/app';
import { useDatasetStore } from '@/store/dataset';
const VariableEditModal = dynamic(() => import('../VariableEditModal'));
const InfoModal = dynamic(() => import('../InfoModal'));
const KBSelectModal = dynamic(() => import('../KBSelectModal'));
const Settings = ({ appId }: { appId: string }) => {
const theme = useTheme();
const router = useRouter();
const { t } = useTranslation();
const { toast } = useToast();
const { appDetail, updateAppDetail, loadKbList, myKbList } = useUserStore();
const { appDetail, updateAppDetail } = useUserStore();
const { loadAllDatasets, datasets } = useDatasetStore();
const { isPc } = useGlobalStore();
const [editVariable, setEditVariable] = useState<VariableItemType>();
......@@ -122,8 +125,8 @@ const Settings = ({ appId }: { appId: string }) => {
);
}, [getValues, refresh]);
const selectedKbList = useMemo(
() => myKbList.filter((item) => kbList.find((kb) => kb.kbId === item._id)),
[myKbList, kbList]
() => datasets.filter((item) => kbList.find((kb) => kb.kbId === item._id)),
[datasets, kbList]
);
/* 点击删除 */
......@@ -167,7 +170,7 @@ const Settings = ({ appId }: { appId: string }) => {
appModule2Form();
}, [appModule2Form]);
useQuery(['initkb', appId], () => loadKbList());
useQuery(['loadAllDatasets'], loadAllDatasets);
const BoxStyles: BoxProps = {
bg: 'myWhite.200',
......@@ -304,6 +307,23 @@ const Settings = ({ appId }: { appId: string }) => {
</Button>
</Flex>
{/* welcome */}
<Box mt={5} {...BoxStyles}>
<Flex alignItems={'center'}>
<Avatar src={'/imgs/module/userGuide.png'} w={'18px'} />
<Box mx={2}>对话开场白</Box>
<MyTooltip label={welcomeTextTip} forceShow>
<QuestionOutlineIcon />
</MyTooltip>
</Flex>
<Textarea
mt={2}
rows={5}
placeholder={welcomeTextTip}
borderColor={'myGray.100'}
{...register('guide.welcome.text')}
/>
</Box>
{/* variable */}
<Box mt={2} {...BoxStyles}>
<Flex alignItems={'center'}>
......@@ -498,24 +518,6 @@ const Settings = ({ appId }: { appId: string }) => {
</Grid>
</Box>
{/* welcome */}
<Box mt={5} {...BoxStyles}>
<Flex alignItems={'center'}>
<Avatar src={'/imgs/module/userGuide.png'} w={'18px'} />
<Box mx={2}>对话开场白</Box>
<MyTooltip label={welcomeTextTip} forceShow>
<QuestionOutlineIcon />
</MyTooltip>
</Flex>
<Textarea
mt={2}
rows={5}
placeholder={welcomeTextTip}
borderColor={'myGray.100'}
{...register('guide.welcome.text')}
/>
</Box>
<ConfirmSaveModal />
<ConfirmDelModal />
{settingAppInfo && (
......@@ -548,7 +550,6 @@ const Settings = ({ appId }: { appId: string }) => {
)}
{isOpenKbSelect && (
<KBSelectModal
kbList={myKbList}
activeKbs={selectedKbList.map((item) => ({
kbId: item._id,
vectorModel: item.vectorModel
......
client/src/pages/kb/detail/components/DataCard.tsx
import React, { useCallback, useState, useRef } from 'react';
import { Box, Card, IconButton, Flex, Button, Input, Grid } from '@chakra-ui/react';
import React, { useCallback, useState, useRef, useMemo } from 'react';
import { Box, Card, IconButton, Flex, Button, Grid, Image } from '@chakra-ui/react';
import type { KbDataItemType } from '@/types/plugin';
import { usePagination } from '@/hooks/usePagination';
import {
getKbDataList,
getExportDataList,
delOneKbDataByDataId,
getTrainingData
getTrainingData,
getFileInfoById
} from '@/api/plugins/kb';
import { DeleteIcon, RepeatIcon } from '@chakra-ui/icons';
import { fileDownload } from '@/utils/file';
......@@ -18,12 +19,19 @@ import { debounce } from 'lodash';
import { getErrText } from '@/utils/tools';
import { useConfirm } from '@/hooks/useConfirm';
import { useTranslation } from 'react-i18next';
import { useRouter } from 'next/router';
import MyIcon from '@/components/Icon';
import MyTooltip from '@/components/MyTooltip';
import MyInput from '@/components/MyInput';
import { fileImgs } from '@/constants/common';
import { useRequest } from '@/hooks/useRequest';
import { feConfigs } from '@/store/static';
const DataCard = ({ kbId }: { kbId: string }) => {
const BoxRef = useRef<HTMLDivElement>(null);
const lastSearch = useRef('');
const router = useRouter();
const { fileId = '' } = router.query as { fileId: string };
const { t } = useTranslation();
const [searchText, setSearchText] = useState('');
const { toast } = useToast();
......@@ -45,7 +53,8 @@ const DataCard = ({ kbId }: { kbId: string }) => {
pageSize: 24,
params: {
kbId,
searchText
searchText,
fileId
},
onChange() {
if (BoxRef.current) {
......@@ -72,33 +81,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
[getData, pageNum, refetchTrainingData]
);
// get al data and export csv
const { mutate: onclickExport, isLoading: isLoadingExport = false } = useMutation({
mutationFn: () => getExportDataList(kbId),
onSuccess(res) {
const text = Papa.unparse({
fields: ['question', 'answer', 'source'],
data: res
});
fileDownload({
text,
type: 'text/csv',
filename: 'data.csv'
});
toast({
title: '导出成功，下次导出需要半小时后',
status: 'success'
});
},
onError(err: any) {
toast({
title: getErrText(err, '导出异常'),
status: 'error'
});
console.log(err);
}
});
// get first page data
const getFirstData = useCallback(
debounce(() => {
getData(1);
......@@ -113,57 +96,100 @@ const DataCard = ({ kbId }: { kbId: string }) => {
enabled: qaListLen > 0 || vectorListLen > 0
});
// get file info
const { data: fileInfo } = useQuery(['getFileInfo', fileId], () => getFileInfoById(fileId));
const fileIcon = useMemo(
() =>
fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(fileInfo?.filename || ''))?.src,
[fileInfo?.filename]
);
// get al data and export csv
const { mutate: onclickExport, isLoading: isLoadingExport = false } = useRequest({
mutationFn: () => getExportDataList({ kbId, fileId }),
onSuccess(res) {
const text = Papa.unparse({
fields: ['question', 'answer', 'source'],
data: res
});
const filenameSplit = fileInfo?.filename?.split('.') || [];
const filename = filenameSplit?.length <= 1 ? 'data' : filenameSplit.slice(0, -1).join('.');
fileDownload({
text,
type: 'text/csv',
filename
});
},
successToast: `导出成功，下次导出需要 ${feConfigs.exportLimitMinutes} 分钟后`,
errorToast: '导出异常'
});
return (
<Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}>
<Flex justifyContent={'space-between'}>
<Box fontWeight={'bold'} fontSize={'lg'} mr={2}>
知识库数据: {total}组
</Box>
<Flex alignItems={'center'}>
<Flex
className="textEllipsis"
flex={'1 0 0'}
mr={[3, 5]}
fontSize={['sm', 'md']}
alignItems={'center'}
>
<Image src={fileIcon} w={'16px'} mr={2} alt={''} />
{t(fileInfo?.filename || 'Filename')}
</Flex>
<Button
mr={2}
size={['sm', 'md']}
variant={'base'}
borderColor={'myBlue.600'}
color={'myBlue.600'}
isLoading={isLoadingExport || isLoading}
title={`${feConfigs} 分钟能导出 1 次`}
onClick={onclickExport}
>
{t('dataset.Export')}
</Button>
<Box>
<MyTooltip label={'刷新'}>
<IconButton
icon={<RepeatIcon />}
size={['sm', 'md']}
aria-label={'refresh'}
variant={'base'}
isLoading={isLoading}
mr={[2, 4]}
size={'sm'}
onClick={() => {
getData(pageNum);
getTrainingData({ kbId, init: true });
}}
/>
</MyTooltip>
<Button
mr={2}
size={'sm'}
variant={'base'}
borderColor={'myBlue.600'}
color={'myBlue.600'}
isLoading={isLoadingExport || isLoading}
title={'半小时仅能导出1次'}
onClick={() => onclickExport()}
>
导出数据
</Button>
</Box>
</Flex>
<Flex my={4}>
{qaListLen > 0 || vectorListLen > 0 ? (
<Box fontSize={'xs'}>
{qaListLen > 0 ? `${qaListLen}条数据正在拆分，` : ''}
{vectorListLen > 0 ? `${vectorListLen}条数据正在生成索引，` : ''}
请耐心等待...
<Flex my={3} alignItems={'center'}>
<Box>
<Box as={'span'} fontSize={['md', 'lg']}>
{total}组
</Box>
<Box as={'span'}>
{(qaListLen > 0 || vectorListLen > 0) && (
<>
({qaListLen > 0 ? `${qaListLen}条数据正在拆分，` : ''}
{vectorListLen > 0 ? `${vectorListLen}条数据正在生成索引，` : ''}
请耐心等待... )
</>
)}
</Box>
) : (
<Box fontSize={'xs'}>所有数据已就绪~</Box>
)}
</Box>
<Box flex={1} mr={1} />
<Input
maxW={['60%', '300px']}
size={'sm'}
value={searchText}
<MyInput
leftIcon={
<MyIcon name="searchLight" position={'absolute'} w={'14px'} color={'myGray.500'} />
}
w={['200px', '300px']}
placeholder="根据匹配知识，预期答案和来源进行搜索"
value={searchText}
onChange={(e) => {
setSearchText(e.target.value);
getFirstData();
......@@ -217,7 +243,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
</Box>
<Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}>
<Box className={'textEllipsis'} flex={1} color={'myGray.500'}>
{item.source?.trim()}
ID:{item.id}
</Box>
<IconButton
className="delete"
......
client/src/pages/kb/detail/components/FileCard.tsx 0 → 100644
import React, { useCallback, useState, useRef, useMemo } from 'react';
import {
Box,
Flex,
TableContainer,
Table,
Thead,
Tr,
Th,
Td,
Tbody,
Image
} from '@chakra-ui/react';
import { getKbFiles, deleteKbFileById } from '@/api/plugins/kb';
import { useQuery } from '@tanstack/react-query';
import { useToast } from '@/hooks/useToast';
import { debounce } from 'lodash';
import { formatFileSize } from '@/utils/tools';
import { useConfirm } from '@/hooks/useConfirm';
import { useTranslation } from 'react-i18next';
import MyIcon from '@/components/Icon';
import MyInput from '@/components/MyInput';
import dayjs from 'dayjs';
import { fileImgs } from '@/constants/common';
import { useRequest } from '@/hooks/useRequest';
import { useLoading } from '@/hooks/useLoading';
import { FileStatusEnum } from '@/constants/kb';
import { useRouter } from 'next/router';
const FileCard = ({ kbId }: { kbId: string }) => {
const BoxRef = useRef<HTMLDivElement>(null);
const lastSearch = useRef('');
const router = useRouter();
const { t } = useTranslation();
const [searchText, setSearchText] = useState('');
const { Loading } = useLoading();
const { openConfirm, ConfirmModal } = useConfirm({
content: t('kb.Confirm to delete the file')
});
const {
data: files = [],
refetch,
isInitialLoading
} = useQuery(['getFiles', kbId], () => getKbFiles({ kbId, searchText }), {
refetchInterval: 6000,
refetchOnWindowFocus: true
});
const debounceRefetch = useCallback(
debounce(() => {
refetch();
lastSearch.current = searchText;
}, 300),
[]
);
const formatFiles = useMemo(
() =>
files.map((file) => ({
...file,
icon: fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(file.filename))?.src
})),
[files]
);
const totalDataLength = useMemo(
() => files.reduce((sum, item) => sum + item.chunkLength, 0),
[files]
);
const { mutate: onDeleteFile, isLoading } = useRequest({
mutationFn: (fileId: string) =>
deleteKbFileById({
fileId,
kbId
}),
onSuccess() {
refetch();
},
successToast: t('common.Delete Success'),
errorToast: t('common.Delete Failed')
});
const statusMap = {
[FileStatusEnum.embedding]: {
color: 'myGray.500',
text: t('file.Embedding')
},
[FileStatusEnum.ready]: {
color: 'green.500',
text: t('file.Ready')
}
};
return (
<Box ref={BoxRef} position={'relative'} py={[1, 5]} h={'100%'} overflow={'overlay'}>
<Flex justifyContent={'space-between'} px={5}>
<Box fontWeight={'bold'} fontSize={'lg'} mr={2}>
{t('kb.Files', { total: files.length })}
</Box>
<Flex alignItems={'center'}>
<MyInput
leftIcon={
<MyIcon name="searchLight" position={'absolute'} w={'14px'} color={'myGray.500'} />
}
w={['100%', '200px']}
placeholder={t('common.Search') || ''}
value={searchText}
onChange={(e) => {
setSearchText(e.target.value);
debounceRefetch();
}}
onBlur={() => {
if (searchText === lastSearch.current) return;
refetch();
}}
onKeyDown={(e) => {
if (searchText === lastSearch.current) return;
if (e.key === 'Enter') {
refetch();
}
}}
/>
</Flex>
</Flex>
<TableContainer mt={[0, 3]}>
<Table variant={'simple'} fontSize={'sm'}>
<Thead>
<Tr>
<Th>{t('kb.Filename')}</Th>
<Th>
{t('kb.Chunk Length')}({totalDataLength})
</Th>
<Th>{t('kb.Upload Time')}</Th>
<Th>{t('kb.File Size')}</Th>
<Th>{t('common.Status')}</Th>
<Th />
</Tr>
</Thead>
<Tbody>
{formatFiles.map((file) => (
<Tr
key={file.id}
_hover={{ bg: 'myWhite.600' }}
cursor={'pointer'}
title={'点击查看数据详情'}
onClick={() =>
router.push({
query: {
kbId,
fileId: file.id,
currentTab: 'dataCard'
}
})
}
>
<Td>
<Flex alignItems={'center'}>
<Image src={file.icon} w={'16px'} mr={2} alt={''} />
<Box maxW={['300px', '400px']} className="textEllipsis">
{t(file.filename)}
</Box>
</Flex>
</Td>
<Td fontSize={'md'} fontWeight={'bold'}>
{file.chunkLength}
</Td>
<Td>{dayjs(file.uploadTime).format('YYYY/MM/DD HH:mm')}</Td>
<Td>{formatFileSize(file.size)}</Td>
<Td
display={'flex'}
alignItems={'center'}
_before={{
content: '""',
w: '10px',
h: '10px',
mr: 2,
borderRadius: 'lg',
bg: statusMap[file.status].color
}}
>
{statusMap[file.status].text}
</Td>
<Td onClick={(e) => e.stopPropagation()}>
<MyIcon
name={'delete'}
w={'14px'}
_hover={{ color: 'red.600' }}
onClick={() =>
openConfirm(() => {
onDeleteFile(file.id);
})()
}
/>
</Td>
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
<ConfirmModal />
<Loading loading={isInitialLoading || isLoading} />
</Box>
);
};
export default React.memo(FileCard);
client/src/pages/kb/detail/components/Import/Chunk.tsx
......@@ -26,12 +26,12 @@ import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { TrainingModeEnum } from '@/constants/plugin';
import FileSelect, { type FileItemType } from './FileSelect';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
const fileExtension = '.txt, .doc, .docx, .pdf, .md';
const ChunkImport = ({ kbId }: { kbId: string }) => {
const { kbDetail } = useUserStore();
const { kbDetail } = useDatasetStore();
const vectorModel = kbDetail.vectorModel;
const unitPrice = vectorModel?.price || 0.2;
......@@ -86,7 +86,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
router.replace({
query: {
kbId,
currentTab: 'data'
currentTab: 'dataset'
}
});
},
......@@ -106,12 +106,15 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
text: file.text,
maxLen: chunkLen
});
return {
...file,
tokens: splitRes.tokens,
chunks: file.chunks.map((chunk, i) => ({
...chunk,
q: splitRes.chunks[i]
chunks: splitRes.chunks.map((chunk) => ({
a: '',
source: file.filename,
file_id: file.id,
q: chunk
}))
};
})
......
client/src/pages/kb/detail/components/Import/CreateFileModal.tsx
......@@ -20,7 +20,7 @@ const CreateFileModal = ({
});
return (
<MyModal title={t('file.Create File')} isOpen onClose={onClose} w={'600px'} top={'15vh'}>
<MyModal title={t('file.Create File')} isOpen onClose={() => {}} w={'600px'} top={'15vh'}>
<ModalBody>
<Box mb={1} fontSize={'sm'}>
文件名
......
client/src/pages/kb/detail/components/Import/Csv.tsx
......@@ -10,12 +10,12 @@ import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
import { TrainingModeEnum } from '@/constants/plugin';
import FileSelect, { type FileItemType } from './FileSelect';
import { useRouter } from 'next/router';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
const fileExtension = '.csv';
const CsvImport = ({ kbId }: { kbId: string }) => {
const { kbDetail } = useUserStore();
const { kbDetail } = useDatasetStore();
const maxToken = kbDetail.vectorModel?.maxToken || 2000;
const theme = useTheme();
......@@ -42,7 +42,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
.flat()
.filter((item) => item?.q);
const filterChunks = chunks.filter((item) => item.q.length < maxToken);
const filterChunks = chunks.filter((item) => item.q.length < maxToken * 1.5);
if (filterChunks.length !== chunks.length) {
toast({
......@@ -73,7 +73,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
router.replace({
query: {
kbId,
currentTab: 'data'
currentTab: 'dataset'
}
});
},
......
client/src/pages/kb/detail/components/Import/FileSelect.tsx
......@@ -19,13 +19,13 @@ import dynamic from 'next/dynamic';
import MyTooltip from '@/components/MyTooltip';
import { FetchResultItem, DatasetItemType } from '@/types/plugin';
import { getErrText } from '@/utils/tools';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
const CreateFileModal = dynamic(() => import('./CreateFileModal'));
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
const csvTemplate = `question,answer,source\n"什么是 laf","laf 是一个云函数开发平台……","laf git doc"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……","sealos git doc"`;
const csvTemplate = `question,answer\n"什么是 laf","laf 是一个云函数开发平台……"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`;
export type FileItemType = {
id: string;
......@@ -55,7 +55,7 @@ const FileSelect = ({
showCreateFile = true,
...props
}: Props) => {
const { kbDetail } = useUserStore();
const { kbDetail } = useDatasetStore();
const { Loading: FileSelectLoading } = useLoading();
const { t } = useTranslation();
......@@ -129,7 +129,7 @@ const FileSelect = ({
maxLen: chunkLen
});
const fileItem: FileItemType = {
id: nanoid(),
id: filesId[0],
filename: file.name,
icon,
text,
......
client/src/pages/kb/detail/components/Import/Manual.tsx
......@@ -6,14 +6,14 @@ import { useRequest } from '@/hooks/useRequest';
import { getErrText } from '@/utils/tools';
import { postKbDataFromList } from '@/api/plugins/kb';
import { TrainingModeEnum } from '@/constants/plugin';
import { useUserStore } from '@/store/user';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useDatasetStore } from '@/store/dataset';
type ManualFormType = { q: string; a: string };
const ManualImport = ({ kbId }: { kbId: string }) => {
const { kbDetail } = useUserStore();
const { kbDetail } = useDatasetStore();
const maxToken = kbDetail.vectorModel?.maxToken || 2000;
const { register, handleSubmit, reset } = useForm({
......
client/src/pages/kb/detail/components/Import/QA.tsx
......@@ -74,7 +74,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
router.replace({
query: {
kbId,
currentTab: 'data'
currentTab: 'dataset'
}
});
},
......@@ -97,9 +97,11 @@ const QAImport = ({ kbId }: { kbId: string }) => {
return {
...file,
tokens: splitRes.tokens,
chunks: file.chunks.map((chunk, i) => ({
...chunk,
q: splitRes.chunks[i]
chunks: splitRes.chunks.map((chunk) => ({
a: '',
source: file.filename,
file_id: file.id,
q: chunk
}))
};
})
......
client/src/pages/kb/detail/components/Info.tsx
......@@ -12,7 +12,7 @@ import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons';
import { delKbById, putKbById } from '@/api/plugins/kb';
import { useSelectFile } from '@/hooks/useSelectFile';
import { useToast } from '@/hooks/useToast';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
import { useConfirm } from '@/hooks/useConfirm';
import { UseFormReturn } from 'react-hook-form';
import { compressImg } from '@/utils/file';
......@@ -47,7 +47,7 @@ const Info = (
multiple: false
});
const { kbDetail, getKbDetail, loadKbList, myKbList } = useUserStore();
const { kbDetail, getKbDetail, loadKbList } = useDatasetStore();
/* 点击删除 */
const onclickDelKb = useCallback(async () => {
......
client/src/pages/kb/detail/components/InputDataModal.tsx
......@@ -9,10 +9,10 @@ import MyIcon from '@/components/Icon';
import MyModal from '@/components/MyModal';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useUserStore } from '@/store/user';
import { useQuery } from '@tanstack/react-query';
import { DatasetItemType } from '@/types/plugin';
import { useTranslation } from 'react-i18next';
import { useDatasetStore } from '@/store/dataset';
export type FormData = { dataId?: string } & DatasetItemType;
......@@ -36,7 +36,7 @@ const InputDataModal = ({
const [loading, setLoading] = useState(false);
const { toast } = useToast();
const { kbDetail, getKbDetail } = useUserStore();
const { kbDetail, getKbDetail } = useDatasetStore();
const { getValues, register, handleSubmit, reset } = useForm<FormData>({
defaultValues
......@@ -261,13 +261,11 @@ export function RawFileText({ fileId, filename = '', ...props }: RawFileTextProp
<Box
color={'myGray.600'}
display={'inline-block'}
whiteSpace={'nowrap'}
{...(!!fileId
? {
cursor: 'pointer',
textDecoration: ['underline', 'none'],
_hover: {
textDecoration: 'underline'
},
textDecoration: 'underline',
onClick: async () => {
try {
const url = await getFileViewUrl(fileId);
......
client/src/pages/kb/detail/components/Test.tsx
import React, { useEffect, useMemo, useState } from 'react';
import { Box, Textarea, Button, Flex, useTheme, Grid, Progress } from '@chakra-ui/react';
import { useKbStore } from '@/store/kb';
import { useDatasetStore } from '@/store/dataset';
import type { KbTestItemType } from '@/types/plugin';
import { searchText, getKbDataItemById } from '@/api/plugins/kb';
import MyIcon from '@/components/Icon';
......@@ -13,16 +13,14 @@ import { useToast } from '@/hooks/useToast';
import { customAlphabet } from 'nanoid';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useUserStore } from '@/store/user';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
const Test = ({ kbId }: { kbId: string }) => {
const { kbDetail } = useUserStore();
const theme = useTheme();
const { toast } = useToast();
const { setLoading } = useGlobalStore();
const { kbTestList, pushKbTestItem, delKbTestItemById, updateKbItemById } = useKbStore();
const { kbDetail, kbTestList, pushKbTestItem, delKbTestItemById, updateKbItemById } =
useDatasetStore();
const [inputText, setInputText] = useState('');
const [kbTestItem, setKbTestItem] = useState<KbTestItemType>();
const [editData, setEditData] = useState<FormData>();
......
client/src/pages/kb/detail/index.tsx
import React, { useCallback, useRef } from 'react';
import React, { useCallback, useEffect, useRef } from 'react';
import { useRouter } from 'next/router';
import { Box, Flex, IconButton, useTheme } from '@chakra-ui/react';
import { useToast } from '@/hooks/useToast';
import { useForm } from 'react-hook-form';
import { useQuery } from '@tanstack/react-query';
import { useUserStore } from '@/store/user';
import { KbItemType } from '@/types/plugin';
import { getErrText } from '@/utils/tools';
import { useGlobalStore } from '@/store/global';
import { type ComponentRef } from './components/Info';
import Tabs from '@/components/Tabs';
import dynamic from 'next/dynamic';
import DataCard from './components/DataCard';
import MyIcon from '@/components/Icon';
import SideTabs from '@/components/SideTabs';
import PageContainer from '@/components/PageContainer';
......@@ -19,11 +17,17 @@ import Avatar from '@/components/Avatar';
import Info from './components/Info';
import { serviceSideProps } from '@/utils/i18n';
import { useTranslation } from 'react-i18next';
import { getTrainingQueueLen } from '@/api/plugins/kb';
import { delEmptyFiles, getTrainingQueueLen } from '@/api/plugins/kb';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { feConfigs } from '@/store/static';
import Script from 'next/script';
import FileCard from './components/FileCard';
import { useDatasetStore } from '@/store/dataset';
const DataCard = dynamic(() => import('./components/DataCard'), {
ssr: false
});
const ImportData = dynamic(() => import('./components/Import'), {
ssr: false
});
......@@ -32,7 +36,8 @@ const Test = dynamic(() => import('./components/Test'), {
});
enum TabEnum {
data = 'data',
dataCard = 'dataCard',
dataset = 'dataset',
import = 'import',
test = 'test',
info = 'info'
......@@ -45,10 +50,10 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` }
const { toast } = useToast();
const router = useRouter();
const { isPc } = useGlobalStore();
const { kbDetail, getKbDetail } = useUserStore();
const { kbDetail, getKbDetail } = useDatasetStore();
const tabList = useRef([
{ label: '数据集', id: TabEnum.data, icon: 'overviewLight' },
{ label: '数据集', id: TabEnum.dataset, icon: 'overviewLight' },
{ label: '导入数据', id: TabEnum.import, icon: 'importLight' },
{ label: '搜索测试', id: TabEnum.test, icon: 'kbTest' },
{ label: '配置', id: TabEnum.info, icon: 'settingLight' }
......@@ -85,105 +90,117 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` }
});
const { data: trainingQueueLen = 0 } = useQuery(['getTrainingQueueLen'], getTrainingQueueLen, {
refetchInterval: 5000
refetchInterval: 10000
});
useEffect(() => {
return () => {
try {
delEmptyFiles(kbId);
} catch (error) {}
};
}, [kbId]);
return (
<PageContainer>
<Box display={['block', 'flex']} h={'100%'} pt={[4, 0]}>
{isPc ? (
<Flex
flexDirection={'column'}
p={4}
h={'100%'}
flex={'0 0 200px'}
borderRight={theme.borders.base}
>
<Flex mb={4} alignItems={'center'}>
<Avatar src={kbDetail.avatar} w={'34px'} borderRadius={'lg'} />
<Box ml={2} fontWeight={'bold'}>
{kbDetail.name}
</Box>
</Flex>
<SideTabs
flex={1}
mx={'auto'}
mt={2}
w={'100%'}
list={tabList.current}
activeId={currentTab}
onChange={(e: any) => {
setCurrentTab(e);
}}
/>
<Box textAlign={'center'}>
<Flex justifyContent={'center'} alignItems={'center'}>
<MyIcon mr={1} name="overviewLight" w={'16px'} color={'green.500'} />
<Box>{t('dataset.System Data Queue')}</Box>
<MyTooltip
label={t('dataset.Queue Desc', { title: feConfigs?.systemTitle })}
placement={'top'}
>
<QuestionOutlineIcon ml={1} w={'16px'} />
</MyTooltip>
</Flex>
<Box mt={1} fontWeight={'bold'}>
{trainingQueueLen}
</Box>
</Box>
<>
<Script src="/js/pdf.js" strategy="lazyOnload"></Script>
<PageContainer>
<Box display={['block', 'flex']} h={'100%'} pt={[4, 0]}>
{isPc ? (
<Flex
alignItems={'center'}
cursor={'pointer'}
py={2}
px={3}
borderRadius={'md'}
_hover={{ bg: 'myGray.100' }}
onClick={() => router.replace('/kb/list')}
flexDirection={'column'}
p={4}
h={'100%'}
flex={'0 0 200px'}
borderRight={theme.borders.base}
>
<IconButton
mr={3}
icon={<MyIcon name={'backFill'} w={'18px'} color={'myBlue.600'} />}
bg={'white'}
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
h={'28px'}
size={'sm'}
borderRadius={'50%'}
aria-label={''}
<Flex mb={4} alignItems={'center'}>
<Avatar src={kbDetail.avatar} w={'34px'} borderRadius={'lg'} />
<Box ml={2} fontWeight={'bold'}>
{kbDetail.name}
</Box>
</Flex>
<SideTabs
flex={1}
mx={'auto'}
mt={2}
w={'100%'}
list={tabList.current}
activeId={currentTab}
onChange={(e: any) => {
setCurrentTab(e);
}}
/>
全部知识库
<Box textAlign={'center'}>
<Flex justifyContent={'center'} alignItems={'center'}>
<MyIcon mr={1} name="overviewLight" w={'16px'} color={'green.500'} />
<Box>{t('dataset.System Data Queue')}</Box>
<MyTooltip
label={t('dataset.Queue Desc', { title: feConfigs?.systemTitle })}
placement={'top'}
>
<QuestionOutlineIcon ml={1} w={'16px'} />
</MyTooltip>
</Flex>
<Box mt={1} fontWeight={'bold'}>
{trainingQueueLen}
</Box>
</Box>
<Flex
alignItems={'center'}
cursor={'pointer'}
py={2}
px={3}
borderRadius={'md'}
_hover={{ bg: 'myGray.100' }}
onClick={() => router.replace('/kb/list')}
>
<IconButton
mr={3}
icon={<MyIcon name={'backFill'} w={'18px'} color={'myBlue.600'} />}
bg={'white'}
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
h={'28px'}
size={'sm'}
borderRadius={'50%'}
aria-label={''}
/>
全部知识库
</Flex>
</Flex>
</Flex>
) : (
<Box mb={3}>
<Tabs
m={'auto'}
w={'260px'}
size={isPc ? 'md' : 'sm'}
list={tabList.current.map((item) => ({
id: item.id,
label: item.label
}))}
activeId={currentTab}
onChange={(e: any) => setCurrentTab(e)}
/>
</Box>
)}
) : (
<Box mb={3}>
<Tabs
m={'auto'}
w={'260px'}
size={isPc ? 'md' : 'sm'}
list={tabList.current.map((item) => ({
id: item.id,
label: item.label
}))}
activeId={currentTab}
onChange={(e: any) => setCurrentTab(e)}
/>
</Box>
)}
{!!kbDetail._id && (
<Box flex={'1 0 0'} h={'100%'} pb={[4, 0]}>
{currentTab === TabEnum.data && <DataCard kbId={kbId} />}
{currentTab === TabEnum.import && <ImportData kbId={kbId} />}
{currentTab === TabEnum.test && <Test kbId={kbId} />}
{currentTab === TabEnum.info && <Info ref={InfoRef} kbId={kbId} form={form} />}
</Box>
)}
</Box>
</PageContainer>
{!!kbDetail._id && (
<Box flex={'1 0 0'} h={'100%'} pb={[4, 0]}>
{currentTab === TabEnum.dataset && <FileCard kbId={kbId} />}
{currentTab === TabEnum.dataCard && <DataCard kbId={kbId} />}
{currentTab === TabEnum.import && <ImportData kbId={kbId} />}
{currentTab === TabEnum.test && <Test kbId={kbId} />}
{currentTab === TabEnum.info && <Info ref={InfoRef} kbId={kbId} form={form} />}
</Box>
)}
</Box>
</PageContainer>
</>
);
};
export async function getServerSideProps(context: any) {
const currentTab = context?.query?.currentTab || TabEnum.data;
const currentTab = context?.query?.currentTab || TabEnum.dataset;
const kbId = context?.query?.kbId;
return {
......
client/src/pages/kb/list/component/CreateModal.tsx
......@@ -18,7 +18,7 @@ import MySelect from '@/components/Select';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import Tag from '@/components/Tag';
const CreateModal = ({ onClose }: { onClose: () => void }) => {
const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: string }) => {
const [refresh, setRefresh] = useState(false);
const { toast } = useToast();
const router = useRouter();
......@@ -28,7 +28,9 @@ const CreateModal = ({ onClose }: { onClose: () => void }) => {
avatar: '/icon/logo.svg',
name: '',
tags: [],
vectorModel: vectorModelList[0].model
vectorModel: vectorModelList[0].model,
type: 'dataset',
parentId
}
});
const InputRef = useRef<HTMLInputElement>(null);
......
client/src/pages/kb/list/component/EditFolderModal.tsx 0 → 100644
import React, { useMemo, useRef } from 'react';
import { ModalFooter, ModalBody, Input, Button } from '@chakra-ui/react';
import MyModal from '@/components/MyModal';
import { useTranslation } from 'react-i18next';
import { useRequest } from '@/hooks/useRequest';
import { postCreateKb, putKbById } from '@/api/plugins/kb';
import { FolderAvatarSrc, KbTypeEnum } from '@/constants/kb';
const EditFolderModal = ({
onClose,
onSuccess,
id,
parentId,
name
}: {
onClose: () => void;
onSuccess: () => void;
id?: string;
parentId?: string;
name?: string;
}) => {
const { t } = useTranslation();
const inputRef = useRef<HTMLInputElement>(null);
const typeMap = useMemo(
() =>
id
? {
title: t('kb.Edit Folder')
}
: {
title: t('kb.Create Folder')
},
[id, t]
);
const { mutate: onSave, isLoading } = useRequest({
mutationFn: () => {
const val = inputRef.current?.value;
if (!val) return Promise.resolve('');
if (id) {
return putKbById({
id,
name: val
});
}
return postCreateKb({
parentId,
name: val,
type: KbTypeEnum.folder,
avatar: FolderAvatarSrc,
tags: []
});
},
onSuccess: (res) => {
if (!res) return;
onSuccess();
onClose();
}
});
return (
<MyModal isOpen onClose={onClose} title={typeMap.title}>
<ModalBody>
<Input
ref={inputRef}
defaultValue={name}
placeholder={t('kb.Folder Name') || ''}
autoFocus
maxLength={20}
/>
</ModalBody>
<ModalFooter>
<Button mr={3} variant={'base'} onClick={onClose}>
{t('common.Cancel')}
</Button>
<Button isLoading={isLoading} onClick={onSave}>
{t('Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};
export default EditFolderModal;
client/src/pages/kb/list/index.tsx
import React, { useCallback } from 'react';
import React, { useCallback, useMemo, useRef, useState } from 'react';
import {
Box,
Card,
Flex,
Grid,
useTheme,
Button,
useDisclosure,
Card,
IconButton,
useDisclosure
MenuButton,
Image
} from '@chakra-ui/react';
import { useRouter } from 'next/router';
import { useUserStore } from '@/store/user';
import { useDatasetStore } from '@/store/dataset';
import PageContainer from '@/components/PageContainer';
import { useConfirm } from '@/hooks/useConfirm';
import { AddIcon } from '@chakra-ui/icons';
import { useQuery } from '@tanstack/react-query';
import { useToast } from '@/hooks/useToast';
import { delKbById } from '@/api/plugins/kb';
import { delKbById, getKbPaths } from '@/api/plugins/kb';
import { useTranslation } from 'react-i18next';
import Avatar from '@/components/Avatar';
import MyIcon from '@/components/Icon';
import Tag from '@/components/Tag';
import { serviceSideProps } from '@/utils/i18n';
import dynamic from 'next/dynamic';
import { FolderAvatarSrc, KbTypeEnum } from '@/constants/kb';
import Tag from '@/components/Tag';
import MyMenu from '@/components/MyMenu';
import { useRequest } from '@/hooks/useRequest';
import { useGlobalStore } from '@/store/global';
const CreateModal = dynamic(() => import('./component/CreateModal'), { ssr: false });
const EditFolderModal = dynamic(() => import('./component/EditFolderModal'), { ssr: false });
const Kb = () => {
const { t } = useTranslation();
const theme = useTheme();
const router = useRouter();
const { parentId } = router.query as { parentId: string };
const { toast } = useToast();
const { setLoading } = useGlobalStore();
const DeleteTipsMap = useRef({
[KbTypeEnum.folder]: t('kb.deleteFolderTips'),
[KbTypeEnum.dataset]: t('kb.deleteDatasetTips')
});
const { openConfirm, ConfirmModal } = useConfirm({
title: '删除提示',
content: '确认删除该知识库？知识库相关的文件、记录将永久删除，无法恢复！'
title: t('common.Delete Warning'),
content: ''
});
const { myKbList, loadKbList, setKbList } = useUserStore();
const { myKbList, loadKbList, setKbList } = useDatasetStore();
const {
isOpen: isOpenCreateModal,
onOpen: onOpenCreateModal,
onClose: onCloseCreateModal
} = useDisclosure();
const { refetch } = useQuery(['loadKbList'], () => loadKbList());
const [editFolderData, setEditFolderData] = useState<{
id?: string;
name?: string;
}>();
/* 点击删除 */
const onclickDelKb = useCallback(
async (id: string) => {
try {
delKbById(id);
toast({
title: '删除成功',
status: 'success'
});
setKbList(myKbList.filter((item) => item._id !== id));
} catch (err: any) {
toast({
title: err?.message || '删除失败',
status: 'error'
});
}
const { mutate: onclickDelKb } = useRequest({
mutationFn: async (id: string) => {
setLoading(true);
await delKbById(id);
return id;
},
onSuccess(id: string) {
setKbList(myKbList.filter((item) => item._id !== id));
},
onSettled() {
setLoading(false);
},
[toast, setKbList, myKbList]
successToast: t('common.Delete Success'),
errorToast: t('kb.Delete Dataset Error')
});
const { data, refetch } = useQuery(['loadKbList', parentId], () => {
return Promise.all([loadKbList(parentId), getKbPaths(parentId)]);
});
const paths = useMemo(
() => [
{
parentId: '',
parentName: t('kb.My Dataset')
},
...(data?.[1] || [])
],
[data, t]
);
return (
<PageContainer>
<Flex pt={3} px={5} alignItems={'center'}>
<Box flex={1} className="textlg" letterSpacing={1} fontSize={'24px'} fontWeight={'bold'}>
我的知识库
</Box>
<Button leftIcon={<AddIcon />} variant={'base'} onClick={onOpenCreateModal}>
新建
</Button>
{/* url path */}
{!!parentId ? (
<Flex flex={1}>
{paths.map((item, i) => (
<Flex key={item.parentId} mr={2} alignItems={'center'}>
<Box
fontSize={'lg'}
px={2}
py={1}
borderRadius={'md'}
{...(i === paths.length - 1
? {
cursor: 'default'
}
: {
cursor: 'pointer',
_hover: {
bg: 'myGray.100'
},
onClick: () => {
router.push({
query: {
parentId: item.parentId
}
});
}
})}
>
{item.parentName}
</Box>
{i !== paths.length - 1 && <MyIcon name={'rightArrowLight'} color={'myGray.500'} />}
</Flex>
))}
</Flex>
) : (
<Box flex={1} className="textlg" letterSpacing={1} fontSize={'24px'} fontWeight={'bold'}>
我的知识库
</Box>
)}
<MyMenu
offset={[-30, 10]}
width={120}
Button={
<MenuButton
_hover={{
color: 'myBlue.600'
}}
>
<Flex
alignItems={'center'}
border={theme.borders.base}
px={5}
py={2}
borderRadius={'md'}
cursor={'pointer'}
>
<AddIcon mr={2} />
<Box>{t('Create New')}</Box>
</Flex>
</MenuButton>
}
menuList={[
{
child: (
<Flex>
<Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={1} />
{t('Folder')}
</Flex>
),
onClick: () => setEditFolderData({})
},
{
child: (
<Flex>
<Image src={'/imgs/module/db.png'} alt={''} w={'20px'} mr={1} />
{t('Dataset')}
</Flex>
),
onClick: onOpenCreateModal
}
]}
/>
</Flex>
<Grid
p={5}
......@@ -86,7 +193,7 @@ const Kb = () => {
py={4}
px={5}
cursor={'pointer'}
h={'140px'}
h={'130px'}
border={theme.borders.md}
boxShadow={'none'}
userSelect={'none'}
......@@ -98,18 +205,28 @@ const Kb = () => {
display: 'block'
}
}}
onClick={() =>
router.push({
pathname: '/kb/detail',
query: {
kbId: kb._id
}
})
}
onClick={() => {
if (kb.type === KbTypeEnum.folder) {
router.push({
pathname: '/kb/list',
query: {
parentId: kb._id
}
});
} else if (kb.type === KbTypeEnum.dataset) {
router.push({
pathname: '/kb/detail',
query: {
kbId: kb._id
}
});
}
}}
>
<Flex alignItems={'center'} h={'38px'}>
<Avatar src={kb.avatar} borderRadius={'lg'} w={'28px'} />
<Box ml={3}>{kb.name}</Box>
<IconButton
className="delete"
position={'absolute'}
......@@ -126,7 +243,11 @@ const Kb = () => {
}}
onClick={(e) => {
e.stopPropagation();
openConfirm(() => onclickDelKb(kb._id))();
openConfirm(
() => onclickDelKb(kb._id),
undefined,
DeleteTipsMap.current[kb.type]
)();
}}
/>
</Flex>
......@@ -140,8 +261,14 @@ const Kb = () => {
</Flex>
</Box>
<Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
<MyIcon mr={1} name="kbTest" w={'12px'} />
<Box color={'myGray.500'}>{kb.vectorModel.name}</Box>
{kb.type === KbTypeEnum.folder ? (
<Box color={'myGray.500'}>{t('Folder')}</Box>
) : (
<>
<MyIcon mr={1} name="kbTest" w={'12px'} />
<Box color={'myGray.500'}>{kb.vectorModel.name}</Box>
</>
)}
</Flex>
</Card>
))}
......@@ -155,7 +282,15 @@ const Kb = () => {
</Flex>
)}
<ConfirmModal />
{isOpenCreateModal && <CreateModal onClose={onCloseCreateModal} />}
{isOpenCreateModal && <CreateModal onClose={onCloseCreateModal} parentId={parentId} />}
{!!editFolderData && (
<EditFolderModal
onClose={() => setEditFolderData(undefined)}
onSuccess={refetch}
parentId={parentId}
{...editFolderData}
/>
)}
</PageContainer>
);
};
......
client/src/pages/login/index.tsx
......@@ -13,6 +13,7 @@ import { serviceSideProps } from '@/utils/i18n';
import { setToken } from '@/utils/user';
import { feConfigs } from '@/store/static';
import CommunityModal from '@/components/CommunityModal';
import Script from 'next/script';
const RegisterForm = dynamic(() => import('./components/RegisterForm'));
const ForgetPasswordForm = dynamic(() => import('./components/ForgetPasswordForm'));
......@@ -53,70 +54,77 @@ const Login = () => {
}
return (
<Flex
alignItems={'center'}
justifyContent={'center'}
className={styles.loginPage}
h={'100%'}
px={[0, '10vw']}
>
<>
{feConfigs.googleClientVerKey && (
<Script
src={`https://www.recaptcha.net/recaptcha/api.js?render=${feConfigs.googleClientVerKey}`}
></Script>
)}
<Flex
height="100%"
w={'100%'}
maxW={'1240px'}
maxH={['auto', 'max(660px,80vh)']}
backgroundColor={'#fff'}
alignItems={'center'}
justifyContent={'center'}
py={[5, 10]}
px={'5vw'}
borderRadius={isPc ? 'md' : 'none'}
gap={5}
className={styles.loginPage}
h={'100%'}
px={[0, '10vw']}
>
{isPc && (
<Image
src={'/icon/loginLeft.svg'}
order={pageType === PageTypeEnum.login ? 0 : 2}
flex={'1 0 0'}
w="0"
maxW={'600px'}
height={'100%'}
maxH={'450px'}
alt=""
/>
)}
<Box
position={'relative'}
order={1}
flex={`0 0 ${isPc ? '400px' : '100%'}`}
height={'100%'}
border="1px"
borderColor="gray.200"
py={5}
px={10}
<Flex
height="100%"
w={'100%'}
maxW={'1240px'}
maxH={['auto', 'max(660px,80vh)']}
backgroundColor={'#fff'}
alignItems={'center'}
justifyContent={'center'}
py={[5, 10]}
px={'5vw'}
borderRadius={isPc ? 'md' : 'none'}
gap={5}
>
<DynamicComponent type={pageType} />
{feConfigs?.show_contact && (
<Box
fontSize={'sm'}
color={'myGray.600'}
cursor={'pointer'}
position={'absolute'}
right={5}
bottom={3}
onClick={onOpen}
>
无法登录，点击联系
</Box>
{isPc && (
<Image
src={'/icon/loginLeft.svg'}
order={pageType === PageTypeEnum.login ? 0 : 2}
flex={'1 0 0'}
w="0"
maxW={'600px'}
height={'100%'}
maxH={'450px'}
alt=""
/>
)}
</Box>
</Flex>
{isOpen && <CommunityModal onClose={onClose} />}
</Flex>
<Box
position={'relative'}
order={1}
flex={`0 0 ${isPc ? '400px' : '100%'}`}
height={'100%'}
border="1px"
borderColor="gray.200"
py={5}
px={10}
borderRadius={isPc ? 'md' : 'none'}
>
<DynamicComponent type={pageType} />
{feConfigs?.show_contact && (
<Box
fontSize={'sm'}
color={'myGray.600'}
cursor={'pointer'}
position={'absolute'}
right={5}
bottom={3}
onClick={onOpen}
>
无法登录，点击联系
</Box>
)}
</Box>
</Flex>
{isOpen && <CommunityModal onClose={onClose} />}
</Flex>
</>
);
};
......
client/src/service/events/generateQA.ts
......@@ -10,6 +10,7 @@ import { ChatCompletionRequestMessage } from 'openai';
import { modelToolMap } from '@/utils/plugin';
import { gptMessage2ChatType } from '@/utils/adapt';
import { addLog } from '../utils/tools';
import { splitText2Chunks } from '@/utils/file';
const reduceQueue = () => {
global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
......@@ -157,7 +158,7 @@ A2:
console.log('openai error: 生成QA错误');
console.log(err.response?.status, err.response?.statusText, err.response?.data);
} else {
console.log('生成QA错误:', err);
addLog.error('生成 QA 错误', err);
}
// message error or openai account error
......@@ -212,5 +213,16 @@ function formatSplitText(text: string) {
}
}
// empty result. direct split chunk
if (result.length === 0) {
const splitRes = splitText2Chunks({ text: text, maxLen: 500 });
splitRes.chunks.forEach((item) => {
result.push({
q: item,
a: ''
});
});
}
return result;
}
client/src/service/events/generateVector.ts
......@@ -96,9 +96,7 @@ export async function generateVector(): Promise<any> {
data: err.response?.data
});
} else {
addLog.info('openai error: 生成向量错误', {
err
});
addLog.error('openai error: 生成向量错误', err);
}
// message error or openai account error
......
client/src/service/lib/gridfs.ts
......@@ -2,19 +2,12 @@ import mongoose, { Types } from 'mongoose';
import fs from 'fs';
import fsp from 'fs/promises';
import { ERROR_ENUM } from '../errorCode';
import type { FileInfo } from '@/types/plugin';
enum BucketNameEnum {
dataset = 'dataset'
}
type FileInfo = {
id: string;
filename: string;
size: number;
contentType: string;
encoding: string;
};
export class GridFSStorage {
readonly type = 'gridfs';
readonly bucket: `${BucketNameEnum}`;
......@@ -88,6 +81,7 @@ export class GridFSStorage {
filename: file.filename,
contentType: file.metadata?.contentType,
encoding: file.metadata?.encoding,
uploadDate: file.uploadDate,
size: file.length
};
}
......
client/src/service/models/bill.ts
......@@ -55,7 +55,6 @@ const BillSchema = new Schema({
try {
BillSchema.index({ userId: 1 });
// BillSchema.index({ time: -1 });
BillSchema.index({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 });
} catch (error) {
console.log(error);
......
client/src/service/models/chatItem.ts
......@@ -48,24 +48,7 @@ const ChatItemSchema = new Schema({
}
},
[TaskResponseKeyEnum.responseData]: {
type: [
{
moduleName: String,
price: String,
model: String,
tokens: Number,
question: String,
answer: String,
temperature: Number,
maxToken: Number,
quoteList: Array,
completeMessages: Array,
similarity: Number,
limit: Number,
cqList: Array,
cqResult: String
}
],
type: Array,
default: []
}
});
......
client/src/service/models/kb.ts
import { Schema, model, models, Model } from 'mongoose';
import { kbSchema as SchemaType } from '@/types/mongoSchema';
import { KbTypeMap } from '@/constants/kb';
const kbSchema = new Schema({
parentId: {
type: Schema.Types.ObjectId,
ref: 'kb',
default: null
},
userId: {
type: Schema.Types.ObjectId,
ref: 'user',
......@@ -24,6 +30,12 @@ const kbSchema = new Schema({
required: true,
default: 'text-embedding-ada-002'
},
type: {
type: String,
enum: Object.keys(KbTypeMap),
required: true,
default: 'dataset'
},
tags: {
type: [String],
default: []
......
client/src/service/pg.ts
......@@ -13,6 +13,7 @@ export const connectPg = async (): Promise<Pool> => {
connectionString: process.env.PG_URL,
max: Number(process.env.DB_MAX_LINK || 5),
keepAlive: true,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 5000
});
......@@ -107,6 +108,7 @@ class Pg {
}
LIMIT ${props.limit || 10} OFFSET ${props.offset || 0}
`;
const pg = await connectPg();
return pg.query<T>(sql);
}
......
client/src/service/response.ts
......@@ -44,7 +44,7 @@ export const jsonRes = <T = any>(
if (typeof error === 'string') {
msg = error;
} else if (proxyError[error?.code]) {
msg = '接口连接异常';
msg = '网络连接异常';
} else if (error?.response?.data?.error?.message) {
msg = error?.response?.data?.error?.message;
} else if (openaiAccountError[error?.response?.data?.error?.code]) {
......@@ -85,7 +85,7 @@ export const sseErrRes = (res: NextApiResponse, error: any) => {
if (typeof error === 'string') {
msg = error;
} else if (proxyError[error?.code]) {
msg = '接口连接异常';
msg = '网络连接异常';
} else if (error?.response?.data?.error?.message) {
msg = error?.response?.data?.error?.message;
} else if (openaiAccountError[error?.response?.data?.error?.code]) {
......
client/src/service/utils/auth.ts
......@@ -155,7 +155,7 @@ export const authUser = async ({
})();
return {
userId: uid,
userId: String(uid),
appId,
authType,
user
......
client/src/store/kb.ts → client/src/store/dataset.ts
......@@ -2,18 +2,62 @@ import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer';
import { type KbTestItemType } from '@/types/plugin';
import type { KbItemType, KbListItemType } from '@/types/plugin';
import { getKbList, getKbById, getAllDataset } from '@/api/plugins/kb';
import { defaultKbDetail } from '@/constants/kb';
type State = {
datasets: KbListItemType[];
loadAllDatasets: () => Promise<KbListItemType[]>;
myKbList: KbListItemType[];
loadKbList: (parentId?: string) => Promise<any>;
setKbList(val: KbListItemType[]): void;
kbDetail: KbItemType;
getKbDetail: (id: string, init?: boolean) => Promise<KbItemType>;
kbTestList: KbTestItemType[];
pushKbTestItem: (data: KbTestItemType) => void;
delKbTestItemById: (id: string) => void;
updateKbItemById: (data: KbTestItemType) => void;
};
export const useKbStore = create<State>()(
export const useDatasetStore = create<State>()(
devtools(
persist(
immer((set, get) => ({
datasets: [],
async loadAllDatasets() {
const res = await getAllDataset();
set((state) => {
state.datasets = res;
});
return res;
},
myKbList: [],
async loadKbList(parentId) {
const res = await getKbList(parentId);
set((state) => {
state.myKbList = res;
});
return res;
},
setKbList(val) {
set((state) => {
state.myKbList = val;
});
},
kbDetail: defaultKbDetail,
async getKbDetail(id: string, init = false) {
if (id === get().kbDetail._id && !init) return get().kbDetail;
const data = await getKbById(id);
set((state) => {
state.kbDetail = data;
});
return data;
},
kbTestList: [],
pushKbTestItem(data) {
set((state) => {
......
client/src/store/user.ts
......@@ -7,9 +7,7 @@ import { formatPrice } from '@/utils/user';
import { getTokenLogin, putUserInfo } from '@/api/user';
import { defaultApp } from '@/constants/model';
import { AppListItemType, AppUpdateParams } from '@/types/app';
import type { KbItemType, KbListItemType } from '@/types/plugin';
import { getKbList, getKbById } from '@/api/plugins/kb';
import { defaultKbDetail } from '@/constants/kb';
import type { AppSchema } from '@/types/mongoSchema';
type State = {
......@@ -24,12 +22,6 @@ type State = {
loadAppDetail: (id: string, init?: boolean) => Promise<AppSchema>;
updateAppDetail(appId: string, data: AppUpdateParams): Promise<void>;
clearAppModules(): void;
// kb
myKbList: KbListItemType[];
loadKbList: () => Promise<any>;
setKbList(val: KbListItemType[]): void;
kbDetail: KbItemType;
getKbDetail: (id: string, init?: boolean) => Promise<KbItemType>;
};
export const useUserStore = create<State>()(
......@@ -106,31 +98,6 @@ export const useUserStore = create<State>()(
modules: []
};
});
},
myKbList: [],
async loadKbList() {
const res = await getKbList();
set((state) => {
state.myKbList = res;
});
return res;
},
setKbList(val: KbListItemType[]) {
set((state) => {
state.myKbList = val;
});
},
kbDetail: defaultKbDetail,
async getKbDetail(id: string, init = false) {
if (id === get().kbDetail._id && !init) return get().kbDetail;
const data = await getKbById(id);
set((state) => {
state.kbDetail = data;
});
return data;
}
})),
{
......
client/src/types/index.d.ts
......@@ -28,6 +28,7 @@ export type FeConfigsType = {
beianText?: string;
googleClientVerKey?: string;
gitLoginKey?: string;
exportLimitMinutes?: number;
scripts?: { [key: string]: string }[];
};
export type SystemEnvType = {
......@@ -58,7 +59,6 @@ declare global {
interface Window {
['pdfjs-dist/build/pdf']: any;
particlesJS: any;
grecaptcha: any;
QRCode: any;
umami?: {
......
client/src/types/mongoSchema.d.ts
......@@ -6,6 +6,7 @@ import { TrainingModeEnum } from '@/constants/plugin';
import type { AppModuleItemType } from './app';
import { ChatSourceEnum, OutLinkTypeEnum } from '@/constants/chat';
import { AppTypeEnum } from '@/constants/app';
import { KbTypeEnum } from '@/constants/kb';
export interface UserModelSchema {
_id: string;
......@@ -166,15 +167,17 @@ export interface OutLinkSchema {
type: `${OutLinkTypeEnum}`;
}
export interface kbSchema {
export type kbSchema = {
_id: string;
userId: string;
parentId: string;
updateTime: Date;
avatar: string;
name: string;
vectorModel: string;
tags: string[];
}
type: `${KbTypeEnum}`;
};
export interface informSchema {
_id: string;
......
client/src/types/plugin.d.ts
import { FileStatusEnum } from '@/constants/kb';
import { VectorModelItemType } from './model';
import type { kbSchema } from './mongoSchema';
export type SelectedKbType = { kbId: string; vectorModel: VectorModelItemType }[];
export type KbListItemType = {
_id: string;
avatar: string;
name: string;
tags: string[];
export type KbListItemType = Omit<kbSchema, 'vectorModel'> & {
vectorModel: VectorModelItemType;
};
export type KbPathItemType = {
parentId: string;
parentName: string;
};
/* kb type */
export interface KbItemType {
_id: string;
......@@ -20,6 +23,15 @@ export interface KbItemType {
tags: string;
}
export type KbFileItemType = {
id: string;
size: number;
filename: string;
uploadTime: Date;
chunkLength: number;
status: `${FileStatusEnum}`;
};
export type DatasetItemType = {
q: string; // 提问词
a: string; // 原文
......@@ -42,3 +54,12 @@ export type FetchResultItem = {
url: string;
content: string;
};
export type FileInfo = {
id: string;
filename: string;
size: number;
contentType: string;
encoding: string;
uploadDate: Date;
};
client/src/utils/plugin/openai.ts
......@@ -51,9 +51,18 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) {
const adaptMessages = adaptChatItem_openAI({ messages, reserveId: true });
const token = adaptMessages.reduce((sum, item) => {
const text = `${item.role}\n${item.content}`;
const enc = getOpenAiEncMap();
const encodeText = enc.encode(text);
const tokens = encodeText.length + 3; // 补充估算值
/* use textLen as tokens if encode error */
const tokens = (() => {
try {
const enc = getOpenAiEncMap();
const encodeText = enc.encode(text);
return encodeText.length + 3; // 补充估算值
} catch (error) {
return text.length;
}
})();
return sum + tokens;
}, 0);
......@@ -62,9 +71,14 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) {
export const openAiSliceTextByToken = ({ text, length }: { text: string; length: number }) => {
const enc = getOpenAiEncMap();
const encodeText = enc.encode(text);
const decoder = new TextDecoder();
return decoder.decode(enc.decode(encodeText.slice(0, length)));
try {
const encodeText = enc.encode(text);
const decoder = new TextDecoder();
return decoder.decode(enc.decode(encodeText.slice(0, length)));
} catch (error) {
return text.slice(0, length);
}
};
export const authOpenAiKey = async (key: string) => {
......
client/src/utils/sse.ts
......@@ -44,7 +44,7 @@ export class SSEParseData {
data: parseData
};
} catch (error) {
if (typeof item.data === 'string' && !item.data.startsWith(": ping")) {
if (typeof item.data === 'string' && !item.data.startsWith(': ping')) {
this.storeReadData += item.data;
} else {
this.storeReadData = '';
......
client/src/utils/tools.ts
......@@ -59,6 +59,9 @@ export const Obj2Query = (obj: Record<string, string | number>) => {
return queryParams.toString();
};
/**
* parse string to query object
*/
export const parseQueryString = (str: string) => {
const queryObject: Record<string, any> = {};
......@@ -125,6 +128,16 @@ export const formatTimeToChatTime = (time: Date) => {
return target.format('YYYY/M/D');
};
export const formatFileSize = (bytes: number): string => {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
};
export const hasVoiceApi = typeof window !== 'undefined' && 'speechSynthesis' in window;
/**
* voice broadcast
......
docSite/content/docs/installation/upgrading/44.md 0 → 100644
---
title: '升级到 V4.4'
description: 'FastGPT 从旧版本升级到 V4.4 操作指南'
icon: 'upgrade'
draft: false
toc: true
weight: 996
---
## 执行初始化 API
发起 1 个 HTTP 请求（记得携带 `headers.rootkey`，这个值是环境变量里的）
1. https://xxxxx/api/admin/initv44
```bash
curl --location --request POST 'https://{{host}}/api/admin/initv44' \
--header 'rootkey: {{rootkey}}' \
--header 'Content-Type: application/json'
```
会给初始化 Mongo 的部分字段。
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment