Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 79177660 by Archer Committed by GitHub
Options

Dataset folder manager (#274) 

* feat: retry send

* perf: qa default value

* feat: dataset folder

* feat: kb folder delete and path

* fix: ts

* perf: script load

* feat: fileCard and dataCard

* feat: search file

* feat: max token

* feat: select dataset

* fix: preview chunk

* perf: source update

* export data limit file_id

* docs

* fix: export limit
parent a1a63260
Showing with 1770 additions and 592 deletions
README.md
...@@ -48,7 +48,7 @@ FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开 ...@@ -48,7 +48,7 @@ FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开
- [x] 支持 url 读取、 CSV 批量导入 - [x] 支持 url 读取、 CSV 批量导入
- [x] 支持知识库单独设置向量模型 - [x] 支持知识库单独设置向量模型
- [x] 源文件存储 - [x] 源文件存储
- [x] 文件学习 Agent - [ ] 文件学习 Agent
3. 多种效果测试渠道 3. 多种效果测试渠道
- [x] 知识库单点搜索测试 - [x] 知识库单点搜索测试
- [x] 对话时反馈引用并可修改与删除 - [x] 对话时反馈引用并可修改与删除
......
client/data/config.json
...@@ -9,7 +9,7 @@ ...@@ -9,7 +9,7 @@
"show_doc": true, "show_doc": true,
"systemTitle": "FastGPT", "systemTitle": "FastGPT",
"authorText": "Made by FastGPT Team.", "authorText": "Made by FastGPT Team.",
"gitLoginKey": "", "exportLimitMinutes": 0,
"scripts": [] "scripts": []
}, },
"SystemParams": { "SystemParams": {
...@@ -61,4 +61,4 @@ ...@@ -61,4 +61,4 @@
"maxToken": 16000, "maxToken": 16000,
"price": 0 "price": 0
} }
} }
\ No newline at end of file
client/public/docs/versionIntro.md
### Fast GPT V4.3 ### Fast GPT V4.4
1. 新增 - 知识库源文件存储，可以从引用窗口点击文件名，查看源文件。 1. 新增 - 知识库目录结构
2. 新增 - 用户反馈和管理员标注预期答案，以不断提高模型回复准确率。 该功能为测试版，未来交互可能会有变化，欢迎大家提出宝贵意见。 2. 优化 - [使用文档](https://doc.fastgpt.run/docs/intro/)
3. 优化 - [使用文档](https://doc.fastgpt.run/docs/intro/) 3. [点击查看高级编排介绍文档](https://doc.fastgpt.run/docs/workflow)
4. [点击查看高级编排介绍文档](https://doc.fastgpt.run/docs/workflow) 4. [点击查看商业版](https://fael3z0zfze.feishu.cn/docx/F155dbirfo8vDDx2WgWc6extnwf)
5. [点击查看商业版](https://fael3z0zfze.feishu.cn/docx/F155dbirfo8vDDx2WgWc6extnwf)
client/public/imgs/files/file.svg 0 → 100644
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694327751771" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4992" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M0 0h1024v1024H0V0z" fill="#202425" opacity=".01" p-id="4993"></path><path d="M136.533333 68.266667a68.266667 68.266667 0 0 0-68.266666 68.266666v428.305067a17.066667 17.066667 0 0 0 28.842666 12.356267l237.738667-226.440534a34.133333 34.133333 0 0 1 42.496-3.6864l268.288 178.858667a17.066667 17.066667 0 0 0 22.766933-3.447467L951.978667 171.4176A17.066667 17.066667 0 0 0 955.733333 160.699733V136.533333a68.266667 68.266667 0 0 0-68.266666-68.266666H136.533333z m819.2 255.3856a17.066667 17.066667 0 0 0-30.344533-10.717867l-221.866667 274.705067a17.066667 17.066667 0 0 0-3.7888 10.717866v340.309334a17.066667 17.066667 0 0 0 17.066667 17.066666h170.666667a68.266667 68.266667 0 0 0 68.266666-68.266666V323.652267zM614.4 955.733333a17.066667 17.066667 0 0 0 17.066667-17.066666v-330.990934a17.066667 17.066667 0 0 0-7.611734-14.199466l-204.8-136.533334a17.066667 17.066667 0 0 0-26.5216 14.199467V938.666667a17.066667 17.066667 0 0 0 17.066667 17.066666h204.8z m-307.2 0a17.066667 17.066667 0 0 0 17.066667-17.066666v-443.733334a17.066667 17.066667 0 0 0-28.842667-12.356266l-221.866667 211.285333a17.066667 17.066667 0 0 0-5.290666 12.3904V887.466667a68.266667 68.266667 0 0 0 68.266666 68.266666h170.666667z" fill="#FFAA44" p-id="4994"></path><path d="M73.557333 693.8624a17.066667 17.066667 0 0 0-5.290666 12.3904V887.466667a68.266667 68.266667 0 0 0 68.266666 68.266666h170.666667a17.066667 17.066667 0 0 0 17.066667-17.066666v-443.733334a17.066667 17.066667 0 0 0-28.842667-12.356266l-221.866667 211.285333zM392.533333 938.666667a17.066667 17.066667 0 0 0 17.066667 17.066666h204.8a17.066667 17.066667 0 0 0 17.066667-17.066666v-330.990934a17.066667 17.066667 0 0 0-7.611734-14.199466l-204.8-136.533334a17.066667 17.066667 0 0 0-26.5216 14.199467V938.666667z m307.2 0a17.066667 17.066667 0 0 0 17.066667 17.066666h170.666667a68.266667 68.266667 0 0 0 68.266666-68.266666V323.6864a17.066667 17.066667 0 0 0-30.344533-10.752l-221.866667 274.705067a17.066667 17.066667 0 0 0-3.7888 10.717866v340.309334z" fill="#11AA66" p-id="4995"></path></svg>
\ No newline at end of file
client/public/imgs/files/folder.svg 0 → 100644
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694141197423" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4891" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M855.04 385.024q19.456 2.048 38.912 10.24t33.792 23.04 21.504 37.376 2.048 54.272q-2.048 8.192-8.192 40.448t-14.336 74.24-18.432 86.528-19.456 76.288q-5.12 18.432-14.848 37.888t-25.088 35.328-36.864 26.112-51.2 10.24l-567.296 0q-21.504 0-44.544-9.216t-42.496-26.112-31.744-40.96-12.288-53.76l0-439.296q0-62.464 33.792-97.792t95.232-35.328l503.808 0q22.528 0 46.592 8.704t43.52 24.064 31.744 35.84 12.288 44.032l0 11.264-53.248 0q-40.96 0-95.744-0.512t-116.736-0.512-115.712-0.512-92.672-0.512l-47.104 0q-26.624 0-41.472 16.896t-23.04 44.544q-8.192 29.696-18.432 62.976t-18.432 61.952q-10.24 33.792-20.48 65.536-2.048 8.192-2.048 13.312 0 17.408 11.776 29.184t29.184 11.776q31.744 0 43.008-39.936l54.272-198.656q133.12 1.024 243.712 1.024l286.72 0z" fill="#FFCC66" p-id="4892"></path></svg>
\ No newline at end of file
client/public/locales/en/common.json
...@@ -2,6 +2,10 @@ ...@@ -2,6 +2,10 @@
"App": "App", "App": "App",
"Cancel": "No", "Cancel": "No",
"Confirm": "Yes", "Confirm": "Yes",
"Create New": "Create",
"Dataset": "Dataset",
"Folder": "Folder",
"Name": "Name",
"Running": "Running", "Running": "Running",
"Select value is empty": "Select value is empty", "Select value is empty": "Select value is empty",
"UnKnow": "UnKnow", "UnKnow": "UnKnow",
...@@ -63,10 +67,8 @@ ...@@ -63,10 +67,8 @@
"online": "Online Chat", "online": "Online Chat",
"share": "Share", "share": "Share",
"test": "Test Chat " "test": "Test Chat "
} },
}, "retry": "Retry"
"commom": {
"Password inconsistency": "Password inconsistency"
}, },
"common": { "common": {
"Add": "Add", "Add": "Add",
...@@ -76,14 +78,22 @@ ...@@ -76,14 +78,22 @@
"Copy Successful": "Copy Successful", "Copy Successful": "Copy Successful",
"Course": "", "Course": "",
"Delete": "Delete", "Delete": "Delete",
"Delete Failed": "Delete Failed",
"Delete Success": "Delete Successful",
"Delete Warning": "Warning",
"Filed is repeat": "Filed is repeated", "Filed is repeat": "Filed is repeated",
"Filed is repeated": "", "Filed is repeated": "",
"Input": "Input", "Input": "Input",
"Output": "Output", "Output": "Output",
"Password inconsistency": "Password inconsistency",
"Rename": "Rename",
"Search": "Search",
"Status": "Status",
"export": "" "export": ""
}, },
"dataset": { "dataset": {
"Confirm to delete the data": "Confirm to delete the data?", "Confirm to delete the data": "Confirm to delete the data?",
"Export": "Export",
"Queue Desc": "This data refers to the current amount of training for the entire system. FastGPT uses queued training, and if you have too much data to train, you may need to wait for a while", "Queue Desc": "This data refers to the current amount of training for the entire system. FastGPT uses queued training, and if you have too much data to train, you may need to wait for a while",
"System Data Queue": "Data Queue" "System Data Queue": "Data Queue"
}, },
...@@ -93,9 +103,11 @@ ...@@ -93,9 +103,11 @@
"Create File": "Create File", "Create File": "Create File",
"Create file": "Create file", "Create file": "Create file",
"Drag and drop": "Drag and drop files here", "Drag and drop": "Drag and drop files here",
"Embedding": "Embedding",
"Fetch Url": "Fetch Url", "Fetch Url": "Fetch Url",
"If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format", "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format",
"Parse": "{{name}} Parsing...", "Parse": "{{name}} Parsing...",
"Ready": "Ready",
"Release the mouse to upload the file": "Release the mouse to upload the file", "Release the mouse to upload the file": "Release the mouse to upload the file",
"Select a maximum of 10 files": "Select a maximum of 10 files", "Select a maximum of 10 files": "Select a maximum of 10 files",
"Uploading": "Uploading: {{name}}, Progress: {{percent}}%", "Uploading": "Uploading: {{name}}, Progress: {{percent}}%",
...@@ -149,6 +161,24 @@ ...@@ -149,6 +161,24 @@
"desc": "AI knowledge base question and answer platform based on LLM large model", "desc": "AI knowledge base question and answer platform based on LLM large model",
"slogan": "Let the AI know more about you" "slogan": "Let the AI know more about you"
}, },
"kb": {
"Chunk Length": "Chunk Length",
"Confirm to delete the file": "Are you sure to delete the file and all its data?",
"Create Folder": "Create Folder",
"Delete Dataset Error": "Delete dataset failed",
"Edit Folder": "Edit Folder",
"File Size": "File Size",
"Filename": "Filename",
"Files": "{{total}} Files",
"Folder Name": "Input folder name",
"My Dataset": "My Dataset",
"Other Data": "Other Data",
"Select Dataset": "Select Dataset",
"Select Folder": "Enter folder",
"Upload Time": "Upload Time",
"deleteDatasetTips": "Are you sure to delete the knowledge base? Data cannot be recovered after deletion, please confirm!",
"deleteFolderTips": "Are you sure to delete this folder and all the knowledge bases it contains? Data cannot be recovered after deletion, please confirm!"
},
"navbar": { "navbar": {
"Account": "Account", "Account": "Account",
"Apps": "Apps", "Apps": "Apps",
......
client/public/locales/zh/common.json
...@@ -2,6 +2,10 @@ ...@@ -2,6 +2,10 @@
"App": "应用", "App": "应用",
"Cancel": "取消", "Cancel": "取消",
"Confirm": "确认", "Confirm": "确认",
"Create New": "新建",
"Dataset": "知识库",
"Folder": "文件夹",
"Name": "名称",
"Running": "运行中", "Running": "运行中",
"Select value is empty": "选择的内容为空", "Select value is empty": "选择的内容为空",
"UnKnow": "未知", "UnKnow": "未知",
...@@ -63,10 +67,8 @@ ...@@ -63,10 +67,8 @@
"online": "在线使用", "online": "在线使用",
"share": "外部链接调用", "share": "外部链接调用",
"test": "测试" "test": "测试"
} },
}, "retry": "重新生成"
"commom": {
"Password inconsistency": "两次密码不一致"
}, },
"common": { "common": {
"Add": "添加", "Add": "添加",
...@@ -76,14 +78,22 @@ ...@@ -76,14 +78,22 @@
"Copy Successful": "复制成功", "Copy Successful": "复制成功",
"Course": "", "Course": "",
"Delete": "删除", "Delete": "删除",
"Delete Failed": "删除失败",
"Delete Success": "删除成功",
"Delete Warning": "删除警告",
"Filed is repeat": "", "Filed is repeat": "",
"Filed is repeated": "字段重复了", "Filed is repeated": "字段重复了",
"Input": "输入", "Input": "输入",
"Output": "输出", "Output": "输出",
"Password inconsistency": "两次密码不一致",
"Rename": "重命名",
"Search": "搜索",
"Status": "状态",
"export": "" "export": ""
}, },
"dataset": { "dataset": {
"Confirm to delete the data": "确认删除该数据？", "Confirm to delete the data": "确认删除该数据？",
"Export": "导出",
"Queue Desc": "该数据是指整个系统当前待训练的数量。{{title}} 采用排队训练的方式，如果待训练的数据过多，可能需要等待一段时间", "Queue Desc": "该数据是指整个系统当前待训练的数量。{{title}} 采用排队训练的方式，如果待训练的数据过多，可能需要等待一段时间",
"System Data Queue": "排队长度" "System Data Queue": "排队长度"
}, },
...@@ -93,9 +103,11 @@ ...@@ -93,9 +103,11 @@
"Create File": "创建新文件", "Create File": "创建新文件",
"Create file": "创建文件", "Create file": "创建文件",
"Drag and drop": "拖拽文件至此", "Drag and drop": "拖拽文件至此",
"Embedding": "索引中",
"Fetch Url": "链接读取", "Fetch Url": "链接读取",
"If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式", "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式",
"Parse": "{{name}} 解析中...", "Parse": "{{name}} 解析中...",
"Ready": "可用",
"Release the mouse to upload the file": "松开鼠标上传文件", "Release the mouse to upload the file": "松开鼠标上传文件",
"Select a maximum of 10 files": "最多选择10个文件", "Select a maximum of 10 files": "最多选择10个文件",
"Uploading": "正在上传 {{name}}，进度: {{percent}}%", "Uploading": "正在上传 {{name}}，进度: {{percent}}%",
...@@ -149,6 +161,24 @@ ...@@ -149,6 +161,24 @@
"desc": "基于 LLM 大模型的 AI 知识库问答平台", "desc": "基于 LLM 大模型的 AI 知识库问答平台",
"slogan": "让 AI 更懂你的知识" "slogan": "让 AI 更懂你的知识"
}, },
"kb": {
"Chunk Length": "数据总量",
"Confirm to delete the file": "确认删除该文件及其所有数据？",
"Create Folder": "创建文件夹",
"Delete Dataset Error": "删除知识库异常",
"Edit Folder": "编辑文件夹",
"File Size": "文件大小",
"Filename": "文件名",
"Files": "文件: {{total}}个",
"Folder Name": "输入文件夹名称",
"My Dataset": "我的知识库",
"Other Data": "其他数据",
"Select Dataset": "选择该知识库",
"Select Folder": "进入文件夹",
"Upload Time": "上传时间",
"deleteDatasetTips": "确认删除该知识库？删除后数据无法恢复，请确认！",
"deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！"
},
"navbar": { "navbar": {
"Account": "账号", "Account": "账号",
"Apps": "应用", "Apps": "应用",
......
client/src/api/plugins/kb.ts
import { GET, POST, PUT, DELETE } from '../request'; import { GET, POST, PUT, DELETE } from '../request';
import type { DatasetItemType, KbItemType, KbListItemType } from '@/types/plugin'; import type {
import { RequestPaging } from '@/types/index'; DatasetItemType,
FileInfo,
KbFileItemType,
KbItemType,
KbListItemType,
KbPathItemType
} from '@/types/plugin';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import { import {
Props as PushDataProps, Props as PushDataProps,
...@@ -10,13 +16,17 @@ import { ...@@ -10,13 +16,17 @@ import {
Props as SearchTestProps, Props as SearchTestProps,
Response as SearchTestResponse Response as SearchTestResponse
} from '@/pages/api/openapi/kb/searchTest'; } from '@/pages/api/openapi/kb/searchTest';
import { Response as KbDataItemType } from '@/pages/api/plugins/kb/data/getDataById';
import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData'; import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData';
import type { KbUpdateParams, CreateKbParams } from '../request/kb'; import type { KbUpdateParams, CreateKbParams, GetKbDataListProps } from '../request/kb';
import { QuoteItemType } from '@/types/chat'; import { QuoteItemType } from '@/types/chat';
/* knowledge base */ /* knowledge base */
export const getKbList = () => GET<KbListItemType[]>(`/plugins/kb/list`); export const getKbList = (parentId?: string) =>
GET<KbListItemType[]>(`/plugins/kb/list`, { parentId });
export const getAllDataset = () => GET<KbListItemType[]>(`/plugins/kb/allDataset`);
export const getKbPaths = (parentId?: string) =>
GET<KbPathItemType[]>('/plugins/kb/paths', { parentId });
export const getKbById = (id: string) => GET<KbItemType>(`/plugins/kb/detail?id=${id}`); export const getKbById = (id: string) => GET<KbItemType>(`/plugins/kb/detail?id=${id}`);
...@@ -26,25 +36,27 @@ export const putKbById = (data: KbUpdateParams) => PUT(`/plugins/kb/update`, dat ...@@ -26,25 +36,27 @@ export const putKbById = (data: KbUpdateParams) => PUT(`/plugins/kb/update`, dat
export const delKbById = (id: string) => DELETE(`/plugins/kb/delete?id=${id}`); export const delKbById = (id: string) => DELETE(`/plugins/kb/delete?id=${id}`);
/* kb file */
export const getKbFiles = (data: { kbId: string; searchText: string }) =>
GET<KbFileItemType[]>(`/plugins/kb/file/list`, data);
export const deleteKbFileById = (params: { fileId: string; kbId: string }) =>
DELETE(`/plugins/kb/file/delFileByFileId`, params);
export const getFileInfoById = (fileId: string) =>
GET<FileInfo>(`/plugins/kb/file/getFileInfo`, { fileId });
export const delEmptyFiles = (kbId: string) =>
DELETE(`/plugins/kb/file/deleteEmptyFiles`, { kbId });
/* kb data */ /* kb data */
type GetKbDataListProps = RequestPaging & {
kbId: string;
searchText: string;
};
export const getKbDataList = (data: GetKbDataListProps) => export const getKbDataList = (data: GetKbDataListProps) =>
POST(`/plugins/kb/data/getDataList`, data); POST(`/plugins/kb/data/getDataList`, data);
/** /**
* 获取导出数据（不分页） * 获取导出数据（不分页）
*/ */
export const getExportDataList = (kbId: string) => export const getExportDataList = (data: { kbId: string; fileId: string }) =>
GET<[string, string, string][]>( GET<[string, string, string][]>(`/plugins/kb/data/exportModelData`, data, {
`/plugins/kb/data/exportModelData`, timeout: 600000
{ kbId }, });
{
timeout: 600000
}
);
/** /**
* 获取模型正在拆分数据的数量 * 获取模型正在拆分数据的数量
......
client/src/api/request/kb.d.ts
import { KbTypeEnum } from '@/constants/kb';
import type { RequestPaging } from '@/types';
export type KbUpdateParams = { export type KbUpdateParams = {
id: string; id: string;
name: string; tags?: string;
tags: string; name?: string;
avatar: string; avatar?: string;
}; };
export type CreateKbParams = { export type CreateKbParams = {
parentId?: string;
name: string; name: string;
tags: string[]; tags: string[];
avatar: string; avatar: string;
vectorModel: string; vectorModel?: string;
type: `${KbTypeEnum}`;
};
export type GetKbDataListProps = RequestPaging & {
kbId: string;
searchText: string;
fileId: string;
}; };
client/src/components/ChatBox/QuoteModal.tsx
...@@ -19,7 +19,7 @@ const QuoteModal = ({ ...@@ -19,7 +19,7 @@ const QuoteModal = ({
rawSearch = [], rawSearch = [],
onClose onClose
}: { }: {
onUpdateQuote: (quoteId: string, sourceText: string) => Promise<void>; onUpdateQuote: (quoteId: string, sourceText?: string) => Promise<void>;
rawSearch: SearchType[]; rawSearch: SearchType[];
onClose: () => void; onClose: () => void;
}) => { }) => {
...@@ -129,7 +129,7 @@ const QuoteModal = ({ ...@@ -129,7 +129,7 @@ const QuoteModal = ({
{editDataItem && ( {editDataItem && (
<InputDataModal <InputDataModal
onClose={() => setEditDataItem(undefined)} onClose={() => setEditDataItem(undefined)}
onSuccess={() => onUpdateQuote(editDataItem.id, '手动修改')} onSuccess={() => onUpdateQuote(editDataItem.id)}
onDelete={() => onUpdateQuote(editDataItem.id, '已删除')} onDelete={() => onUpdateQuote(editDataItem.id, '已删除')}
kbId={editDataItem.kb_id} kbId={editDataItem.kb_id}
defaultValues={{ defaultValues={{
......
client/src/components/ChatBox/ResponseTags.tsx
...@@ -44,7 +44,7 @@ const ResponseTags = ({ ...@@ -44,7 +44,7 @@ const ResponseTags = ({
}; };
}, [responseData]); }, [responseData]);
const updateQuote = useCallback(async (quoteId: string, sourceText: string) => {}, []); const updateQuote = useCallback(async (quoteId: string, sourceText?: string) => {}, []);
const TagStyles: BoxProps = { const TagStyles: BoxProps = {
mr: 2, mr: 2,
......
client/src/components/ChatBox/SelectDataset.tsx
...@@ -12,7 +12,7 @@ import { ...@@ -12,7 +12,7 @@ import {
import MyModal from '../MyModal'; import MyModal from '../MyModal';
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
import { useQuery } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query';
import { useUserStore } from '@/store/user'; import { useDatasetStore } from '@/store/dataset';
import { useToast } from '@/hooks/useToast'; import { useToast } from '@/hooks/useToast';
import Avatar from '../Avatar'; import Avatar from '../Avatar';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
...@@ -29,10 +29,10 @@ const SelectDataset = ({ ...@@ -29,10 +29,10 @@ const SelectDataset = ({
const theme = useTheme(); const theme = useTheme();
const { isPc } = useGlobalStore(); const { isPc } = useGlobalStore();
const { toast } = useToast(); const { toast } = useToast();
const { myKbList, loadKbList } = useUserStore(); const { myKbList, loadKbList } = useDatasetStore();
const [selectedId, setSelectedId] = useState<string>(); const [selectedId, setSelectedId] = useState<string>();
useQuery(['loadKbList'], loadKbList); useQuery(['loadKbList'], () => loadKbList());
return ( return (
<MyModal isOpen={true} onClose={onClose} w={'100%'} maxW={['90vw', '900px']} isCentered={!isPc}> <MyModal isOpen={true} onClose={onClose} w={'100%'} maxW={['90vw', '900px']} isCentered={!isPc}>
......
client/src/components/ChatBox/WholeResponseModal.tsx
import React, { useMemo } from 'react'; import React, { useMemo } from 'react';
import { Box, ModalBody, useTheme, ModalHeader, Flex } from '@chakra-ui/react'; import { Box, ModalBody, useTheme, Flex } from '@chakra-ui/react';
import type { ChatHistoryItemResType } from '@/types/chat'; import type { ChatHistoryItemResType } from '@/types/chat';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
......
client/src/components/ChatBox/index.tsx
...@@ -55,6 +55,7 @@ const SelectDataset = dynamic(() => import('./SelectDataset')); ...@@ -55,6 +55,7 @@ const SelectDataset = dynamic(() => import('./SelectDataset'));
const InputDataModal = dynamic(() => import('@/pages/kb/detail/components/InputDataModal')); const InputDataModal = dynamic(() => import('@/pages/kb/detail/components/InputDataModal'));
import styles from './index.module.scss'; import styles from './index.module.scss';
import Script from 'next/script';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24); const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24);
...@@ -293,7 +294,7 @@ const ChatBox = ( ...@@ -293,7 +294,7 @@ const ChatBox = (
* user confirm send prompt * user confirm send prompt
*/ */
const sendPrompt = useCallback( const sendPrompt = useCallback(
async (variables: Record<string, any> = {}, inputVal = '') => { async (variables: Record<string, any> = {}, inputVal = '', history = chatHistory) => {
if (!onStartChat) return; if (!onStartChat) return;
if (isChatting) { if (isChatting) {
toast({ toast({
...@@ -314,7 +315,7 @@ const ChatBox = ( ...@@ -314,7 +315,7 @@ const ChatBox = (
} }
const newChatList: ChatSiteItemType[] = [ const newChatList: ChatSiteItemType[] = [
...chatHistory, ...history,
{ {
dataId: nanoid(), dataId: nanoid(),
obj: 'Human', obj: 'Human',
...@@ -407,6 +408,22 @@ const ChatBox = ( ...@@ -407,6 +408,22 @@ const ChatBox = (
] ]
); );
// retry input
const retryInput = useCallback(
async (index: number) => {
if (!onDelMessage) return;
const delHistory = chatHistory.slice(index);
setChatHistory((state) => (index === 0 ? [] : state.slice(0, index)));
await Promise.all(
delHistory.map((item, i) => onDelMessage({ contentId: item.dataId, index: index + i }))
);
sendPrompt(variables, delHistory[0].value, chatHistory.slice(0, index));
},
[chatHistory, onDelMessage, sendPrompt, variables]
);
// output data // output data
useImperativeHandle(ref, () => ({ useImperativeHandle(ref, () => ({
getChatHistory: () => chatHistory, getChatHistory: () => chatHistory,
...@@ -470,7 +487,7 @@ const ChatBox = ( ...@@ -470,7 +487,7 @@ const ChatBox = (
); );
const statusBoxData = useMemo(() => { const statusBoxData = useMemo(() => {
const colorMap = { const colorMap = {
loading: '#67c13b', loading: 'myGray.700',
running: '#67c13b', running: '#67c13b',
finish: 'myBlue.600' finish: 'myBlue.600'
}; };
...@@ -484,6 +501,7 @@ const ChatBox = ( ...@@ -484,6 +501,7 @@ const ChatBox = (
}; };
}, [chatHistory, isChatting, t]); }, [chatHistory, isChatting, t]);
// page change and abort request
useEffect(() => { useEffect(() => {
return () => { return () => {
controller.current?.abort('leave'); controller.current?.abort('leave');
...@@ -492,16 +510,7 @@ const ChatBox = ( ...@@ -492,16 +510,7 @@ const ChatBox = (
}; };
}, [router.query]); }, [router.query]);
useEffect(() => { // page destroy and abort request
event.on('guideClick', ({ text }: { text: string }) => {
if (!text) return;
handleSubmit((data) => sendPrompt(data, text))();
});
return () => {
event.off('guideClick');
};
}, [handleSubmit, sendPrompt]);
useEffect(() => { useEffect(() => {
const listen = () => { const listen = () => {
cancelBroadcast(); cancelBroadcast();
...@@ -513,8 +522,22 @@ const ChatBox = ( ...@@ -513,8 +522,22 @@ const ChatBox = (
}; };
}, []); }, []);
// add guide text listener
useEffect(() => {
event.on('guideClick', ({ text }: { text: string }) => {
if (!text) return;
handleSubmit((data) => sendPrompt(data, text))();
});
return () => {
event.off('guideClick');
};
}, [handleSubmit, sendPrompt]);
return ( return (
<Flex flexDirection={'column'} h={'100%'}> <Flex flexDirection={'column'} h={'100%'}>
<Script src="/js/html2pdf.bundle.min.js" strategy="lazyOnload"></Script>
<Box ref={ChatBoxRef} flex={'1 0 0'} h={0} w={'100%'} overflow={'overlay'} px={[4, 0]} pb={3}> <Box ref={ChatBoxRef} flex={'1 0 0'} h={0} w={'100%'} overflow={'overlay'} px={[4, 0]} pb={3}>
<Box id="chat-container" maxW={['100%', '92%']} h={'100%'} mx={'auto'}> <Box id="chat-container" maxW={['100%', '92%']} h={'100%'} mx={'auto'}>
{showEmpty && <Empty />} {showEmpty && <Empty />}
...@@ -616,7 +639,7 @@ const ChatBox = ( ...@@ -616,7 +639,7 @@ const ChatBox = (
justifyContent={'flex-end'} justifyContent={'flex-end'}
mr={3} mr={3}
> >
<MyTooltip label={'复制'}> <MyTooltip label={t('common.Copy')}>
<MyIcon <MyIcon
{...controlIconStyle} {...controlIconStyle}
name={'copy'} name={'copy'}
...@@ -624,8 +647,18 @@ const ChatBox = ( ...@@ -624,8 +647,18 @@ const ChatBox = (
onClick={() => onclickCopy(item.value)} onClick={() => onclickCopy(item.value)}
/> />
</MyTooltip> </MyTooltip>
{!!onDelMessage && (
<MyTooltip label={t('chat.retry')}>
<MyIcon
{...controlIconStyle}
name={'retryLight'}
_hover={{ color: 'green.500' }}
onClick={() => retryInput(index)}
/>
</MyTooltip>
)}
{onDelMessage && ( {onDelMessage && (
<MyTooltip label={'删除'}> <MyTooltip label={t('common.Delete')}>
<MyIcon <MyIcon
{...controlIconStyle} {...controlIconStyle}
mr={0} mr={0}
......
client/src/components/Icon/icons/fill/plus.svg 0 → 100644
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694331723034"
class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5978"
xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64">
<path
d="M512 70.283C267.486 70.283 69.268 268.046 69.268 512S267.486 953.717 512 953.717 954.732 755.954 954.732 512 756.514 70.283 512 70.283m223.045 488.321H558.603v176.442c0 25.738-20.866 46.604-46.604 46.604s-46.604-20.866-46.604-46.604V558.604H288.953c-25.738 0-46.604-20.866-46.604-46.604s20.866-46.604 46.604-46.604h176.442V288.954c0-25.738 20.866-46.604 46.604-46.604s46.604 20.866 46.604 46.604v176.442h176.442c25.738 0 46.604 20.866 46.604 46.604s-20.866 46.604-46.604 46.604z"
p-id="5979"></path>
</svg>
\ No newline at end of file
client/src/components/Icon/icons/light/retry.svg 0 → 100644
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694067364830"
class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5118"
xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64">
<path
d="M727.950222 274.773333l-55.296-9.329777a38.741333 38.741333 0 0 0-12.856889 76.344888l193.308445 32.597334c1.991111 0.113778 1.991111 0.113778 2.844444 0 2.275556 0.227556 4.266667 0.113778 7.850667-0.284445l0.682667-0.056889a28.216889 28.216889 0 0 0 5.632-0.967111c1.080889 0 1.080889 0 3.185777-0.568889a15.530667 15.530667 0 0 0 4.039111-2.332444l1.137778-0.796444 0.796445-0.398223a28.444444 28.444444 0 0 0 4.152889-2.730666 37.091556 37.091556 0 0 0 6.542222-6.826667l0.796444-0.967111c1.080889-1.422222 1.080889-1.422222 2.161778-3.128889a37.432889 37.432889 0 0 0 3.697778-9.557333c0.568889-1.194667 0.568889-1.194667 1.137778-3.128889 0.113778-1.763556 0.113778-1.763556 0-2.503111v0.910222a36.579556 36.579556 0 0 0-0.341334-10.24l-0.113778-0.967111a22.755556 22.755556 0 0 0-0.682666-3.982222c0-1.080889 0-1.080889-0.568889-3.128889l-68.494222-183.751111a38.798222 38.798222 0 0 0-49.777778-22.755556 38.798222 38.798222 0 0 0-22.755556 49.777778l16.270223 43.804444A397.880889 397.880889 0 0 0 512 113.777778C292.408889 113.777778 113.777778 292.408889 113.777778 512s178.631111 398.222222 398.222222 398.222222 398.222222-178.631111 398.222222-398.222222a38.684444 38.684444 0 1 0-77.368889 0c0 176.924444-143.928889 320.853333-320.853333 320.853333S191.146667 688.924444 191.146667 512 335.075556 191.146667 512 191.146667c80.099556 0 157.070222 29.980444 215.950222 83.626666z"
p-id="5119"></path>
</svg>
\ No newline at end of file
client/src/components/Icon/icons/light/rightArrow.svg 0 → 100644
<svg width="24" height="24" viewBox="0 0 24 24">
<path d="M8.3 5.7a1 1 0 011.4-1.4l7.71 7.7-7.7 7.7a1 1 0 11-1.42-1.4l6.3-6.3-6.3-6.3z" fill-rule="nonzero"></path>
</svg>
\ No newline at end of file
client/src/components/Icon/icons/light/search.svg 0 → 100644
<?xml version="1.0" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694224177076"
class="icon" viewBox="0 0 1026 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="3984"
xmlns:xlink="http://www.w3.org/1999/xlink" width="64.125" height="64">
<path
d="M989.365124 873.455175c21.85764 24.973422 33.760499 46.831061 35.714294 65.567202 1.948078 18.730424-5.271103 37.076377-21.661831 55.026425-18.736141 21.075836-39.416072 31.030616-62.055515 29.858625-22.633727-1.171991-44.491366-10.344968-65.567202-27.513213L679.093265 806.715982c-35.128298 22.633727-72.786383 40.197876-112.989976 52.68673-40.197876 12.488855-82.545355 18.730424-127.036721 18.730424-60.882095 0-117.863745-11.512672-170.940663-34.536586-53.078347-23.029631-99.523509-54.446147-139.331197-94.252406-39.811976-39.811976-71.228492-86.25285-94.252406-139.329768C11.512672 556.93603 0 499.950092 0 439.066568c0-60.883524 11.512672-117.863745 34.542303-170.940663 23.023914-53.078347 54.44043-99.523509 94.252406-139.331197 39.807688-39.811976 86.25285-71.228492 139.331197-94.252406 53.076918-23.029631 110.058568-34.542303 170.940663-34.542303 60.883524 0 117.869462 11.512672 170.94638 34.542303 53.078347 23.023914 99.517792 54.44043 139.329768 94.252406 39.807688 39.807688 71.222775 86.25285 94.252406 139.331197 23.023914 53.076918 34.536586 110.057139 34.536586 170.940663 0 46.054974-6.633185 89.764536-19.903844 131.134403s-32.002511 79.619664-56.198417 114.742246l38.639985 38.639985c18.730424 18.730424 38.439889 38.249797 59.124108 58.543829s39.61188 39.416072 56.784413 57.371837C973.754771 857.448917 984.680017 868.771497 989.365124 873.455175L989.365124 873.455175zM443.751675 731.779995c40.588063 0 78.83786-7.609369 114.742246-22.829535 35.904385-15.224454 67.13081-36.105911 93.66641-62.641511 26.541317-26.541317 47.422774-57.762025 62.641511-93.66641 15.218737-35.910102 22.835252-74.154183 22.835252-114.747963 0-40.589492-7.615086-78.832143-22.835252-114.742246-15.218737-35.905815-36.100194-67.125093-62.641511-93.667839-26.5356-26.5356-57.762025-47.415628-93.66641-62.641511-35.904385-15.218737-74.154183-22.828106-114.742246-22.828106-40.589492 0-78.83929 7.609369-114.743675 22.828106-35.904385 15.225883-67.129381 36.105911-93.66641 62.641511-26.541317 26.542747-47.422774 57.762025-62.641511 93.667839-15.218737 35.910102-22.829535 74.152753-22.829535 114.742246 0 40.59378 7.610798 78.83786 22.829535 114.747963 15.218737 35.904385 36.100194 67.125093 62.641511 93.66641 26.53703 26.5356 57.762025 47.417057 93.66641 62.641511C364.912385 724.170627 403.162183 731.779995 443.751675 731.779995L443.751675 731.779995zM443.751675 731.779995"
p-id="3985"></path>
</svg>
\ No newline at end of file
client/src/components/Icon/index.tsx
...@@ -79,7 +79,11 @@ const map = { ...@@ -79,7 +79,11 @@ const map = {
promotionLight: require('./icons/light/promotion.svg').default, promotionLight: require('./icons/light/promotion.svg').default,
logsLight: require('./icons/light/logs.svg').default, logsLight: require('./icons/light/logs.svg').default,
badLight: require('./icons/light/bad.svg').default, badLight: require('./icons/light/bad.svg').default,
markLight: require('./icons/light/mark.svg').default markLight: require('./icons/light/mark.svg').default,
retryLight: require('./icons/light/retry.svg').default,
rightArrowLight: require('./icons/light/rightArrow.svg').default,
searchLight: require('./icons/light/search.svg').default,
plusFill: require('./icons/fill/plus.svg').default
}; };
export type IconName = keyof typeof map; export type IconName = keyof typeof map;
......
client/src/components/MyInput/index.tsx 0 → 100644
import React from 'react';
import { Flex, Input, InputProps } from '@chakra-ui/react';
interface Props extends InputProps {
leftIcon?: React.ReactNode;
}
const MyInput = ({ leftIcon, ...props }: Props) => {
return (
<Flex position={'relative'} alignItems={'center'}>
<Input w={'100%'} pl={leftIcon ? '30px' : 3} {...props} />
{leftIcon && (
<Flex
alignItems={'center'}
position={'absolute'}
left={3}
w={'20px'}
zIndex={10}
transform={'translateY(1.5px)'}
>
{leftIcon}
</Flex>
)}
</Flex>
);
};
export default MyInput;
client/src/components/MyMenu/index.tsx 0 → 100644
import React from 'react';
import { Menu, MenuList, MenuItem } from '@chakra-ui/react';
interface Props {
width: number;
offset?: [number, number];
Button: React.ReactNode;
menuList: {
isActive?: boolean;
child: React.ReactNode;
onClick: () => void;
}[];
}
const MyMenu = ({ width, offset = [0, 10], Button, menuList }: Props) => {
const menuItemStyles = {
borderRadius: 'sm',
py: 3,
display: 'flex',
alignItems: 'center',
_hover: {
backgroundColor: 'myWhite.600',
color: 'hover.blue'
}
};
return (
<Menu offset={offset} autoSelect={false} isLazy>
{Button}
<MenuList
minW={`${width}px !important`}
p={'6px'}
border={'1px solid #fff'}
boxShadow={'0px 2px 4px rgba(161, 167, 179, 0.25), 0px 0px 1px rgba(121, 141, 159, 0.25);'}
>
{menuList.map((item, i) => (
<MenuItem
key={i}
{...menuItemStyles}
onClick={item.onClick}
color={item.isActive ? 'hover.blue' : ''}
>
{item.child}
</MenuItem>
))}
</MenuList>
</Menu>
);
};
export default MyMenu;
client/src/constants/common.ts
...@@ -10,7 +10,8 @@ export const fileImgs = [ ...@@ -10,7 +10,8 @@ export const fileImgs = [
{ suffix: 'csv', src: '/imgs/files/csv.svg' }, { suffix: 'csv', src: '/imgs/files/csv.svg' },
{ suffix: '(doc|docs)', src: '/imgs/files/doc.svg' }, { suffix: '(doc|docs)', src: '/imgs/files/doc.svg' },
{ suffix: 'txt', src: '/imgs/files/txt.svg' }, { suffix: 'txt', src: '/imgs/files/txt.svg' },
{ suffix: 'md', src: '/imgs/files/markdown.svg' } { suffix: 'md', src: '/imgs/files/markdown.svg' },
{ suffix: '.', src: '/imgs/files/file.svg' }
]; ];
export enum TrackEventName { export enum TrackEventName {
......
client/src/constants/kb.ts
...@@ -14,3 +14,24 @@ export const defaultKbDetail: KbItemType = { ...@@ -14,3 +14,24 @@ export const defaultKbDetail: KbItemType = {
maxToken: 3000 maxToken: 3000
} }
}; };
export enum KbTypeEnum {
folder = 'folder',
dataset = 'dataset'
}
export enum FileStatusEnum {
embedding = 'embedding',
ready = 'ready'
}
export const KbTypeMap = {
[KbTypeEnum.folder]: {
name: 'folder'
},
[KbTypeEnum.dataset]: {
name: 'dataset'
}
};
export const FolderAvatarSrc = '/imgs/files/folder.svg';
export const OtherFileId = 'other';
client/src/hooks/useConfirm.tsx
import { useCallback, useRef } from 'react'; import { useCallback, useRef, useState } from 'react';
import { import {
AlertDialog, AlertDialog,
AlertDialogBody, AlertDialogBody,
...@@ -11,21 +11,25 @@ import { ...@@ -11,21 +11,25 @@ import {
} from '@chakra-ui/react'; } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
export const useConfirm = (props: { title?: string; content: string }) => { export const useConfirm = (props: { title?: string | null; content?: string | null }) => {
const { t } = useTranslation(); const { t } = useTranslation();
const { title = t('Warning'), content } = props; const { title = t('Warning'), content } = props;
const [customContent, setCustomContent] = useState(content);
const { isOpen, onOpen, onClose } = useDisclosure(); const { isOpen, onOpen, onClose } = useDisclosure();
const cancelRef = useRef(null); const cancelRef = useRef(null);
const confirmCb = useRef<any>(); const confirmCb = useRef<any>();
const cancelCb = useRef<any>(); const cancelCb = useRef<any>();
return { return {
openConfirm: useCallback( openConfirm: useCallback(
(confirm?: any, cancel?: any) => { (confirm?: any, cancel?: any, customContent?: string) => {
confirmCb.current = confirm; confirmCb.current = confirm;
cancelCb.current = cancel; cancelCb.current = cancel;
customContent && setCustomContent(customContent);
return onOpen; return onOpen;
}, },
[onOpen] [onOpen]
...@@ -44,7 +48,7 @@ export const useConfirm = (props: { title?: string; content: string }) => { ...@@ -44,7 +48,7 @@ export const useConfirm = (props: { title?: string; content: string }) => {
{title} {title}
</AlertDialogHeader> </AlertDialogHeader>
<AlertDialogBody>{content}</AlertDialogBody> <AlertDialogBody>{customContent}</AlertDialogBody>
<AlertDialogFooter> <AlertDialogFooter>
<Button <Button
...@@ -70,7 +74,7 @@ export const useConfirm = (props: { title?: string; content: string }) => { ...@@ -70,7 +74,7 @@ export const useConfirm = (props: { title?: string; content: string }) => {
</AlertDialogOverlay> </AlertDialogOverlay>
</AlertDialog> </AlertDialog>
), ),
[content, isOpen, onClose, title] [customContent, isOpen, onClose, t, title]
) )
}; };
}; };
client/src/hooks/useEditInfo.tsx
import React, { useCallback, useRef } from 'react'; import React, { useCallback, useRef, useState } from 'react';
import { ModalFooter, ModalBody, Input, useDisclosure, Button } from '@chakra-ui/react'; import { ModalFooter, ModalBody, Input, useDisclosure, Button } from '@chakra-ui/react';
import MyModal from '@/components/MyModal'; import MyModal from '@/components/MyModal';
......
client/src/pages/_app.tsx
...@@ -41,16 +41,14 @@ function App({ Component, pageProps }: AppProps) { ...@@ -41,16 +41,14 @@ function App({ Component, pageProps }: AppProps) {
const { setLastRoute } = useGlobalStore(); const { setLastRoute } = useGlobalStore();
const [scripts, setScripts] = useState<FeConfigsType['scripts']>([]); const [scripts, setScripts] = useState<FeConfigsType['scripts']>([]);
const [googleClientVerKey, setGoogleVerKey] = useState<string>();
useEffect(() => { useEffect(() => {
// get init data // get init data
(async () => { (async () => {
const { const {
feConfigs: { scripts, googleClientVerKey } feConfigs: { scripts }
} = await clientInitData(); } = await clientInitData();
setScripts(scripts || []); setScripts(scripts || []);
setGoogleVerKey(googleClientVerKey);
})(); })();
// add window error track // add window error track
window.onerror = function (msg, url) { window.onerror = function (msg, url) {
...@@ -94,20 +92,10 @@ function App({ Component, pageProps }: AppProps) { ...@@ -94,20 +92,10 @@ function App({ Component, pageProps }: AppProps) {
/> />
<link rel="icon" href="/favicon.ico" /> <link rel="icon" href="/favicon.ico" />
</Head> </Head>
<Script src="/js/qrcode.min.js" strategy="lazyOnload"></Script>
<Script src="/js/pdf.js" strategy="lazyOnload"></Script>
<Script src="/js/html2pdf.bundle.min.js" strategy="lazyOnload"></Script>
{scripts?.map((item, i) => ( {scripts?.map((item, i) => (
<Script key={i} strategy="lazyOnload" {...item}></Script> <Script key={i} strategy="lazyOnload" {...item}></Script>
))} ))}
{googleClientVerKey && (
<>
<Script
src={`https://www.recaptcha.net/recaptcha/api.js?render=${googleClientVerKey}`}
strategy="lazyOnload"
></Script>
</>
)}
<QueryClientProvider client={queryClient}> <QueryClientProvider client={queryClient}>
<ChakraProvider theme={theme}> <ChakraProvider theme={theme}>
<ColorModeScript initialColorMode={theme.config.initialColorMode} /> <ColorModeScript initialColorMode={theme.config.initialColorMode} />
......
client/src/pages/account/components/UpdatePswModal.tsx
...@@ -25,7 +25,7 @@ const UpdatePswModal = ({ onClose }: { onClose: () => void }) => { ...@@ -25,7 +25,7 @@ const UpdatePswModal = ({ onClose }: { onClose: () => void }) => {
const { mutate: onSubmit, isLoading } = useRequest({ const { mutate: onSubmit, isLoading } = useRequest({
mutationFn: (data: FormType) => { mutationFn: (data: FormType) => {
if (data.newPsw !== data.confirmPsw) { if (data.newPsw !== data.confirmPsw) {
return Promise.reject(t('commom.Password inconsistency')); return Promise.reject(t('common.Password inconsistency'));
} }
return updatePasswordByOld(data); return updatePasswordByOld(data);
}, },
......
client/src/pages/account/index.tsx
...@@ -13,6 +13,7 @@ import UserInfo from './components/Info'; ...@@ -13,6 +13,7 @@ import UserInfo from './components/Info';
import { serviceSideProps } from '@/utils/i18n'; import { serviceSideProps } from '@/utils/i18n';
import { feConfigs } from '@/store/static'; import { feConfigs } from '@/store/static';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import Script from 'next/script';
const Promotion = dynamic(() => import('./components/Promotion')); const Promotion = dynamic(() => import('./components/Promotion'));
const BillTable = dynamic(() => import('./components/BillTable')); const BillTable = dynamic(() => import('./components/BillTable'));
...@@ -97,51 +98,54 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => { ...@@ -97,51 +98,54 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
); );
return ( return (
<PageContainer> <>
<Flex flexDirection={['column', 'row']} h={'100%'} pt={[4, 0]}> <Script src="/js/qrcode.min.js" strategy="lazyOnload"></Script>
{isPc ? ( <PageContainer>
<Flex <Flex flexDirection={['column', 'row']} h={'100%'} pt={[4, 0]}>
flexDirection={'column'} {isPc ? (
p={4} <Flex
h={'100%'} flexDirection={'column'}
flex={'0 0 200px'} p={4}
borderRight={theme.borders.base} h={'100%'}
> flex={'0 0 200px'}
<SideTabs borderRight={theme.borders.base}
flex={1} >
mx={'auto'} <SideTabs
mt={2} flex={1}
w={'100%'} mx={'auto'}
list={tabList.current} mt={2}
activeId={currentTab} w={'100%'}
onChange={setCurrentTab} list={tabList.current}
/> activeId={currentTab}
</Flex> onChange={setCurrentTab}
) : ( />
<Box mb={3}> </Flex>
<Tabs ) : (
m={'auto'} <Box mb={3}>
size={isPc ? 'md' : 'sm'} <Tabs
list={tabList.current.map((item) => ({ m={'auto'}
id: item.id, size={isPc ? 'md' : 'sm'}
label: item.label list={tabList.current.map((item) => ({
}))} id: item.id,
activeId={currentTab} label: item.label
onChange={setCurrentTab} }))}
/> activeId={currentTab}
</Box> onChange={setCurrentTab}
)} />
</Box>
)}
<Box flex={'1 0 0'} h={'100%'} pb={[4, 0]}> <Box flex={'1 0 0'} h={'100%'} pb={[4, 0]}>
{currentTab === TabEnum.info && <UserInfo />} {currentTab === TabEnum.info && <UserInfo />}
{currentTab === TabEnum.promotion && <Promotion />} {currentTab === TabEnum.promotion && <Promotion />}
{currentTab === TabEnum.bill && <BillTable />} {currentTab === TabEnum.bill && <BillTable />}
{currentTab === TabEnum.pay && <PayRecordTable />} {currentTab === TabEnum.pay && <PayRecordTable />}
{currentTab === TabEnum.inform && <InformTable />} {currentTab === TabEnum.inform && <InformTable />}
</Box> </Box>
</Flex> </Flex>
<ConfirmModal /> <ConfirmModal />
</PageContainer> </PageContainer>
</>
); );
}; };
......
client/src/pages/api/admin/initv44.ts 0 → 100644
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase, KB } from '@/service/mongo';
import { KbTypeEnum, KbTypeMap } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authRoot: true });
await KB.updateMany(
{
type: { $exists: false }
},
{
$set: {
type: KbTypeEnum.dataset,
parentId: null
}
}
);
jsonRes(res, {});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}
client/src/pages/api/openapi/kb/pushData.ts
...@@ -88,7 +88,7 @@ export async function pushDataToKb({ ...@@ -88,7 +88,7 @@ export async function pushDataToKb({
]); ]);
const modeMaxToken = { const modeMaxToken = {
[TrainingModeEnum.index]: vectorModel.maxToken, [TrainingModeEnum.index]: vectorModel.maxToken * 1.5,
[TrainingModeEnum.qa]: global.qaModel.maxToken * 0.8 [TrainingModeEnum.qa]: global.qaModel.maxToken * 0.8
}; };
...@@ -146,7 +146,6 @@ export async function pushDataToKb({ ...@@ -146,7 +146,6 @@ export async function pushDataToKb({
} }
} catch (error) { } catch (error) {
console.log(error); console.log(error);
error;
} }
return Promise.resolve(data); return Promise.resolve(data);
}) })
......
client/src/pages/api/openapi/kb/updateData.ts
...@@ -50,7 +50,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex ...@@ -50,7 +50,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
await PgClient.update(PgTrainingTableName, { await PgClient.update(PgTrainingTableName, {
where: [['id', dataId], 'AND', ['user_id', userId]], where: [['id', dataId], 'AND', ['user_id', userId]],
values: [ values: [
{ key: 'source', value: '手动修改' },
{ key: 'a', value: a.replace(/'/g, '"') }, { key: 'a', value: a.replace(/'/g, '"') },
...(q ...(q
? [ ? [
......
client/src/pages/api/openapi/plugin/vector.ts
...@@ -69,7 +69,7 @@ export async function getVector({ ...@@ -69,7 +69,7 @@ export async function getVector({
.then(async (res) => { .then(async (res) => {
if (!res.data?.data?.[0]?.embedding) { if (!res.data?.data?.[0]?.embedding) {
// @ts-ignore // @ts-ignore
return Promise.reject(res.data?.error?.message || 'Embedding API Error'); return Promise.reject(res.data?.err?.message || 'Embedding API Error');
} }
return { return {
tokenLen: res.data.usage.total_tokens || 0, tokenLen: res.data.usage.total_tokens || 0,
......
client/src/pages/api/plugins/kb/allDataset.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { getVectorModel } from '@/service/utils/data';
import { KbListItemType } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const kbList = await KB.find({
userId,
type: 'dataset'
});
const data = kbList.map((item) => ({
...item.toJSON(),
vectorModel: getVectorModel(item.vectorModel)
}));
jsonRes<KbListItemType[]>(res, {
data
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
client/src/pages/api/plugins/kb/create.ts
...@@ -6,11 +6,7 @@ import type { CreateKbParams } from '@/api/request/kb'; ...@@ -6,11 +6,7 @@ import type { CreateKbParams } from '@/api/request/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
const { name, tags, avatar, vectorModel } = req.body as CreateKbParams; const { name, tags, avatar, vectorModel, parentId, type } = req.body as CreateKbParams;
if (!name || !vectorModel) {
throw new Error('缺少参数');
}
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
...@@ -22,7 +18,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -22,7 +18,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
userId, userId,
tags, tags,
vectorModel, vectorModel,
avatar avatar,
parentId: parentId || null,
type
}); });
jsonRes(res, { data: _id }); jsonRes(res, { data: _id });
......
client/src/pages/api/plugins/kb/data/exportModelData.ts
...@@ -4,11 +4,13 @@ import { connectToDatabase, User } from '@/service/mongo'; ...@@ -4,11 +4,13 @@ import { connectToDatabase, User } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgTrainingTableName } from '@/constants/plugin';
import { OtherFileId } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
let { kbId } = req.query as { let { kbId, fileId } = req.query as {
kbId: string; kbId: string;
fileId: string;
}; };
if (!kbId) { if (!kbId) {
...@@ -20,7 +22,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -20,7 +22,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
const thirtyMinutesAgo = new Date(Date.now() - 30 * 60 * 1000); const thirtyMinutesAgo = new Date(
Date.now() - (global.feConfigs?.exportLimitMinutes || 0) * 60 * 1000
);
// auth export times // auth export times
const authTimes = await User.findOne( const authTimes = await User.findOne(
...@@ -35,21 +39,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -35,21 +39,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
); );
if (!authTimes) { if (!authTimes) {
throw new Error('上次导出未到半小时，每半小时仅可导出一次。'); const minutes = `${global.feConfigs?.exportLimitMinutes || 0} 分钟`;
throw new Error(`上次导出未到 ${minutes}，每 ${minutes}仅可导出一次。`);
} }
// 统计数据 const where: any = [['kb_id', kbId], 'AND', ['user_id', userId]];
const count = await PgClient.count(PgTrainingTableName, {
where: [['kb_id', kbId], 'AND', ['user_id', userId]]
});
// 从 pg 中获取所有数据 // 从 pg 中获取所有数据
const pgData = await PgClient.select<{ q: string; a: string; source: string }>( const pgData = await PgClient.select<{ q: string; a: string; source: string }>(
PgTrainingTableName, PgTrainingTableName,
{ {
where: [['kb_id', kbId], 'AND', ['user_id', userId]], where,
fields: ['q', 'a', 'source'], fields: ['q', 'a', 'source'],
order: [{ field: 'id', mode: 'DESC' }], order: [{ field: 'id', mode: 'DESC' }],
limit: count limit: 1000000
} }
); );
...@@ -78,7 +80,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -78,7 +80,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
export const config = { export const config = {
api: { api: {
bodyParser: { bodyParser: {
sizeLimit: '100mb' sizeLimit: '200mb'
} }
} }
}; };
client/src/pages/api/plugins/kb/data/getDataList.ts
...@@ -5,6 +5,7 @@ import { authUser } from '@/service/utils/auth'; ...@@ -5,6 +5,7 @@ import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import type { KbDataItemType } from '@/types/plugin'; import type { KbDataItemType } from '@/types/plugin';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgTrainingTableName } from '@/constants/plugin';
import { OtherFileId } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
...@@ -12,12 +13,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -12,12 +13,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
kbId, kbId,
pageNum = 1, pageNum = 1,
pageSize = 10, pageSize = 10,
searchText = '' searchText = '',
fileId = ''
} = req.body as { } = req.body as {
kbId: string; kbId: string;
pageNum: number; pageNum: number;
pageSize: number; pageSize: number;
searchText: string; searchText: string;
fileId: string;
}; };
if (!kbId) { if (!kbId) {
throw new Error('缺少参数'); throw new Error('缺少参数');
...@@ -33,6 +36,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -33,6 +36,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
['user_id', userId], ['user_id', userId],
'AND', 'AND',
['kb_id', kbId], ['kb_id', kbId],
...(fileId
? fileId === OtherFileId
? ["AND (file_id IS NULL OR file_id = '')"]
: ['AND', ['file_id', fileId]]
: []),
...(searchText ...(searchText
? [ ? [
'AND', 'AND',
......
client/src/pages/api/plugins/kb/delete.ts
...@@ -3,12 +3,12 @@ import { jsonRes } from '@/service/response'; ...@@ -3,12 +3,12 @@ import { jsonRes } from '@/service/response';
import { connectToDatabase, KB, App, TrainingData } from '@/service/mongo'; import { connectToDatabase, KB, App, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { Types } from 'mongoose';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgTrainingTableName } from '@/constants/plugin';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { id } = req.query as { const { id } = req.query as {
id: string; id: string;
}; };
...@@ -20,26 +20,30 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -20,26 +20,30 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); const deletedIds = [id, ...(await findAllChildrenIds(id))];
// delete training data // delete training data
await TrainingData.deleteMany({ await TrainingData.deleteMany({
userId, userId,
kbId: id kbId: { $in: deletedIds }
}); });
// delete all pg data // delete all pg data
await PgClient.delete(PgTrainingTableName, { await PgClient.delete(PgTrainingTableName, {
where: [['user_id', userId], 'AND', ['kb_id', id]] where: [
['user_id', userId],
'AND',
`kb_id IN (${deletedIds.map((id) => `'${id}'`).join(',')})`
]
}); });
// delete related files // delete related files
const gridFs = new GridFSStorage('dataset', userId); const gridFs = new GridFSStorage('dataset', userId);
await gridFs.deleteFilesByKbId(id); await Promise.all(deletedIds.map((id) => gridFs.deleteFilesByKbId(id)));
// delete kb data // delete kb data
await KB.findOneAndDelete({ await KB.deleteMany({
_id: id, _id: { $in: deletedIds },
userId userId
}); });
...@@ -51,3 +55,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -51,3 +55,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}); });
} }
} }
async function findAllChildrenIds(id: string) {
// find children
const children = await KB.find({ parentId: id });
let allChildrenIds = children.map((child) => String(child._id));
for (const child of children) {
const grandChildrenIds = await findAllChildrenIds(child._id);
allChildrenIds = allChildrenIds.concat(grandChildrenIds);
}
return allChildrenIds;
}
client/src/pages/api/plugins/kb/file/delFileByFileId.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { Types } from 'mongoose';
import { OtherFileId } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { fileId, kbId } = req.query as { fileId: string; kbId: string };
if (!fileId || !kbId) {
throw new Error('fileId and kbId is required');
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
if (fileId === OtherFileId) {
await PgClient.delete(PgTrainingTableName, {
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
"AND (file_id IS NULL OR file_id = '')"
]
});
} else {
const gridFs = new GridFSStorage('dataset', userId);
const bucket = gridFs.GridFSBucket();
await gridFs.findAndAuthFile(fileId);
// delete all pg data
await PgClient.delete(PgTrainingTableName, {
where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['file_id', fileId]]
});
// delete file
await bucket.delete(new Types.ObjectId(fileId));
}
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { Types } from 'mongoose';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { kbId } = req.query as { kbId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const gridFs = new GridFSStorage('dataset', userId);
const bucket = gridFs.GridFSBucket();
const files = await bucket
// 1 hours expired
.find({
uploadDate: { $lte: new Date(Date.now() - 60 * 1000) },
['metadata.kbId']: kbId,
['metadata.userId']: userId
})
.sort({ _id: -1 })
.toArray();
const data = await Promise.all(
files.map(async (file) => {
return {
id: file._id,
chunkLength: await PgClient.count(PgTrainingTableName, {
fields: ['id'],
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
'AND',
['file_id', String(file._id)]
]
})
};
})
);
await Promise.all(
data
.filter((item) => item.chunkLength === 0)
.map((file) => bucket.delete(new Types.ObjectId(file.id)))
);
jsonRes(res);
} catch (err) {
jsonRes(res);
}
}
client/src/pages/api/plugins/kb/file/getFileInfo.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { OtherFileId } from '@/constants/kb';
import type { FileInfo } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { fileId } = req.query as { kbId: string; fileId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
if (fileId === OtherFileId) {
return jsonRes<FileInfo>(res, {
data: {
id: OtherFileId,
size: 0,
filename: 'kb.Other Data',
uploadDate: new Date(),
encoding: '',
contentType: ''
}
});
}
const gridFs = new GridFSStorage('dataset', userId);
const file = await gridFs.findAndAuthFile(fileId);
jsonRes<FileInfo>(res, {
data: file
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
client/src/pages/api/plugins/kb/file/list.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { KbFileItemType } from '@/types/plugin';
import { FileStatusEnum, OtherFileId } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
let { kbId, searchText } = req.query as { kbId: string; searchText: string };
searchText = searchText.replace(/'/g, '');
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const gridFs = new GridFSStorage('dataset', userId);
const bucket = gridFs.GridFSBucket();
const files = await bucket
.find({ ['metadata.kbId']: kbId, ...(searchText && { filename: { $regex: searchText } }) })
.sort({ _id: -1 })
.toArray();
async function GetOtherData() {
return {
id: OtherFileId,
size: 0,
filename: 'kb.Other Data',
uploadTime: new Date(),
status: (await TrainingData.findOne({ userId, kbId, file_id: '' }))
? FileStatusEnum.embedding
: FileStatusEnum.ready,
chunkLength: await PgClient.count(PgTrainingTableName, {
fields: ['id'],
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
"AND (file_id IS NULL OR file_id = '')"
]
})
};
}
const data = await Promise.all([
GetOtherData(),
...files.map(async (file) => {
return {
id: String(file._id),
size: file.length,
filename: file.filename,
uploadTime: file.uploadDate,
status: (await TrainingData.findOne({ userId, kbId, file_id: file._id }))
? FileStatusEnum.embedding
: FileStatusEnum.ready,
chunkLength: await PgClient.count(PgTrainingTableName, {
fields: ['id'],
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
'AND',
['file_id', String(file._id)]
]
})
};
})
]);
jsonRes<KbFileItemType[]>(res, {
data: data.flat().filter((item) => item.chunkLength > 0)
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
client/src/pages/api/plugins/kb/list.ts
...@@ -2,29 +2,25 @@ import type { NextApiRequest, NextApiResponse } from 'next'; ...@@ -2,29 +2,25 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo'; import { connectToDatabase, KB } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { KbListItemType } from '@/types/plugin';
import { getVectorModel } from '@/service/utils/data'; import { getVectorModel } from '@/service/utils/data';
import { KbListItemType } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
const { parentId } = req.query as { parentId: string };
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); await connectToDatabase();
const kbList = await KB.find( const kbList = await KB.find({
{ userId,
userId parentId: parentId || null
}, }).sort({ updateTime: -1 });
'_id avatar name tags vectorModel'
).sort({ updateTime: -1 });
const data = await Promise.all( const data = await Promise.all(
kbList.map(async (item) => ({ kbList.map(async (item) => ({
_id: item._id, ...item.toJSON(),
avatar: item.avatar,
name: item.name,
tags: item.tags,
vectorModel: getVectorModel(item.vectorModel) vectorModel: getVectorModel(item.vectorModel)
})) }))
); );
......
client/src/pages/api/plugins/kb/paths.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { KbPathItemType } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { parentId } = req.query as { parentId: string };
jsonRes<KbPathItemType[]>(res, {
data: await getParents(parentId)
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
async function getParents(parentId?: string): Promise<KbPathItemType[]> {
if (!parentId) {
return [];
}
const parent = await KB.findById(parentId, 'name parentId');
if (!parent) return [];
const paths = await getParents(parent.parentId);
paths.push({ parentId, parentName: parent.name });
return paths;
}
client/src/pages/api/plugins/kb/update.ts
...@@ -6,7 +6,7 @@ import type { KbUpdateParams } from '@/api/request/kb'; ...@@ -6,7 +6,7 @@ import type { KbUpdateParams } from '@/api/request/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
const { id, name, tags, avatar } = req.body as KbUpdateParams; const { id, name, avatar, tags = '' } = req.body as KbUpdateParams;
if (!id || !name) { if (!id || !name) {
throw new Error('缺少参数'); throw new Error('缺少参数');
...@@ -23,8 +23,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -23,8 +23,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
userId userId
}, },
{ {
avatar, ...(name && { name }),
name, ...(avatar && { avatar }),
tags: tags.split(' ').filter((item) => item) tags: tags.split(' ').filter((item) => item)
} }
); );
......
client/src/pages/api/system/getInitData.ts
import type { FeConfigsType } from '@/types'; import type { FeConfigsType, SystemEnvType } from '@/types';
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { readFileSync } from 'fs'; import { readFileSync } from 'fs';
...@@ -29,12 +29,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -29,12 +29,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}); });
} }
const defaultSystemEnv = { const defaultSystemEnv: SystemEnvType = {
vectorMaxProcess: 15, vectorMaxProcess: 15,
qaMaxProcess: 15, qaMaxProcess: 15,
pgIvfflatProbe: 20 pgIvfflatProbe: 20
}; };
const defaultFeConfigs = { const defaultFeConfigs: FeConfigsType = {
show_emptyChat: true, show_emptyChat: true,
show_register: false, show_register: false,
show_appStore: false, show_appStore: false,
...@@ -44,7 +44,7 @@ const defaultFeConfigs = { ...@@ -44,7 +44,7 @@ const defaultFeConfigs = {
show_doc: true, show_doc: true,
systemTitle: 'FastGPT', systemTitle: 'FastGPT',
authorText: 'Made by FastGPT Team.', authorText: 'Made by FastGPT Team.',
gitLoginKey: '', exportLimitMinutes: 0,
scripts: [] scripts: []
}; };
const defaultChatModels = [ const defaultChatModels = [
...@@ -99,8 +99,10 @@ export async function getInitConfig() { ...@@ -99,8 +99,10 @@ export async function getInitConfig() {
const res = JSON.parse(readFileSync(filename, 'utf-8')); const res = JSON.parse(readFileSync(filename, 'utf-8'));
console.log(res); console.log(res);
global.systemEnv = res.SystemParams || defaultSystemEnv; global.systemEnv = res.SystemParams
global.feConfigs = res.FeConfig || defaultFeConfigs; ? { ...defaultSystemEnv, ...res.SystemParams }
: defaultSystemEnv;
global.feConfigs = res.FeConfig ? { ...defaultFeConfigs, ...res.FeConfig } : defaultFeConfigs;
global.chatModels = res.ChatModels || defaultChatModels; global.chatModels = res.ChatModels || defaultChatModels;
global.qaModel = res.QAModel || defaultQAModel; global.qaModel = res.QAModel || defaultQAModel;
global.vectorModels = res.VectorModels || defaultVectorModels; global.vectorModels = res.VectorModels || defaultVectorModels;
......
client/src/pages/app/detail/components/AdEdit/components/Nodes/NodeKbSearch.tsx
import React, { useMemo, useState } from 'react'; import React, { useMemo } from 'react';
import { NodeProps } from 'reactflow'; import { NodeProps } from 'reactflow';
import { FlowModuleItemType } from '@/types/flow'; import { FlowModuleItemType } from '@/types/flow';
import { Flex, Box, Button, useTheme, useDisclosure, Grid } from '@chakra-ui/react'; import { Flex, Box, Button, useTheme, useDisclosure, Grid } from '@chakra-ui/react';
import { useUserStore } from '@/store/user'; import { useDatasetStore } from '@/store/dataset';
import { useQuery } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query';
import NodeCard from '../modules/NodeCard'; import NodeCard from '../modules/NodeCard';
import Divider from '../modules/Divider'; import Divider from '../modules/Divider';
...@@ -21,7 +21,7 @@ const KBSelect = ({ ...@@ -21,7 +21,7 @@ const KBSelect = ({
onChange: (e: SelectedKbType) => void; onChange: (e: SelectedKbType) => void;
}) => { }) => {
const theme = useTheme(); const theme = useTheme();
const { myKbList, loadKbList } = useUserStore(); const { datasets, loadAllDatasets } = useDatasetStore();
const { const {
isOpen: isOpenKbSelect, isOpen: isOpenKbSelect,
onOpen: onOpenKbSelect, onOpen: onOpenKbSelect,
...@@ -29,11 +29,11 @@ const KBSelect = ({ ...@@ -29,11 +29,11 @@ const KBSelect = ({
} = useDisclosure(); } = useDisclosure();
const showKbList = useMemo( const showKbList = useMemo(
() => myKbList.filter((item) => activeKbs.find((kb) => kb.kbId === item._id)), () => datasets.filter((item) => activeKbs.find((kb) => kb.kbId === item._id)),
[myKbList, activeKbs] [datasets, activeKbs]
); );
useQuery(['initkb'], loadKbList); useQuery(['loadAllDatasets'], loadAllDatasets);
return ( return (
<> <>
...@@ -58,12 +58,7 @@ const KBSelect = ({ ...@@ -58,12 +58,7 @@ const KBSelect = ({
))} ))}
</Grid> </Grid>
{isOpenKbSelect && ( {isOpenKbSelect && (
<KBSelectModal <KBSelectModal activeKbs={activeKbs} onChange={onChange} onClose={onCloseKbSelect} />
kbList={myKbList}
activeKbs={activeKbs}
onChange={onChange}
onClose={onCloseKbSelect}
/>
)} )}
</> </>
); );
......
client/src/pages/app/detail/components/BasicEdit/index.tsx
...@@ -56,18 +56,21 @@ import MyIcon from '@/components/Icon'; ...@@ -56,18 +56,21 @@ import MyIcon from '@/components/Icon';
import ChatBox, { type ComponentRef, type StartChatFnProps } from '@/components/ChatBox'; import ChatBox, { type ComponentRef, type StartChatFnProps } from '@/components/ChatBox';
import { addVariable } from '../VariableEditModal'; import { addVariable } from '../VariableEditModal';
import { KBSelectModal, KbParamsModal } from '../KBSelectModal'; import { KbParamsModal } from '../KBSelectModal';
import { AppTypeEnum } from '@/constants/app'; import { AppTypeEnum } from '@/constants/app';
import { useDatasetStore } from '@/store/dataset';
const VariableEditModal = dynamic(() => import('../VariableEditModal')); const VariableEditModal = dynamic(() => import('../VariableEditModal'));
const InfoModal = dynamic(() => import('../InfoModal')); const InfoModal = dynamic(() => import('../InfoModal'));
const KBSelectModal = dynamic(() => import('../KBSelectModal'));
const Settings = ({ appId }: { appId: string }) => { const Settings = ({ appId }: { appId: string }) => {
const theme = useTheme(); const theme = useTheme();
const router = useRouter(); const router = useRouter();
const { t } = useTranslation(); const { t } = useTranslation();
const { toast } = useToast(); const { toast } = useToast();
const { appDetail, updateAppDetail, loadKbList, myKbList } = useUserStore(); const { appDetail, updateAppDetail } = useUserStore();
const { loadAllDatasets, datasets } = useDatasetStore();
const { isPc } = useGlobalStore(); const { isPc } = useGlobalStore();
const [editVariable, setEditVariable] = useState<VariableItemType>(); const [editVariable, setEditVariable] = useState<VariableItemType>();
...@@ -122,8 +125,8 @@ const Settings = ({ appId }: { appId: string }) => { ...@@ -122,8 +125,8 @@ const Settings = ({ appId }: { appId: string }) => {
); );
}, [getValues, refresh]); }, [getValues, refresh]);
const selectedKbList = useMemo( const selectedKbList = useMemo(
() => myKbList.filter((item) => kbList.find((kb) => kb.kbId === item._id)), () => datasets.filter((item) => kbList.find((kb) => kb.kbId === item._id)),
[myKbList, kbList] [datasets, kbList]
); );
/* 点击删除 */ /* 点击删除 */
...@@ -167,7 +170,7 @@ const Settings = ({ appId }: { appId: string }) => { ...@@ -167,7 +170,7 @@ const Settings = ({ appId }: { appId: string }) => {
appModule2Form(); appModule2Form();
}, [appModule2Form]); }, [appModule2Form]);
useQuery(['initkb', appId], () => loadKbList()); useQuery(['loadAllDatasets'], loadAllDatasets);
const BoxStyles: BoxProps = { const BoxStyles: BoxProps = {
bg: 'myWhite.200', bg: 'myWhite.200',
...@@ -304,6 +307,23 @@ const Settings = ({ appId }: { appId: string }) => { ...@@ -304,6 +307,23 @@ const Settings = ({ appId }: { appId: string }) => {
</Button> </Button>
</Flex> </Flex>
{/* welcome */}
<Box mt={5} {...BoxStyles}>
<Flex alignItems={'center'}>
<Avatar src={'/imgs/module/userGuide.png'} w={'18px'} />
<Box mx={2}>对话开场白</Box>
<MyTooltip label={welcomeTextTip} forceShow>
<QuestionOutlineIcon />
</MyTooltip>
</Flex>
<Textarea
mt={2}
rows={5}
placeholder={welcomeTextTip}
borderColor={'myGray.100'}
{...register('guide.welcome.text')}
/>
</Box>
{/* variable */} {/* variable */}
<Box mt={2} {...BoxStyles}> <Box mt={2} {...BoxStyles}>
<Flex alignItems={'center'}> <Flex alignItems={'center'}>
...@@ -498,24 +518,6 @@ const Settings = ({ appId }: { appId: string }) => { ...@@ -498,24 +518,6 @@ const Settings = ({ appId }: { appId: string }) => {
</Grid> </Grid>
</Box> </Box>
{/* welcome */}
<Box mt={5} {...BoxStyles}>
<Flex alignItems={'center'}>
<Avatar src={'/imgs/module/userGuide.png'} w={'18px'} />
<Box mx={2}>对话开场白</Box>
<MyTooltip label={welcomeTextTip} forceShow>
<QuestionOutlineIcon />
</MyTooltip>
</Flex>
<Textarea
mt={2}
rows={5}
placeholder={welcomeTextTip}
borderColor={'myGray.100'}
{...register('guide.welcome.text')}
/>
</Box>
<ConfirmSaveModal /> <ConfirmSaveModal />
<ConfirmDelModal /> <ConfirmDelModal />
{settingAppInfo && ( {settingAppInfo && (
...@@ -548,7 +550,6 @@ const Settings = ({ appId }: { appId: string }) => { ...@@ -548,7 +550,6 @@ const Settings = ({ appId }: { appId: string }) => {
)} )}
{isOpenKbSelect && ( {isOpenKbSelect && (
<KBSelectModal <KBSelectModal
kbList={myKbList}
activeKbs={selectedKbList.map((item) => ({ activeKbs={selectedKbList.map((item) => ({
kbId: item._id, kbId: item._id,
vectorModel: item.vectorModel vectorModel: item.vectorModel
......
client/src/pages/kb/detail/components/DataCard.tsx
import React, { useCallback, useState, useRef } from 'react'; import React, { useCallback, useState, useRef, useMemo } from 'react';
import { Box, Card, IconButton, Flex, Button, Input, Grid } from '@chakra-ui/react'; import { Box, Card, IconButton, Flex, Button, Grid, Image } from '@chakra-ui/react';
import type { KbDataItemType } from '@/types/plugin'; import type { KbDataItemType } from '@/types/plugin';
import { usePagination } from '@/hooks/usePagination'; import { usePagination } from '@/hooks/usePagination';
import { import {
getKbDataList, getKbDataList,
getExportDataList, getExportDataList,
delOneKbDataByDataId, delOneKbDataByDataId,
getTrainingData getTrainingData,
getFileInfoById
} from '@/api/plugins/kb'; } from '@/api/plugins/kb';
import { DeleteIcon, RepeatIcon } from '@chakra-ui/icons'; import { DeleteIcon, RepeatIcon } from '@chakra-ui/icons';
import { fileDownload } from '@/utils/file'; import { fileDownload } from '@/utils/file';
...@@ -18,12 +19,19 @@ import { debounce } from 'lodash'; ...@@ -18,12 +19,19 @@ import { debounce } from 'lodash';
import { getErrText } from '@/utils/tools'; import { getErrText } from '@/utils/tools';
import { useConfirm } from '@/hooks/useConfirm'; import { useConfirm } from '@/hooks/useConfirm';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { useRouter } from 'next/router';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
import MyTooltip from '@/components/MyTooltip'; import MyTooltip from '@/components/MyTooltip';
import MyInput from '@/components/MyInput';
import { fileImgs } from '@/constants/common';
import { useRequest } from '@/hooks/useRequest';
import { feConfigs } from '@/store/static';
const DataCard = ({ kbId }: { kbId: string }) => { const DataCard = ({ kbId }: { kbId: string }) => {
const BoxRef = useRef<HTMLDivElement>(null); const BoxRef = useRef<HTMLDivElement>(null);
const lastSearch = useRef(''); const lastSearch = useRef('');
const router = useRouter();
const { fileId = '' } = router.query as { fileId: string };
const { t } = useTranslation(); const { t } = useTranslation();
const [searchText, setSearchText] = useState(''); const [searchText, setSearchText] = useState('');
const { toast } = useToast(); const { toast } = useToast();
...@@ -45,7 +53,8 @@ const DataCard = ({ kbId }: { kbId: string }) => { ...@@ -45,7 +53,8 @@ const DataCard = ({ kbId }: { kbId: string }) => {
pageSize: 24, pageSize: 24,
params: { params: {
kbId, kbId,
searchText searchText,
fileId
}, },
onChange() { onChange() {
if (BoxRef.current) { if (BoxRef.current) {
...@@ -72,33 +81,7 @@ const DataCard = ({ kbId }: { kbId: string }) => { ...@@ -72,33 +81,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
[getData, pageNum, refetchTrainingData] [getData, pageNum, refetchTrainingData]
); );
// get al data and export csv // get first page data
const { mutate: onclickExport, isLoading: isLoadingExport = false } = useMutation({
mutationFn: () => getExportDataList(kbId),
onSuccess(res) {
const text = Papa.unparse({
fields: ['question', 'answer', 'source'],
data: res
});
fileDownload({
text,
type: 'text/csv',
filename: 'data.csv'
});
toast({
title: '导出成功，下次导出需要半小时后',
status: 'success'
});
},
onError(err: any) {
toast({
title: getErrText(err, '导出异常'),
status: 'error'
});
console.log(err);
}
});
const getFirstData = useCallback( const getFirstData = useCallback(
debounce(() => { debounce(() => {
getData(1); getData(1);
...@@ -113,57 +96,100 @@ const DataCard = ({ kbId }: { kbId: string }) => { ...@@ -113,57 +96,100 @@ const DataCard = ({ kbId }: { kbId: string }) => {
enabled: qaListLen > 0 || vectorListLen > 0 enabled: qaListLen > 0 || vectorListLen > 0
}); });
// get file info
const { data: fileInfo } = useQuery(['getFileInfo', fileId], () => getFileInfoById(fileId));
const fileIcon = useMemo(
() =>
fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(fileInfo?.filename || ''))?.src,
[fileInfo?.filename]
);
// get al data and export csv
const { mutate: onclickExport, isLoading: isLoadingExport = false } = useRequest({
mutationFn: () => getExportDataList({ kbId, fileId }),
onSuccess(res) {
const text = Papa.unparse({
fields: ['question', 'answer', 'source'],
data: res
});
const filenameSplit = fileInfo?.filename?.split('.') || [];
const filename = filenameSplit?.length <= 1 ? 'data' : filenameSplit.slice(0, -1).join('.');
fileDownload({
text,
type: 'text/csv',
filename
});
},
successToast: `导出成功，下次导出需要 ${feConfigs.exportLimitMinutes} 分钟后`,
errorToast: '导出异常'
});
return ( return (
<Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}> <Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}>
<Flex justifyContent={'space-between'}> <Flex alignItems={'center'}>
<Box fontWeight={'bold'} fontSize={'lg'} mr={2}> <Flex
知识库数据: {total}组 className="textEllipsis"
</Box> flex={'1 0 0'}
mr={[3, 5]}
fontSize={['sm', 'md']}
alignItems={'center'}
>
<Image src={fileIcon} w={'16px'} mr={2} alt={''} />
{t(fileInfo?.filename || 'Filename')}
</Flex>
<Button
mr={2}
size={['sm', 'md']}
variant={'base'}
borderColor={'myBlue.600'}
color={'myBlue.600'}
isLoading={isLoadingExport || isLoading}
title={`${feConfigs} 分钟能导出 1 次`}
onClick={onclickExport}
>
{t('dataset.Export')}
</Button>
<Box> <Box>
<MyTooltip label={'刷新'}> <MyTooltip label={'刷新'}>
<IconButton <IconButton
icon={<RepeatIcon />} icon={<RepeatIcon />}
size={['sm', 'md']}
aria-label={'refresh'} aria-label={'refresh'}
variant={'base'} variant={'base'}
isLoading={isLoading} isLoading={isLoading}
mr={[2, 4]}
size={'sm'}
onClick={() => { onClick={() => {
getData(pageNum); getData(pageNum);
getTrainingData({ kbId, init: true }); getTrainingData({ kbId, init: true });
}} }}
/> />
</MyTooltip> </MyTooltip>
<Button
mr={2}
size={'sm'}
variant={'base'}
borderColor={'myBlue.600'}
color={'myBlue.600'}
isLoading={isLoadingExport || isLoading}
title={'半小时仅能导出1次'}
onClick={() => onclickExport()}
>
导出数据
</Button>
</Box> </Box>
</Flex> </Flex>
<Flex my={4}> <Flex my={3} alignItems={'center'}>
{qaListLen > 0 || vectorListLen > 0 ? ( <Box>
<Box fontSize={'xs'}> <Box as={'span'} fontSize={['md', 'lg']}>
{qaListLen > 0 ? `${qaListLen}条数据正在拆分，` : ''} {total}组
{vectorListLen > 0 ? `${vectorListLen}条数据正在生成索引，` : ''} </Box>
请耐心等待... <Box as={'span'}>
{(qaListLen > 0 || vectorListLen > 0) && (
<>
({qaListLen > 0 ? `${qaListLen}条数据正在拆分，` : ''}
{vectorListLen > 0 ? `${vectorListLen}条数据正在生成索引，` : ''}
请耐心等待... )
</>
)}
</Box> </Box>
) : ( </Box>
<Box fontSize={'xs'}>所有数据已就绪~</Box>
)}
<Box flex={1} mr={1} /> <Box flex={1} mr={1} />
<Input <MyInput
maxW={['60%', '300px']} leftIcon={
size={'sm'} <MyIcon name="searchLight" position={'absolute'} w={'14px'} color={'myGray.500'} />
value={searchText} }
w={['200px', '300px']}
placeholder="根据匹配知识，预期答案和来源进行搜索" placeholder="根据匹配知识，预期答案和来源进行搜索"
value={searchText}
onChange={(e) => { onChange={(e) => {
setSearchText(e.target.value); setSearchText(e.target.value);
getFirstData(); getFirstData();
...@@ -217,7 +243,7 @@ const DataCard = ({ kbId }: { kbId: string }) => { ...@@ -217,7 +243,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
</Box> </Box>
<Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}> <Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}>
<Box className={'textEllipsis'} flex={1} color={'myGray.500'}> <Box className={'textEllipsis'} flex={1} color={'myGray.500'}>
{item.source?.trim()} ID:{item.id}
</Box> </Box>
<IconButton <IconButton
className="delete" className="delete"
......
client/src/pages/kb/detail/components/FileCard.tsx 0 → 100644
import React, { useCallback, useState, useRef, useMemo } from 'react';
import {
Box,
Flex,
TableContainer,
Table,
Thead,
Tr,
Th,
Td,
Tbody,
Image
} from '@chakra-ui/react';
import { getKbFiles, deleteKbFileById } from '@/api/plugins/kb';
import { useQuery } from '@tanstack/react-query';
import { useToast } from '@/hooks/useToast';
import { debounce } from 'lodash';
import { formatFileSize } from '@/utils/tools';
import { useConfirm } from '@/hooks/useConfirm';
import { useTranslation } from 'react-i18next';
import MyIcon from '@/components/Icon';
import MyInput from '@/components/MyInput';
import dayjs from 'dayjs';
import { fileImgs } from '@/constants/common';
import { useRequest } from '@/hooks/useRequest';
import { useLoading } from '@/hooks/useLoading';
import { FileStatusEnum } from '@/constants/kb';
import { useRouter } from 'next/router';
const FileCard = ({ kbId }: { kbId: string }) => {
const BoxRef = useRef<HTMLDivElement>(null);
const lastSearch = useRef('');
const router = useRouter();
const { t } = useTranslation();
const [searchText, setSearchText] = useState('');
const { Loading } = useLoading();
const { openConfirm, ConfirmModal } = useConfirm({
content: t('kb.Confirm to delete the file')
});
const {
data: files = [],
refetch,
isInitialLoading
} = useQuery(['getFiles', kbId], () => getKbFiles({ kbId, searchText }), {
refetchInterval: 6000,
refetchOnWindowFocus: true
});
const debounceRefetch = useCallback(
debounce(() => {
refetch();
lastSearch.current = searchText;
}, 300),
[]
);
const formatFiles = useMemo(
() =>
files.map((file) => ({
...file,
icon: fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(file.filename))?.src
})),
[files]
);
const totalDataLength = useMemo(
() => files.reduce((sum, item) => sum + item.chunkLength, 0),
[files]
);
const { mutate: onDeleteFile, isLoading } = useRequest({
mutationFn: (fileId: string) =>
deleteKbFileById({
fileId,
kbId
}),
onSuccess() {
refetch();
},
successToast: t('common.Delete Success'),
errorToast: t('common.Delete Failed')
});
const statusMap = {
[FileStatusEnum.embedding]: {
color: 'myGray.500',
text: t('file.Embedding')
},
[FileStatusEnum.ready]: {
color: 'green.500',
text: t('file.Ready')
}
};
return (
<Box ref={BoxRef} position={'relative'} py={[1, 5]} h={'100%'} overflow={'overlay'}>
<Flex justifyContent={'space-between'} px={5}>
<Box fontWeight={'bold'} fontSize={'lg'} mr={2}>
{t('kb.Files', { total: files.length })}
</Box>
<Flex alignItems={'center'}>
<MyInput
leftIcon={
<MyIcon name="searchLight" position={'absolute'} w={'14px'} color={'myGray.500'} />
}
w={['100%', '200px']}
placeholder={t('common.Search') || ''}
value={searchText}
onChange={(e) => {
setSearchText(e.target.value);
debounceRefetch();
}}
onBlur={() => {
if (searchText === lastSearch.current) return;
refetch();
}}
onKeyDown={(e) => {
if (searchText === lastSearch.current) return;
if (e.key === 'Enter') {
refetch();
}
}}
/>
</Flex>
</Flex>
<TableContainer mt={[0, 3]}>
<Table variant={'simple'} fontSize={'sm'}>
<Thead>
<Tr>
<Th>{t('kb.Filename')}</Th>
<Th>
{t('kb.Chunk Length')}({totalDataLength})
</Th>
<Th>{t('kb.Upload Time')}</Th>
<Th>{t('kb.File Size')}</Th>
<Th>{t('common.Status')}</Th>
<Th />
</Tr>
</Thead>
<Tbody>
{formatFiles.map((file) => (
<Tr
key={file.id}
_hover={{ bg: 'myWhite.600' }}
cursor={'pointer'}
title={'点击查看数据详情'}
onClick={() =>
router.push({
query: {
kbId,
fileId: file.id,
currentTab: 'dataCard'
}
})
}
>
<Td>
<Flex alignItems={'center'}>
<Image src={file.icon} w={'16px'} mr={2} alt={''} />
<Box maxW={['300px', '400px']} className="textEllipsis">
{t(file.filename)}
</Box>
</Flex>
</Td>
<Td fontSize={'md'} fontWeight={'bold'}>
{file.chunkLength}
</Td>
<Td>{dayjs(file.uploadTime).format('YYYY/MM/DD HH:mm')}</Td>
<Td>{formatFileSize(file.size)}</Td>
<Td
display={'flex'}
alignItems={'center'}
_before={{
content: '""',
w: '10px',
h: '10px',
mr: 2,
borderRadius: 'lg',
bg: statusMap[file.status].color
}}
>
{statusMap[file.status].text}
</Td>
<Td onClick={(e) => e.stopPropagation()}>
<MyIcon
name={'delete'}
w={'14px'}
_hover={{ color: 'red.600' }}
onClick={() =>
openConfirm(() => {
onDeleteFile(file.id);
})()
}
/>
</Td>
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
<ConfirmModal />
<Loading loading={isInitialLoading || isLoading} />
</Box>
);
};
export default React.memo(FileCard);
client/src/pages/kb/detail/components/Import/Chunk.tsx
...@@ -26,12 +26,12 @@ import MyTooltip from '@/components/MyTooltip'; ...@@ -26,12 +26,12 @@ import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import FileSelect, { type FileItemType } from './FileSelect'; import FileSelect, { type FileItemType } from './FileSelect';
import { useUserStore } from '@/store/user'; import { useDatasetStore } from '@/store/dataset';
const fileExtension = '.txt, .doc, .docx, .pdf, .md'; const fileExtension = '.txt, .doc, .docx, .pdf, .md';
const ChunkImport = ({ kbId }: { kbId: string }) => { const ChunkImport = ({ kbId }: { kbId: string }) => {
const { kbDetail } = useUserStore(); const { kbDetail } = useDatasetStore();
const vectorModel = kbDetail.vectorModel; const vectorModel = kbDetail.vectorModel;
const unitPrice = vectorModel?.price || 0.2; const unitPrice = vectorModel?.price || 0.2;
...@@ -86,7 +86,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => { ...@@ -86,7 +86,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
router.replace({ router.replace({
query: { query: {
kbId, kbId,
currentTab: 'data' currentTab: 'dataset'
} }
}); });
}, },
...@@ -106,12 +106,15 @@ const ChunkImport = ({ kbId }: { kbId: string }) => { ...@@ -106,12 +106,15 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
text: file.text, text: file.text,
maxLen: chunkLen maxLen: chunkLen
}); });
return { return {
...file, ...file,
tokens: splitRes.tokens, tokens: splitRes.tokens,
chunks: file.chunks.map((chunk, i) => ({ chunks: splitRes.chunks.map((chunk) => ({
...chunk, a: '',
q: splitRes.chunks[i] source: file.filename,
file_id: file.id,
q: chunk
})) }))
}; };
}) })
......
client/src/pages/kb/detail/components/Import/CreateFileModal.tsx
...@@ -20,7 +20,7 @@ const CreateFileModal = ({ ...@@ -20,7 +20,7 @@ const CreateFileModal = ({
}); });
return ( return (
<MyModal title={t('file.Create File')} isOpen onClose={onClose} w={'600px'} top={'15vh'}> <MyModal title={t('file.Create File')} isOpen onClose={() => {}} w={'600px'} top={'15vh'}>
<ModalBody> <ModalBody>
<Box mb={1} fontSize={'sm'}> <Box mb={1} fontSize={'sm'}>
文件名 文件名
......
client/src/pages/kb/detail/components/Import/Csv.tsx
...@@ -10,12 +10,12 @@ import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete'; ...@@ -10,12 +10,12 @@ import DeleteIcon, { hoverDeleteStyles } from '@/components/Icon/delete';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import FileSelect, { type FileItemType } from './FileSelect'; import FileSelect, { type FileItemType } from './FileSelect';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { useUserStore } from '@/store/user'; import { useDatasetStore } from '@/store/dataset';
const fileExtension = '.csv'; const fileExtension = '.csv';
const CsvImport = ({ kbId }: { kbId: string }) => { const CsvImport = ({ kbId }: { kbId: string }) => {
const { kbDetail } = useUserStore(); const { kbDetail } = useDatasetStore();
const maxToken = kbDetail.vectorModel?.maxToken || 2000; const maxToken = kbDetail.vectorModel?.maxToken || 2000;
const theme = useTheme(); const theme = useTheme();
...@@ -42,7 +42,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -42,7 +42,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
.flat() .flat()
.filter((item) => item?.q); .filter((item) => item?.q);
const filterChunks = chunks.filter((item) => item.q.length < maxToken); const filterChunks = chunks.filter((item) => item.q.length < maxToken * 1.5);
if (filterChunks.length !== chunks.length) { if (filterChunks.length !== chunks.length) {
toast({ toast({
...@@ -73,7 +73,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -73,7 +73,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
router.replace({ router.replace({
query: { query: {
kbId, kbId,
currentTab: 'data' currentTab: 'dataset'
} }
}); });
}, },
......
client/src/pages/kb/detail/components/Import/FileSelect.tsx
...@@ -19,13 +19,13 @@ import dynamic from 'next/dynamic'; ...@@ -19,13 +19,13 @@ import dynamic from 'next/dynamic';
import MyTooltip from '@/components/MyTooltip'; import MyTooltip from '@/components/MyTooltip';
import { FetchResultItem, DatasetItemType } from '@/types/plugin'; import { FetchResultItem, DatasetItemType } from '@/types/plugin';
import { getErrText } from '@/utils/tools'; import { getErrText } from '@/utils/tools';
import { useUserStore } from '@/store/user'; import { useDatasetStore } from '@/store/dataset';
const UrlFetchModal = dynamic(() => import('./UrlFetchModal')); const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
const CreateFileModal = dynamic(() => import('./CreateFileModal')); const CreateFileModal = dynamic(() => import('./CreateFileModal'));
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12); const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
const csvTemplate = `question,answer,source\n"什么是 laf","laf 是一个云函数开发平台……","laf git doc"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……","sealos git doc"`; const csvTemplate = `question,answer\n"什么是 laf","laf 是一个云函数开发平台……"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`;
export type FileItemType = { export type FileItemType = {
id: string; id: string;
...@@ -55,7 +55,7 @@ const FileSelect = ({ ...@@ -55,7 +55,7 @@ const FileSelect = ({
showCreateFile = true, showCreateFile = true,
...props ...props
}: Props) => { }: Props) => {
const { kbDetail } = useUserStore(); const { kbDetail } = useDatasetStore();
const { Loading: FileSelectLoading } = useLoading(); const { Loading: FileSelectLoading } = useLoading();
const { t } = useTranslation(); const { t } = useTranslation();
...@@ -129,7 +129,7 @@ const FileSelect = ({ ...@@ -129,7 +129,7 @@ const FileSelect = ({
maxLen: chunkLen maxLen: chunkLen
}); });
const fileItem: FileItemType = { const fileItem: FileItemType = {
id: nanoid(), id: filesId[0],
filename: file.name, filename: file.name,
icon, icon,
text, text,
......
client/src/pages/kb/detail/components/Import/Manual.tsx
...@@ -6,14 +6,14 @@ import { useRequest } from '@/hooks/useRequest'; ...@@ -6,14 +6,14 @@ import { useRequest } from '@/hooks/useRequest';
import { getErrText } from '@/utils/tools'; import { getErrText } from '@/utils/tools';
import { postKbDataFromList } from '@/api/plugins/kb'; import { postKbDataFromList } from '@/api/plugins/kb';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import { useUserStore } from '@/store/user';
import MyTooltip from '@/components/MyTooltip'; import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useDatasetStore } from '@/store/dataset';
type ManualFormType = { q: string; a: string }; type ManualFormType = { q: string; a: string };
const ManualImport = ({ kbId }: { kbId: string }) => { const ManualImport = ({ kbId }: { kbId: string }) => {
const { kbDetail } = useUserStore(); const { kbDetail } = useDatasetStore();
const maxToken = kbDetail.vectorModel?.maxToken || 2000; const maxToken = kbDetail.vectorModel?.maxToken || 2000;
const { register, handleSubmit, reset } = useForm({ const { register, handleSubmit, reset } = useForm({
......
client/src/pages/kb/detail/components/Import/QA.tsx
...@@ -74,7 +74,7 @@ const QAImport = ({ kbId }: { kbId: string }) => { ...@@ -74,7 +74,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
router.replace({ router.replace({
query: { query: {
kbId, kbId,
currentTab: 'data' currentTab: 'dataset'
} }
}); });
}, },
...@@ -97,9 +97,11 @@ const QAImport = ({ kbId }: { kbId: string }) => { ...@@ -97,9 +97,11 @@ const QAImport = ({ kbId }: { kbId: string }) => {
return { return {
...file, ...file,
tokens: splitRes.tokens, tokens: splitRes.tokens,
chunks: file.chunks.map((chunk, i) => ({ chunks: splitRes.chunks.map((chunk) => ({
...chunk, a: '',
q: splitRes.chunks[i] source: file.filename,
file_id: file.id,
q: chunk
})) }))
}; };
}) })
......
client/src/pages/kb/detail/components/Info.tsx
...@@ -12,7 +12,7 @@ import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons'; ...@@ -12,7 +12,7 @@ import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons';
import { delKbById, putKbById } from '@/api/plugins/kb'; import { delKbById, putKbById } from '@/api/plugins/kb';
import { useSelectFile } from '@/hooks/useSelectFile'; import { useSelectFile } from '@/hooks/useSelectFile';
import { useToast } from '@/hooks/useToast'; import { useToast } from '@/hooks/useToast';
import { useUserStore } from '@/store/user'; import { useDatasetStore } from '@/store/dataset';
import { useConfirm } from '@/hooks/useConfirm'; import { useConfirm } from '@/hooks/useConfirm';
import { UseFormReturn } from 'react-hook-form'; import { UseFormReturn } from 'react-hook-form';
import { compressImg } from '@/utils/file'; import { compressImg } from '@/utils/file';
...@@ -47,7 +47,7 @@ const Info = ( ...@@ -47,7 +47,7 @@ const Info = (
multiple: false multiple: false
}); });
const { kbDetail, getKbDetail, loadKbList, myKbList } = useUserStore(); const { kbDetail, getKbDetail, loadKbList } = useDatasetStore();
/* 点击删除 */ /* 点击删除 */
const onclickDelKb = useCallback(async () => { const onclickDelKb = useCallback(async () => {
......
client/src/pages/kb/detail/components/InputDataModal.tsx
...@@ -9,10 +9,10 @@ import MyIcon from '@/components/Icon'; ...@@ -9,10 +9,10 @@ import MyIcon from '@/components/Icon';
import MyModal from '@/components/MyModal'; import MyModal from '@/components/MyModal';
import MyTooltip from '@/components/MyTooltip'; import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useUserStore } from '@/store/user';
import { useQuery } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query';
import { DatasetItemType } from '@/types/plugin'; import { DatasetItemType } from '@/types/plugin';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { useDatasetStore } from '@/store/dataset';
export type FormData = { dataId?: string } & DatasetItemType; export type FormData = { dataId?: string } & DatasetItemType;
...@@ -36,7 +36,7 @@ const InputDataModal = ({ ...@@ -36,7 +36,7 @@ const InputDataModal = ({
const [loading, setLoading] = useState(false); const [loading, setLoading] = useState(false);
const { toast } = useToast(); const { toast } = useToast();
const { kbDetail, getKbDetail } = useUserStore(); const { kbDetail, getKbDetail } = useDatasetStore();
const { getValues, register, handleSubmit, reset } = useForm<FormData>({ const { getValues, register, handleSubmit, reset } = useForm<FormData>({
defaultValues defaultValues
...@@ -261,13 +261,11 @@ export function RawFileText({ fileId, filename = '', ...props }: RawFileTextProp ...@@ -261,13 +261,11 @@ export function RawFileText({ fileId, filename = '', ...props }: RawFileTextProp
<Box <Box
color={'myGray.600'} color={'myGray.600'}
display={'inline-block'} display={'inline-block'}
whiteSpace={'nowrap'}
{...(!!fileId {...(!!fileId
? { ? {
cursor: 'pointer', cursor: 'pointer',
textDecoration: ['underline', 'none'], textDecoration: 'underline',
_hover: {
textDecoration: 'underline'
},
onClick: async () => { onClick: async () => {
try { try {
const url = await getFileViewUrl(fileId); const url = await getFileViewUrl(fileId);
......
client/src/pages/kb/detail/components/Test.tsx
import React, { useEffect, useMemo, useState } from 'react'; import React, { useEffect, useMemo, useState } from 'react';
import { Box, Textarea, Button, Flex, useTheme, Grid, Progress } from '@chakra-ui/react'; import { Box, Textarea, Button, Flex, useTheme, Grid, Progress } from '@chakra-ui/react';
import { useKbStore } from '@/store/kb'; import { useDatasetStore } from '@/store/dataset';
import type { KbTestItemType } from '@/types/plugin'; import type { KbTestItemType } from '@/types/plugin';
import { searchText, getKbDataItemById } from '@/api/plugins/kb'; import { searchText, getKbDataItemById } from '@/api/plugins/kb';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
...@@ -13,16 +13,14 @@ import { useToast } from '@/hooks/useToast'; ...@@ -13,16 +13,14 @@ import { useToast } from '@/hooks/useToast';
import { customAlphabet } from 'nanoid'; import { customAlphabet } from 'nanoid';
import MyTooltip from '@/components/MyTooltip'; import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useUserStore } from '@/store/user';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12); const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
const Test = ({ kbId }: { kbId: string }) => { const Test = ({ kbId }: { kbId: string }) => {
const { kbDetail } = useUserStore();
const theme = useTheme(); const theme = useTheme();
const { toast } = useToast(); const { toast } = useToast();
const { setLoading } = useGlobalStore(); const { setLoading } = useGlobalStore();
const { kbTestList, pushKbTestItem, delKbTestItemById, updateKbItemById } = useKbStore(); const { kbDetail, kbTestList, pushKbTestItem, delKbTestItemById, updateKbItemById } =
useDatasetStore();
const [inputText, setInputText] = useState(''); const [inputText, setInputText] = useState('');
const [kbTestItem, setKbTestItem] = useState<KbTestItemType>(); const [kbTestItem, setKbTestItem] = useState<KbTestItemType>();
const [editData, setEditData] = useState<FormData>(); const [editData, setEditData] = useState<FormData>();
......
client/src/pages/kb/detail/index.tsx
import React, { useCallback, useRef } from 'react'; import React, { useCallback, useEffect, useRef } from 'react';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { Box, Flex, IconButton, useTheme } from '@chakra-ui/react'; import { Box, Flex, IconButton, useTheme } from '@chakra-ui/react';
import { useToast } from '@/hooks/useToast'; import { useToast } from '@/hooks/useToast';
import { useForm } from 'react-hook-form'; import { useForm } from 'react-hook-form';
import { useQuery } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query';
import { useUserStore } from '@/store/user';
import { KbItemType } from '@/types/plugin'; import { KbItemType } from '@/types/plugin';
import { getErrText } from '@/utils/tools'; import { getErrText } from '@/utils/tools';
import { useGlobalStore } from '@/store/global'; import { useGlobalStore } from '@/store/global';
import { type ComponentRef } from './components/Info'; import { type ComponentRef } from './components/Info';
import Tabs from '@/components/Tabs'; import Tabs from '@/components/Tabs';
import dynamic from 'next/dynamic'; import dynamic from 'next/dynamic';
import DataCard from './components/DataCard';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
import SideTabs from '@/components/SideTabs'; import SideTabs from '@/components/SideTabs';
import PageContainer from '@/components/PageContainer'; import PageContainer from '@/components/PageContainer';
...@@ -19,11 +17,17 @@ import Avatar from '@/components/Avatar'; ...@@ -19,11 +17,17 @@ import Avatar from '@/components/Avatar';
import Info from './components/Info'; import Info from './components/Info';
import { serviceSideProps } from '@/utils/i18n'; import { serviceSideProps } from '@/utils/i18n';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { getTrainingQueueLen } from '@/api/plugins/kb'; import { delEmptyFiles, getTrainingQueueLen } from '@/api/plugins/kb';
import MyTooltip from '@/components/MyTooltip'; import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { feConfigs } from '@/store/static'; import { feConfigs } from '@/store/static';
import Script from 'next/script';
import FileCard from './components/FileCard';
import { useDatasetStore } from '@/store/dataset';
const DataCard = dynamic(() => import('./components/DataCard'), {
ssr: false
});
const ImportData = dynamic(() => import('./components/Import'), { const ImportData = dynamic(() => import('./components/Import'), {
ssr: false ssr: false
}); });
...@@ -32,7 +36,8 @@ const Test = dynamic(() => import('./components/Test'), { ...@@ -32,7 +36,8 @@ const Test = dynamic(() => import('./components/Test'), {
}); });
enum TabEnum { enum TabEnum {
data = 'data', dataCard = 'dataCard',
dataset = 'dataset',
import = 'import', import = 'import',
test = 'test', test = 'test',
info = 'info' info = 'info'
...@@ -45,10 +50,10 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` } ...@@ -45,10 +50,10 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` }
const { toast } = useToast(); const { toast } = useToast();
const router = useRouter(); const router = useRouter();
const { isPc } = useGlobalStore(); const { isPc } = useGlobalStore();
const { kbDetail, getKbDetail } = useUserStore(); const { kbDetail, getKbDetail } = useDatasetStore();
const tabList = useRef([ const tabList = useRef([
{ label: '数据集', id: TabEnum.data, icon: 'overviewLight' }, { label: '数据集', id: TabEnum.dataset, icon: 'overviewLight' },
{ label: '导入数据', id: TabEnum.import, icon: 'importLight' }, { label: '导入数据', id: TabEnum.import, icon: 'importLight' },
{ label: '搜索测试', id: TabEnum.test, icon: 'kbTest' }, { label: '搜索测试', id: TabEnum.test, icon: 'kbTest' },
{ label: '配置', id: TabEnum.info, icon: 'settingLight' } { label: '配置', id: TabEnum.info, icon: 'settingLight' }
...@@ -85,105 +90,117 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` } ...@@ -85,105 +90,117 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` }
}); });
const { data: trainingQueueLen = 0 } = useQuery(['getTrainingQueueLen'], getTrainingQueueLen, { const { data: trainingQueueLen = 0 } = useQuery(['getTrainingQueueLen'], getTrainingQueueLen, {
refetchInterval: 5000 refetchInterval: 10000
}); });
useEffect(() => {
return () => {
try {
delEmptyFiles(kbId);
} catch (error) {}
};
}, [kbId]);
return ( return (
<PageContainer> <>
<Box display={['block', 'flex']} h={'100%'} pt={[4, 0]}> <Script src="/js/pdf.js" strategy="lazyOnload"></Script>
{isPc ? ( <PageContainer>
<Flex <Box display={['block', 'flex']} h={'100%'} pt={[4, 0]}>
flexDirection={'column'} {isPc ? (
p={4}
h={'100%'}
flex={'0 0 200px'}
borderRight={theme.borders.base}
>
<Flex mb={4} alignItems={'center'}>
<Avatar src={kbDetail.avatar} w={'34px'} borderRadius={'lg'} />
<Box ml={2} fontWeight={'bold'}>
{kbDetail.name}
</Box>
</Flex>
<SideTabs
flex={1}
mx={'auto'}
mt={2}
w={'100%'}
list={tabList.current}
activeId={currentTab}
onChange={(e: any) => {
setCurrentTab(e);
}}
/>
<Box textAlign={'center'}>
<Flex justifyContent={'center'} alignItems={'center'}>
<MyIcon mr={1} name="overviewLight" w={'16px'} color={'green.500'} />
<Box>{t('dataset.System Data Queue')}</Box>
<MyTooltip
label={t('dataset.Queue Desc', { title: feConfigs?.systemTitle })}
placement={'top'}
>
<QuestionOutlineIcon ml={1} w={'16px'} />
</MyTooltip>
</Flex>
<Box mt={1} fontWeight={'bold'}>
{trainingQueueLen}
</Box>
</Box>
<Flex <Flex
alignItems={'center'} flexDirection={'column'}
cursor={'pointer'} p={4}
py={2} h={'100%'}
px={3} flex={'0 0 200px'}
borderRadius={'md'} borderRight={theme.borders.base}
_hover={{ bg: 'myGray.100' }}
onClick={() => router.replace('/kb/list')}
> >
<IconButton <Flex mb={4} alignItems={'center'}>
mr={3} <Avatar src={kbDetail.avatar} w={'34px'} borderRadius={'lg'} />
icon={<MyIcon name={'backFill'} w={'18px'} color={'myBlue.600'} />} <Box ml={2} fontWeight={'bold'}>
bg={'white'} {kbDetail.name}
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'} </Box>
h={'28px'} </Flex>
size={'sm'} <SideTabs
borderRadius={'50%'} flex={1}
aria-label={''} mx={'auto'}
mt={2}
w={'100%'}
list={tabList.current}
activeId={currentTab}
onChange={(e: any) => {
setCurrentTab(e);
}}
/> />
全部知识库 <Box textAlign={'center'}>
<Flex justifyContent={'center'} alignItems={'center'}>
<MyIcon mr={1} name="overviewLight" w={'16px'} color={'green.500'} />
<Box>{t('dataset.System Data Queue')}</Box>
<MyTooltip
label={t('dataset.Queue Desc', { title: feConfigs?.systemTitle })}
placement={'top'}
>
<QuestionOutlineIcon ml={1} w={'16px'} />
</MyTooltip>
</Flex>
<Box mt={1} fontWeight={'bold'}>
{trainingQueueLen}
</Box>
</Box>
<Flex
alignItems={'center'}
cursor={'pointer'}
py={2}
px={3}
borderRadius={'md'}
_hover={{ bg: 'myGray.100' }}
onClick={() => router.replace('/kb/list')}
>
<IconButton
mr={3}
icon={<MyIcon name={'backFill'} w={'18px'} color={'myBlue.600'} />}
bg={'white'}
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
h={'28px'}
size={'sm'}
borderRadius={'50%'}
aria-label={''}
/>
全部知识库
</Flex>
</Flex> </Flex>
</Flex> ) : (
) : ( <Box mb={3}>
<Box mb={3}> <Tabs
<Tabs m={'auto'}
m={'auto'} w={'260px'}
w={'260px'} size={isPc ? 'md' : 'sm'}
size={isPc ? 'md' : 'sm'} list={tabList.current.map((item) => ({
list={tabList.current.map((item) => ({ id: item.id,
id: item.id, label: item.label
label: item.label }))}
}))} activeId={currentTab}
activeId={currentTab} onChange={(e: any) => setCurrentTab(e)}
onChange={(e: any) => setCurrentTab(e)} />
/> </Box>
</Box> )}
)}
{!!kbDetail._id && ( {!!kbDetail._id && (
<Box flex={'1 0 0'} h={'100%'} pb={[4, 0]}> <Box flex={'1 0 0'} h={'100%'} pb={[4, 0]}>
{currentTab === TabEnum.data && <DataCard kbId={kbId} />} {currentTab === TabEnum.dataset && <FileCard kbId={kbId} />}
{currentTab === TabEnum.import && <ImportData kbId={kbId} />} {currentTab === TabEnum.dataCard && <DataCard kbId={kbId} />}
{currentTab === TabEnum.test && <Test kbId={kbId} />} {currentTab === TabEnum.import && <ImportData kbId={kbId} />}
{currentTab === TabEnum.info && <Info ref={InfoRef} kbId={kbId} form={form} />} {currentTab === TabEnum.test && <Test kbId={kbId} />}
</Box> {currentTab === TabEnum.info && <Info ref={InfoRef} kbId={kbId} form={form} />}
)} </Box>
</Box> )}
</PageContainer> </Box>
</PageContainer>
</>
); );
}; };
export async function getServerSideProps(context: any) { export async function getServerSideProps(context: any) {
const currentTab = context?.query?.currentTab || TabEnum.data; const currentTab = context?.query?.currentTab || TabEnum.dataset;
const kbId = context?.query?.kbId; const kbId = context?.query?.kbId;
return { return {
......
client/src/pages/kb/list/component/CreateModal.tsx
...@@ -18,7 +18,7 @@ import MySelect from '@/components/Select'; ...@@ -18,7 +18,7 @@ import MySelect from '@/components/Select';
import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { QuestionOutlineIcon } from '@chakra-ui/icons';
import Tag from '@/components/Tag'; import Tag from '@/components/Tag';
const CreateModal = ({ onClose }: { onClose: () => void }) => { const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: string }) => {
const [refresh, setRefresh] = useState(false); const [refresh, setRefresh] = useState(false);
const { toast } = useToast(); const { toast } = useToast();
const router = useRouter(); const router = useRouter();
...@@ -28,7 +28,9 @@ const CreateModal = ({ onClose }: { onClose: () => void }) => { ...@@ -28,7 +28,9 @@ const CreateModal = ({ onClose }: { onClose: () => void }) => {
avatar: '/icon/logo.svg', avatar: '/icon/logo.svg',
name: '', name: '',
tags: [], tags: [],
vectorModel: vectorModelList[0].model vectorModel: vectorModelList[0].model,
type: 'dataset',
parentId
} }
}); });
const InputRef = useRef<HTMLInputElement>(null); const InputRef = useRef<HTMLInputElement>(null);
......
client/src/pages/kb/list/component/EditFolderModal.tsx 0 → 100644
import React, { useMemo, useRef } from 'react';
import { ModalFooter, ModalBody, Input, Button } from '@chakra-ui/react';
import MyModal from '@/components/MyModal';
import { useTranslation } from 'react-i18next';
import { useRequest } from '@/hooks/useRequest';
import { postCreateKb, putKbById } from '@/api/plugins/kb';
import { FolderAvatarSrc, KbTypeEnum } from '@/constants/kb';
const EditFolderModal = ({
onClose,
onSuccess,
id,
parentId,
name
}: {
onClose: () => void;
onSuccess: () => void;
id?: string;
parentId?: string;
name?: string;
}) => {
const { t } = useTranslation();
const inputRef = useRef<HTMLInputElement>(null);
const typeMap = useMemo(
() =>
id
? {
title: t('kb.Edit Folder')
}
: {
title: t('kb.Create Folder')
},
[id, t]
);
const { mutate: onSave, isLoading } = useRequest({
mutationFn: () => {
const val = inputRef.current?.value;
if (!val) return Promise.resolve('');
if (id) {
return putKbById({
id,
name: val
});
}
return postCreateKb({
parentId,
name: val,
type: KbTypeEnum.folder,
avatar: FolderAvatarSrc,
tags: []
});
},
onSuccess: (res) => {
if (!res) return;
onSuccess();
onClose();
}
});
return (
<MyModal isOpen onClose={onClose} title={typeMap.title}>
<ModalBody>
<Input
ref={inputRef}
defaultValue={name}
placeholder={t('kb.Folder Name') || ''}
autoFocus
maxLength={20}
/>
</ModalBody>
<ModalFooter>
<Button mr={3} variant={'base'} onClick={onClose}>
{t('common.Cancel')}
</Button>
<Button isLoading={isLoading} onClick={onSave}>
{t('Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};
export default EditFolderModal;
client/src/pages/kb/list/index.tsx
import React, { useCallback } from 'react'; import React, { useCallback, useMemo, useRef, useState } from 'react';
import { import {
Box, Box,
Card,
Flex, Flex,
Grid, Grid,
useTheme, useTheme,
Button, useDisclosure,
Card,
IconButton, IconButton,
useDisclosure MenuButton,
Image
} from '@chakra-ui/react'; } from '@chakra-ui/react';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { useUserStore } from '@/store/user'; import { useDatasetStore } from '@/store/dataset';
import PageContainer from '@/components/PageContainer'; import PageContainer from '@/components/PageContainer';
import { useConfirm } from '@/hooks/useConfirm'; import { useConfirm } from '@/hooks/useConfirm';
import { AddIcon } from '@chakra-ui/icons'; import { AddIcon } from '@chakra-ui/icons';
import { useQuery } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query';
import { useToast } from '@/hooks/useToast'; import { useToast } from '@/hooks/useToast';
import { delKbById } from '@/api/plugins/kb'; import { delKbById, getKbPaths } from '@/api/plugins/kb';
import { useTranslation } from 'react-i18next';
import Avatar from '@/components/Avatar'; import Avatar from '@/components/Avatar';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
import Tag from '@/components/Tag';
import { serviceSideProps } from '@/utils/i18n'; import { serviceSideProps } from '@/utils/i18n';
import dynamic from 'next/dynamic'; import dynamic from 'next/dynamic';
import { FolderAvatarSrc, KbTypeEnum } from '@/constants/kb';
import Tag from '@/components/Tag';
import MyMenu from '@/components/MyMenu';
import { useRequest } from '@/hooks/useRequest';
import { useGlobalStore } from '@/store/global';
const CreateModal = dynamic(() => import('./component/CreateModal'), { ssr: false }); const CreateModal = dynamic(() => import('./component/CreateModal'), { ssr: false });
const EditFolderModal = dynamic(() => import('./component/EditFolderModal'), { ssr: false });
const Kb = () => { const Kb = () => {
const { t } = useTranslation();
const theme = useTheme(); const theme = useTheme();
const router = useRouter(); const router = useRouter();
const { parentId } = router.query as { parentId: string };
const { toast } = useToast(); const { toast } = useToast();
const { setLoading } = useGlobalStore();
const DeleteTipsMap = useRef({
[KbTypeEnum.folder]: t('kb.deleteFolderTips'),
[KbTypeEnum.dataset]: t('kb.deleteDatasetTips')
});
const { openConfirm, ConfirmModal } = useConfirm({ const { openConfirm, ConfirmModal } = useConfirm({
title: '删除提示', title: t('common.Delete Warning'),
content: '确认删除该知识库？知识库相关的文件、记录将永久删除，无法恢复！' content: ''
}); });
const { myKbList, loadKbList, setKbList } = useUserStore(); const { myKbList, loadKbList, setKbList } = useDatasetStore();
const { const {
isOpen: isOpenCreateModal, isOpen: isOpenCreateModal,
onOpen: onOpenCreateModal, onOpen: onOpenCreateModal,
onClose: onCloseCreateModal onClose: onCloseCreateModal
} = useDisclosure(); } = useDisclosure();
const [editFolderData, setEditFolderData] = useState<{
const { refetch } = useQuery(['loadKbList'], () => loadKbList()); id?: string;
name?: string;
}>();
/* 点击删除 */ /* 点击删除 */
const onclickDelKb = useCallback( const { mutate: onclickDelKb } = useRequest({
async (id: string) => { mutationFn: async (id: string) => {
try { setLoading(true);
delKbById(id); await delKbById(id);
toast({ return id;
title: '删除成功', },
status: 'success' onSuccess(id: string) {
}); setKbList(myKbList.filter((item) => item._id !== id));
setKbList(myKbList.filter((item) => item._id !== id)); },
} catch (err: any) { onSettled() {
toast({ setLoading(false);
title: err?.message || '删除失败',
status: 'error'
});
}
}, },
[toast, setKbList, myKbList] successToast: t('common.Delete Success'),
errorToast: t('kb.Delete Dataset Error')
});
const { data, refetch } = useQuery(['loadKbList', parentId], () => {
return Promise.all([loadKbList(parentId), getKbPaths(parentId)]);
});
const paths = useMemo(
() => [
{
parentId: '',
parentName: t('kb.My Dataset')
},
...(data?.[1] || [])
],
[data, t]
); );
return ( return (
<PageContainer> <PageContainer>
<Flex pt={3} px={5} alignItems={'center'}> <Flex pt={3} px={5} alignItems={'center'}>
<Box flex={1} className="textlg" letterSpacing={1} fontSize={'24px'} fontWeight={'bold'}> {/* url path */}
我的知识库 {!!parentId ? (
</Box> <Flex flex={1}>
<Button leftIcon={<AddIcon />} variant={'base'} onClick={onOpenCreateModal}> {paths.map((item, i) => (
新建 <Flex key={item.parentId} mr={2} alignItems={'center'}>
</Button> <Box
fontSize={'lg'}
px={2}
py={1}
borderRadius={'md'}
{...(i === paths.length - 1
? {
cursor: 'default'
}
: {
cursor: 'pointer',
_hover: {
bg: 'myGray.100'
},
onClick: () => {
router.push({
query: {
parentId: item.parentId
}
});
}
})}
>
{item.parentName}
</Box>
{i !== paths.length - 1 && <MyIcon name={'rightArrowLight'} color={'myGray.500'} />}
</Flex>
))}
</Flex>
) : (
<Box flex={1} className="textlg" letterSpacing={1} fontSize={'24px'} fontWeight={'bold'}>
我的知识库
</Box>
)}
<MyMenu
offset={[-30, 10]}
width={120}
Button={
<MenuButton
_hover={{
color: 'myBlue.600'
}}
>
<Flex
alignItems={'center'}
border={theme.borders.base}
px={5}
py={2}
borderRadius={'md'}
cursor={'pointer'}
>
<AddIcon mr={2} />
<Box>{t('Create New')}</Box>
</Flex>
</MenuButton>
}
menuList={[
{
child: (
<Flex>
<Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={1} />
{t('Folder')}
</Flex>
),
onClick: () => setEditFolderData({})
},
{
child: (
<Flex>
<Image src={'/imgs/module/db.png'} alt={''} w={'20px'} mr={1} />
{t('Dataset')}
</Flex>
),
onClick: onOpenCreateModal
}
]}
/>
</Flex> </Flex>
<Grid <Grid
p={5} p={5}
...@@ -86,7 +193,7 @@ const Kb = () => { ...@@ -86,7 +193,7 @@ const Kb = () => {
py={4} py={4}
px={5} px={5}
cursor={'pointer'} cursor={'pointer'}
h={'140px'} h={'130px'}
border={theme.borders.md} border={theme.borders.md}
boxShadow={'none'} boxShadow={'none'}
userSelect={'none'} userSelect={'none'}
...@@ -98,18 +205,28 @@ const Kb = () => { ...@@ -98,18 +205,28 @@ const Kb = () => {
display: 'block' display: 'block'
} }
}} }}
onClick={() => onClick={() => {
router.push({ if (kb.type === KbTypeEnum.folder) {
pathname: '/kb/detail', router.push({
query: { pathname: '/kb/list',
kbId: kb._id query: {
} parentId: kb._id
}) }
} });
} else if (kb.type === KbTypeEnum.dataset) {
router.push({
pathname: '/kb/detail',
query: {
kbId: kb._id
}
});
}
}}
> >
<Flex alignItems={'center'} h={'38px'}> <Flex alignItems={'center'} h={'38px'}>
<Avatar src={kb.avatar} borderRadius={'lg'} w={'28px'} /> <Avatar src={kb.avatar} borderRadius={'lg'} w={'28px'} />
<Box ml={3}>{kb.name}</Box> <Box ml={3}>{kb.name}</Box>
<IconButton <IconButton
className="delete" className="delete"
position={'absolute'} position={'absolute'}
...@@ -126,7 +243,11 @@ const Kb = () => { ...@@ -126,7 +243,11 @@ const Kb = () => {
}} }}
onClick={(e) => { onClick={(e) => {
e.stopPropagation(); e.stopPropagation();
openConfirm(() => onclickDelKb(kb._id))(); openConfirm(
() => onclickDelKb(kb._id),
undefined,
DeleteTipsMap.current[kb.type]
)();
}} }}
/> />
</Flex> </Flex>
...@@ -140,8 +261,14 @@ const Kb = () => { ...@@ -140,8 +261,14 @@ const Kb = () => {
</Flex> </Flex>
</Box> </Box>
<Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}> <Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
<MyIcon mr={1} name="kbTest" w={'12px'} /> {kb.type === KbTypeEnum.folder ? (
<Box color={'myGray.500'}>{kb.vectorModel.name}</Box> <Box color={'myGray.500'}>{t('Folder')}</Box>
) : (
<>
<MyIcon mr={1} name="kbTest" w={'12px'} />
<Box color={'myGray.500'}>{kb.vectorModel.name}</Box>
</>
)}
</Flex> </Flex>
</Card> </Card>
))} ))}
...@@ -155,7 +282,15 @@ const Kb = () => { ...@@ -155,7 +282,15 @@ const Kb = () => {
</Flex> </Flex>
)} )}
<ConfirmModal /> <ConfirmModal />
{isOpenCreateModal && <CreateModal onClose={onCloseCreateModal} />} {isOpenCreateModal && <CreateModal onClose={onCloseCreateModal} parentId={parentId} />}
{!!editFolderData && (
<EditFolderModal
onClose={() => setEditFolderData(undefined)}
onSuccess={refetch}
parentId={parentId}
{...editFolderData}
/>
)}
</PageContainer> </PageContainer>
); );
}; };
......
client/src/pages/login/index.tsx
...@@ -13,6 +13,7 @@ import { serviceSideProps } from '@/utils/i18n'; ...@@ -13,6 +13,7 @@ import { serviceSideProps } from '@/utils/i18n';
import { setToken } from '@/utils/user'; import { setToken } from '@/utils/user';
import { feConfigs } from '@/store/static'; import { feConfigs } from '@/store/static';
import CommunityModal from '@/components/CommunityModal'; import CommunityModal from '@/components/CommunityModal';
import Script from 'next/script';
const RegisterForm = dynamic(() => import('./components/RegisterForm')); const RegisterForm = dynamic(() => import('./components/RegisterForm'));
const ForgetPasswordForm = dynamic(() => import('./components/ForgetPasswordForm')); const ForgetPasswordForm = dynamic(() => import('./components/ForgetPasswordForm'));
...@@ -53,70 +54,77 @@ const Login = () => { ...@@ -53,70 +54,77 @@ const Login = () => {
} }
return ( return (
<Flex <>
alignItems={'center'} {feConfigs.googleClientVerKey && (
justifyContent={'center'} <Script
className={styles.loginPage} src={`https://www.recaptcha.net/recaptcha/api.js?render=${feConfigs.googleClientVerKey}`}
h={'100%'} ></Script>
px={[0, '10vw']} )}
>
<Flex <Flex
height="100%"
w={'100%'}
maxW={'1240px'}
maxH={['auto', 'max(660px,80vh)']}
backgroundColor={'#fff'}
alignItems={'center'} alignItems={'center'}
justifyContent={'center'} justifyContent={'center'}
py={[5, 10]} className={styles.loginPage}
px={'5vw'} h={'100%'}
borderRadius={isPc ? 'md' : 'none'} px={[0, '10vw']}
gap={5}
> >
{isPc && ( <Flex
<Image height="100%"
src={'/icon/loginLeft.svg'} w={'100%'}
order={pageType === PageTypeEnum.login ? 0 : 2} maxW={'1240px'}
flex={'1 0 0'} maxH={['auto', 'max(660px,80vh)']}
w="0" backgroundColor={'#fff'}
maxW={'600px'} alignItems={'center'}
height={'100%'} justifyContent={'center'}
maxH={'450px'} py={[5, 10]}
alt="" px={'5vw'}
/>
)}
<Box
position={'relative'}
order={1}
flex={`0 0 ${isPc ? '400px' : '100%'}`}
height={'100%'}
border="1px"
borderColor="gray.200"
py={5}
px={10}
borderRadius={isPc ? 'md' : 'none'} borderRadius={isPc ? 'md' : 'none'}
gap={5}
> >
<DynamicComponent type={pageType} /> {isPc && (
<Image
{feConfigs?.show_contact && ( src={'/icon/loginLeft.svg'}
<Box order={pageType === PageTypeEnum.login ? 0 : 2}
fontSize={'sm'} flex={'1 0 0'}
color={'myGray.600'} w="0"
cursor={'pointer'} maxW={'600px'}
position={'absolute'} height={'100%'}
right={5} maxH={'450px'}
bottom={3} alt=""
onClick={onOpen} />
>
无法登录，点击联系
</Box>
)} )}
</Box>
</Flex>
{isOpen && <CommunityModal onClose={onClose} />} <Box
</Flex> position={'relative'}
order={1}
flex={`0 0 ${isPc ? '400px' : '100%'}`}
height={'100%'}
border="1px"
borderColor="gray.200"
py={5}
px={10}
borderRadius={isPc ? 'md' : 'none'}
>
<DynamicComponent type={pageType} />
{feConfigs?.show_contact && (
<Box
fontSize={'sm'}
color={'myGray.600'}
cursor={'pointer'}
position={'absolute'}
right={5}
bottom={3}
onClick={onOpen}
>
无法登录，点击联系
</Box>
)}
</Box>
</Flex>
{isOpen && <CommunityModal onClose={onClose} />}
</Flex>
</>
); );
}; };
......
client/src/service/events/generateQA.ts
...@@ -10,6 +10,7 @@ import { ChatCompletionRequestMessage } from 'openai'; ...@@ -10,6 +10,7 @@ import { ChatCompletionRequestMessage } from 'openai';
import { modelToolMap } from '@/utils/plugin'; import { modelToolMap } from '@/utils/plugin';
import { gptMessage2ChatType } from '@/utils/adapt'; import { gptMessage2ChatType } from '@/utils/adapt';
import { addLog } from '../utils/tools'; import { addLog } from '../utils/tools';
import { splitText2Chunks } from '@/utils/file';
const reduceQueue = () => { const reduceQueue = () => {
global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0; global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
...@@ -157,7 +158,7 @@ A2: ...@@ -157,7 +158,7 @@ A2:
console.log('openai error: 生成QA错误'); console.log('openai error: 生成QA错误');
console.log(err.response?.status, err.response?.statusText, err.response?.data); console.log(err.response?.status, err.response?.statusText, err.response?.data);
} else { } else {
console.log('生成QA错误:', err); addLog.error('生成 QA 错误', err);
} }
// message error or openai account error // message error or openai account error
...@@ -212,5 +213,16 @@ function formatSplitText(text: string) { ...@@ -212,5 +213,16 @@ function formatSplitText(text: string) {
} }
} }
// empty result. direct split chunk
if (result.length === 0) {
const splitRes = splitText2Chunks({ text: text, maxLen: 500 });
splitRes.chunks.forEach((item) => {
result.push({
q: item,
a: ''
});
});
}
return result; return result;
} }
client/src/service/events/generateVector.ts
...@@ -96,9 +96,7 @@ export async function generateVector(): Promise<any> { ...@@ -96,9 +96,7 @@ export async function generateVector(): Promise<any> {
data: err.response?.data data: err.response?.data
}); });
} else { } else {
addLog.info('openai error: 生成向量错误', { addLog.error('openai error: 生成向量错误', err);
err
});
} }
// message error or openai account error // message error or openai account error
......
client/src/service/lib/gridfs.ts
...@@ -2,19 +2,12 @@ import mongoose, { Types } from 'mongoose'; ...@@ -2,19 +2,12 @@ import mongoose, { Types } from 'mongoose';
import fs from 'fs'; import fs from 'fs';
import fsp from 'fs/promises'; import fsp from 'fs/promises';
import { ERROR_ENUM } from '../errorCode'; import { ERROR_ENUM } from '../errorCode';
import type { FileInfo } from '@/types/plugin';
enum BucketNameEnum { enum BucketNameEnum {
dataset = 'dataset' dataset = 'dataset'
} }
type FileInfo = {
id: string;
filename: string;
size: number;
contentType: string;
encoding: string;
};
export class GridFSStorage { export class GridFSStorage {
readonly type = 'gridfs'; readonly type = 'gridfs';
readonly bucket: `${BucketNameEnum}`; readonly bucket: `${BucketNameEnum}`;
...@@ -88,6 +81,7 @@ export class GridFSStorage { ...@@ -88,6 +81,7 @@ export class GridFSStorage {
filename: file.filename, filename: file.filename,
contentType: file.metadata?.contentType, contentType: file.metadata?.contentType,
encoding: file.metadata?.encoding, encoding: file.metadata?.encoding,
uploadDate: file.uploadDate,
size: file.length size: file.length
}; };
} }
......
client/src/service/models/bill.ts
...@@ -55,7 +55,6 @@ const BillSchema = new Schema({ ...@@ -55,7 +55,6 @@ const BillSchema = new Schema({
try { try {
BillSchema.index({ userId: 1 }); BillSchema.index({ userId: 1 });
// BillSchema.index({ time: -1 });
BillSchema.index({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 }); BillSchema.index({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 });
} catch (error) { } catch (error) {
console.log(error); console.log(error);
......
client/src/service/models/chatItem.ts
...@@ -48,24 +48,7 @@ const ChatItemSchema = new Schema({ ...@@ -48,24 +48,7 @@ const ChatItemSchema = new Schema({
} }
}, },
[TaskResponseKeyEnum.responseData]: { [TaskResponseKeyEnum.responseData]: {
type: [ type: Array,
{
moduleName: String,
price: String,
model: String,
tokens: Number,
question: String,
answer: String,
temperature: Number,
maxToken: Number,
quoteList: Array,
completeMessages: Array,
similarity: Number,
limit: Number,
cqList: Array,
cqResult: String
}
],
default: [] default: []
} }
}); });
......
client/src/service/models/kb.ts
import { Schema, model, models, Model } from 'mongoose'; import { Schema, model, models, Model } from 'mongoose';
import { kbSchema as SchemaType } from '@/types/mongoSchema'; import { kbSchema as SchemaType } from '@/types/mongoSchema';
import { KbTypeMap } from '@/constants/kb';
const kbSchema = new Schema({ const kbSchema = new Schema({
parentId: {
type: Schema.Types.ObjectId,
ref: 'kb',
default: null
},
userId: { userId: {
type: Schema.Types.ObjectId, type: Schema.Types.ObjectId,
ref: 'user', ref: 'user',
...@@ -24,6 +30,12 @@ const kbSchema = new Schema({ ...@@ -24,6 +30,12 @@ const kbSchema = new Schema({
required: true, required: true,
default: 'text-embedding-ada-002' default: 'text-embedding-ada-002'
}, },
type: {
type: String,
enum: Object.keys(KbTypeMap),
required: true,
default: 'dataset'
},
tags: { tags: {
type: [String], type: [String],
default: [] default: []
......
client/src/service/pg.ts
...@@ -13,6 +13,7 @@ export const connectPg = async (): Promise<Pool> => { ...@@ -13,6 +13,7 @@ export const connectPg = async (): Promise<Pool> => {
connectionString: process.env.PG_URL, connectionString: process.env.PG_URL,
max: Number(process.env.DB_MAX_LINK || 5), max: Number(process.env.DB_MAX_LINK || 5),
keepAlive: true, keepAlive: true,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 5000 connectionTimeoutMillis: 5000
}); });
...@@ -107,6 +108,7 @@ class Pg { ...@@ -107,6 +108,7 @@ class Pg {
} }
LIMIT ${props.limit || 10} OFFSET ${props.offset || 0} LIMIT ${props.limit || 10} OFFSET ${props.offset || 0}
`; `;
const pg = await connectPg(); const pg = await connectPg();
return pg.query<T>(sql); return pg.query<T>(sql);
} }
......
client/src/service/response.ts
...@@ -44,7 +44,7 @@ export const jsonRes = <T = any>( ...@@ -44,7 +44,7 @@ export const jsonRes = <T = any>(
if (typeof error === 'string') { if (typeof error === 'string') {
msg = error; msg = error;
} else if (proxyError[error?.code]) { } else if (proxyError[error?.code]) {
msg = '接口连接异常'; msg = '网络连接异常';
} else if (error?.response?.data?.error?.message) { } else if (error?.response?.data?.error?.message) {
msg = error?.response?.data?.error?.message; msg = error?.response?.data?.error?.message;
} else if (openaiAccountError[error?.response?.data?.error?.code]) { } else if (openaiAccountError[error?.response?.data?.error?.code]) {
...@@ -85,7 +85,7 @@ export const sseErrRes = (res: NextApiResponse, error: any) => { ...@@ -85,7 +85,7 @@ export const sseErrRes = (res: NextApiResponse, error: any) => {
if (typeof error === 'string') { if (typeof error === 'string') {
msg = error; msg = error;
} else if (proxyError[error?.code]) { } else if (proxyError[error?.code]) {
msg = '接口连接异常'; msg = '网络连接异常';
} else if (error?.response?.data?.error?.message) { } else if (error?.response?.data?.error?.message) {
msg = error?.response?.data?.error?.message; msg = error?.response?.data?.error?.message;
} else if (openaiAccountError[error?.response?.data?.error?.code]) { } else if (openaiAccountError[error?.response?.data?.error?.code]) {
......
client/src/service/utils/auth.ts
...@@ -155,7 +155,7 @@ export const authUser = async ({ ...@@ -155,7 +155,7 @@ export const authUser = async ({
})(); })();
return { return {
userId: uid, userId: String(uid),
appId, appId,
authType, authType,
user user
......
client/src/store/kb.ts → client/src/store/dataset.ts
...@@ -2,18 +2,62 @@ import { create } from 'zustand'; ...@@ -2,18 +2,62 @@ import { create } from 'zustand';
import { devtools, persist } from 'zustand/middleware'; import { devtools, persist } from 'zustand/middleware';
import { immer } from 'zustand/middleware/immer'; import { immer } from 'zustand/middleware/immer';
import { type KbTestItemType } from '@/types/plugin'; import { type KbTestItemType } from '@/types/plugin';
import type { KbItemType, KbListItemType } from '@/types/plugin';
import { getKbList, getKbById, getAllDataset } from '@/api/plugins/kb';
import { defaultKbDetail } from '@/constants/kb';
type State = { type State = {
datasets: KbListItemType[];
loadAllDatasets: () => Promise<KbListItemType[]>;
myKbList: KbListItemType[];
loadKbList: (parentId?: string) => Promise<any>;
setKbList(val: KbListItemType[]): void;
kbDetail: KbItemType;
getKbDetail: (id: string, init?: boolean) => Promise<KbItemType>;
kbTestList: KbTestItemType[]; kbTestList: KbTestItemType[];
pushKbTestItem: (data: KbTestItemType) => void; pushKbTestItem: (data: KbTestItemType) => void;
delKbTestItemById: (id: string) => void; delKbTestItemById: (id: string) => void;
updateKbItemById: (data: KbTestItemType) => void; updateKbItemById: (data: KbTestItemType) => void;
}; };
export const useKbStore = create<State>()( export const useDatasetStore = create<State>()(
devtools( devtools(
persist( persist(
immer((set, get) => ({ immer((set, get) => ({
datasets: [],
async loadAllDatasets() {
const res = await getAllDataset();
set((state) => {
state.datasets = res;
});
return res;
},
myKbList: [],
async loadKbList(parentId) {
const res = await getKbList(parentId);
set((state) => {
state.myKbList = res;
});
return res;
},
setKbList(val) {
set((state) => {
state.myKbList = val;
});
},
kbDetail: defaultKbDetail,
async getKbDetail(id: string, init = false) {
if (id === get().kbDetail._id && !init) return get().kbDetail;
const data = await getKbById(id);
set((state) => {
state.kbDetail = data;
});
return data;
},
kbTestList: [], kbTestList: [],
pushKbTestItem(data) { pushKbTestItem(data) {
set((state) => { set((state) => {
......
client/src/store/user.ts
...@@ -7,9 +7,7 @@ import { formatPrice } from '@/utils/user'; ...@@ -7,9 +7,7 @@ import { formatPrice } from '@/utils/user';
import { getTokenLogin, putUserInfo } from '@/api/user'; import { getTokenLogin, putUserInfo } from '@/api/user';
import { defaultApp } from '@/constants/model'; import { defaultApp } from '@/constants/model';
import { AppListItemType, AppUpdateParams } from '@/types/app'; import { AppListItemType, AppUpdateParams } from '@/types/app';
import type { KbItemType, KbListItemType } from '@/types/plugin';
import { getKbList, getKbById } from '@/api/plugins/kb';
import { defaultKbDetail } from '@/constants/kb';
import type { AppSchema } from '@/types/mongoSchema'; import type { AppSchema } from '@/types/mongoSchema';
type State = { type State = {
...@@ -24,12 +22,6 @@ type State = { ...@@ -24,12 +22,6 @@ type State = {
loadAppDetail: (id: string, init?: boolean) => Promise<AppSchema>; loadAppDetail: (id: string, init?: boolean) => Promise<AppSchema>;
updateAppDetail(appId: string, data: AppUpdateParams): Promise<void>; updateAppDetail(appId: string, data: AppUpdateParams): Promise<void>;
clearAppModules(): void; clearAppModules(): void;
// kb
myKbList: KbListItemType[];
loadKbList: () => Promise<any>;
setKbList(val: KbListItemType[]): void;
kbDetail: KbItemType;
getKbDetail: (id: string, init?: boolean) => Promise<KbItemType>;
}; };
export const useUserStore = create<State>()( export const useUserStore = create<State>()(
...@@ -106,31 +98,6 @@ export const useUserStore = create<State>()( ...@@ -106,31 +98,6 @@ export const useUserStore = create<State>()(
modules: [] modules: []
}; };
}); });
},
myKbList: [],
async loadKbList() {
const res = await getKbList();
set((state) => {
state.myKbList = res;
});
return res;
},
setKbList(val: KbListItemType[]) {
set((state) => {
state.myKbList = val;
});
},
kbDetail: defaultKbDetail,
async getKbDetail(id: string, init = false) {
if (id === get().kbDetail._id && !init) return get().kbDetail;
const data = await getKbById(id);
set((state) => {
state.kbDetail = data;
});
return data;
} }
})), })),
{ {
......
client/src/types/index.d.ts
...@@ -28,6 +28,7 @@ export type FeConfigsType = { ...@@ -28,6 +28,7 @@ export type FeConfigsType = {
beianText?: string; beianText?: string;
googleClientVerKey?: string; googleClientVerKey?: string;
gitLoginKey?: string; gitLoginKey?: string;
exportLimitMinutes?: number;
scripts?: { [key: string]: string }[]; scripts?: { [key: string]: string }[];
}; };
export type SystemEnvType = { export type SystemEnvType = {
...@@ -58,7 +59,6 @@ declare global { ...@@ -58,7 +59,6 @@ declare global {
interface Window { interface Window {
['pdfjs-dist/build/pdf']: any; ['pdfjs-dist/build/pdf']: any;
particlesJS: any;
grecaptcha: any; grecaptcha: any;
QRCode: any; QRCode: any;
umami?: { umami?: {
......
client/src/types/mongoSchema.d.ts
...@@ -6,6 +6,7 @@ import { TrainingModeEnum } from '@/constants/plugin'; ...@@ -6,6 +6,7 @@ import { TrainingModeEnum } from '@/constants/plugin';
import type { AppModuleItemType } from './app'; import type { AppModuleItemType } from './app';
import { ChatSourceEnum, OutLinkTypeEnum } from '@/constants/chat'; import { ChatSourceEnum, OutLinkTypeEnum } from '@/constants/chat';
import { AppTypeEnum } from '@/constants/app'; import { AppTypeEnum } from '@/constants/app';
import { KbTypeEnum } from '@/constants/kb';
export interface UserModelSchema { export interface UserModelSchema {
_id: string; _id: string;
...@@ -166,15 +167,17 @@ export interface OutLinkSchema { ...@@ -166,15 +167,17 @@ export interface OutLinkSchema {
type: `${OutLinkTypeEnum}`; type: `${OutLinkTypeEnum}`;
} }
export interface kbSchema { export type kbSchema = {
_id: string; _id: string;
userId: string; userId: string;
parentId: string;
updateTime: Date; updateTime: Date;
avatar: string; avatar: string;
name: string; name: string;
vectorModel: string; vectorModel: string;
tags: string[]; tags: string[];
} type: `${KbTypeEnum}`;
};
export interface informSchema { export interface informSchema {
_id: string; _id: string;
......
client/src/types/plugin.d.ts
import { FileStatusEnum } from '@/constants/kb';
import { VectorModelItemType } from './model'; import { VectorModelItemType } from './model';
import type { kbSchema } from './mongoSchema'; import type { kbSchema } from './mongoSchema';
export type SelectedKbType = { kbId: string; vectorModel: VectorModelItemType }[]; export type SelectedKbType = { kbId: string; vectorModel: VectorModelItemType }[];
export type KbListItemType = { export type KbListItemType = Omit<kbSchema, 'vectorModel'> & {
_id: string;
avatar: string;
name: string;
tags: string[];
vectorModel: VectorModelItemType; vectorModel: VectorModelItemType;
}; };
export type KbPathItemType = {
parentId: string;
parentName: string;
};
/* kb type */ /* kb type */
export interface KbItemType { export interface KbItemType {
_id: string; _id: string;
...@@ -20,6 +23,15 @@ export interface KbItemType { ...@@ -20,6 +23,15 @@ export interface KbItemType {
tags: string; tags: string;
} }
export type KbFileItemType = {
id: string;
size: number;
filename: string;
uploadTime: Date;
chunkLength: number;
status: `${FileStatusEnum}`;
};
export type DatasetItemType = { export type DatasetItemType = {
q: string; // 提问词 q: string; // 提问词
a: string; // 原文 a: string; // 原文
...@@ -42,3 +54,12 @@ export type FetchResultItem = { ...@@ -42,3 +54,12 @@ export type FetchResultItem = {
url: string; url: string;
content: string; content: string;
}; };
export type FileInfo = {
id: string;
filename: string;
size: number;
contentType: string;
encoding: string;
uploadDate: Date;
};
client/src/utils/plugin/openai.ts
...@@ -51,9 +51,18 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) { ...@@ -51,9 +51,18 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) {
const adaptMessages = adaptChatItem_openAI({ messages, reserveId: true }); const adaptMessages = adaptChatItem_openAI({ messages, reserveId: true });
const token = adaptMessages.reduce((sum, item) => { const token = adaptMessages.reduce((sum, item) => {
const text = `${item.role}\n${item.content}`; const text = `${item.role}\n${item.content}`;
const enc = getOpenAiEncMap();
const encodeText = enc.encode(text); /* use textLen as tokens if encode error */
const tokens = encodeText.length + 3; // 补充估算值 const tokens = (() => {
try {
const enc = getOpenAiEncMap();
const encodeText = enc.encode(text);
return encodeText.length + 3; // 补充估算值
} catch (error) {
return text.length;
}
})();
return sum + tokens; return sum + tokens;
}, 0); }, 0);
...@@ -62,9 +71,14 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) { ...@@ -62,9 +71,14 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) {
export const openAiSliceTextByToken = ({ text, length }: { text: string; length: number }) => { export const openAiSliceTextByToken = ({ text, length }: { text: string; length: number }) => {
const enc = getOpenAiEncMap(); const enc = getOpenAiEncMap();
const encodeText = enc.encode(text);
const decoder = new TextDecoder(); try {
return decoder.decode(enc.decode(encodeText.slice(0, length))); const encodeText = enc.encode(text);
const decoder = new TextDecoder();
return decoder.decode(enc.decode(encodeText.slice(0, length)));
} catch (error) {
return text.slice(0, length);
}
}; };
export const authOpenAiKey = async (key: string) => { export const authOpenAiKey = async (key: string) => {
......
client/src/utils/sse.ts
...@@ -44,7 +44,7 @@ export class SSEParseData { ...@@ -44,7 +44,7 @@ export class SSEParseData {
data: parseData data: parseData
}; };
} catch (error) { } catch (error) {
if (typeof item.data === 'string' && !item.data.startsWith(": ping")) { if (typeof item.data === 'string' && !item.data.startsWith(': ping')) {
this.storeReadData += item.data; this.storeReadData += item.data;
} else { } else {
this.storeReadData = ''; this.storeReadData = '';
......
client/src/utils/tools.ts
...@@ -59,6 +59,9 @@ export const Obj2Query = (obj: Record<string, string | number>) => { ...@@ -59,6 +59,9 @@ export const Obj2Query = (obj: Record<string, string | number>) => {
return queryParams.toString(); return queryParams.toString();
}; };
/**
* parse string to query object
*/
export const parseQueryString = (str: string) => { export const parseQueryString = (str: string) => {
const queryObject: Record<string, any> = {}; const queryObject: Record<string, any> = {};
...@@ -125,6 +128,16 @@ export const formatTimeToChatTime = (time: Date) => { ...@@ -125,6 +128,16 @@ export const formatTimeToChatTime = (time: Date) => {
return target.format('YYYY/M/D'); return target.format('YYYY/M/D');
}; };
export const formatFileSize = (bytes: number): string => {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
};
export const hasVoiceApi = typeof window !== 'undefined' && 'speechSynthesis' in window; export const hasVoiceApi = typeof window !== 'undefined' && 'speechSynthesis' in window;
/** /**
* voice broadcast * voice broadcast
......
docSite/content/docs/installation/upgrading/44.md 0 → 100644
---
title: '升级到 V4.4'
description: 'FastGPT 从旧版本升级到 V4.4 操作指南'
icon: 'upgrade'
draft: false
toc: true
weight: 996
---
## 执行初始化 API
发起 1 个 HTTP 请求（记得携带 `headers.rootkey`，这个值是环境变量里的）
1. https://xxxxx/api/admin/initv44
```bash
curl --location --request POST 'https://{{host}}/api/admin/initv44' \
--header 'rootkey: {{rootkey}}' \
--header 'Content-Type: application/json'
```
会给初始化 Mongo 的部分字段。
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment