Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit a79429fd by archer
Options

feat: kb crud

parent 021add2a
Showing with 1161 additions and 762 deletions
public/docs/csvSelect.md
接受一个 csv 文件，表格头包含 question 和 answer。question 代表问题，answer 代表答案。
导入前会进行去重，如果问题和答案完全相同，则不会被导入，所以最终导入的内容可能会比文件的内容少。但是，对于带有换行的内容，目前无法去重。
**请保证 csv 文件为 utf-8 编码**
| question | answer |
| --- | --- |
| 什么是 laf | laf 是一个云函数开发平台…… |
导入前会进行去重，如果问题和答案完全相同，则不会被导入，所以最终导入的内容可能会比文件的内容少。但是，对于带有换行的内容，目前无法去重。
### 请保证 csv 文件为 utf-8 编码
| question | answer |
| ------------- | ------------------------------------------------------ |
| 什么是 laf | laf 是一个云函数开发平台…… |
| 什么是 sealos | Sealos 是以 kubernetes 为内核的云操作系统发行版,可以…… |
src/api/model.ts
import { GET, POST, DELETE, PUT } from './request';
import type { ModelSchema, ModelDataSchema } from '@/types/mongoSchema';
import type { ModelSchema } from '@/types/mongoSchema';
import type { ModelUpdateParams, ShareModelItem } from '@/types/model';
import { RequestPaging } from '../types/index';
import { Obj2Query } from '@/utils/tools';
......@@ -31,72 +31,6 @@ export const getModelById = (id: string) => GET<ModelSchema>(`/model/detail?mode
export const putModelById = (id: string, data: ModelUpdateParams) =>
PUT(`/model/update?modelId=${id}`, data);
/* 模型 data */
type GetModelDataListProps = RequestPaging & {
modelId: string;
searchText: string;
};
/**
* 获取模型的知识库数据
*/
export const getModelDataList = (props: GetModelDataListProps) =>
GET(`/model/data/getModelData?${Obj2Query(props)}`);
/**
* 获取导出数据（不分页）
*/
export const getExportDataList = (modelId: string) =>
GET<[string, string][]>(`/model/data/exportModelData?modelId=${modelId}`);
/**
* 获取模型正在拆分数据的数量
*/
export const getModelSplitDataListLen = (modelId: string) =>
GET<{
splitDataQueue: number;
embeddingQueue: number;
}>(`/model/data/getTrainingData?modelId=${modelId}`);
/**
* 获取 web 页面内容
*/
export const getWebContent = (url: string) => POST<string>(`/model/data/fetchingUrlData`, { url });
/**
* 手动输入数据
*/
export const postModelDataInput = (data: {
modelId: string;
data: { a: ModelDataSchema['a']; q: ModelDataSchema['q'] }[];
}) => POST<number>(`/model/data/pushModelDataInput`, data);
/**
* 拆分数据
*/
export const postModelDataSplitData = (data: {
modelId: string;
chunks: string[];
prompt: string;
mode: 'qa' | 'subsection';
}) => POST(`/model/data/splitData`, data);
/**
* json导入数据
*/
export const postModelDataCsvData = (modelId: string, data: string[][]) =>
POST<number>(`/model/data/pushModelDataCsv`, { modelId, data: data });
/**
* 更新模型数据
*/
export const putModelDataById = (data: { dataId: string; a: string; q?: string }) =>
PUT('/model/data/putModelData', data);
/**
* 删除一条模型数据
*/
export const delOneModelData = (dataId: string) =>
DELETE(`/model/data/delModelDataById?dataId=${dataId}`);
/* 共享市场 */
/**
* 获取共享市场模型
......
src/api/plugins/kb.ts 0 → 100644
import { GET, POST, PUT, DELETE } from '../request';
import type { KbItemType } from '@/types/plugin';
import { RequestPaging } from '@/types/index';
import { SplitTextTypEnum } from '@/constants/plugin';
import { KbDataItemType } from '@/types/plugin';
export type KbUpdateParams = { id: string; name: string; tags: string; avatar: string };
/* knowledge base */
export const getKbList = () => GET<KbItemType[]>(`/plugins/kb/list`);
export const postCreateKb = (data: { name: string }) => POST<string>(`/plugins/kb/create`, data);
export const putKbById = (data: KbUpdateParams) => PUT(`/plugins/kb/update`, data);
export const delKbById = (id: string) => DELETE(`/plugins/kb/delete?id=${id}`);
/* kb data */
type GetKbDataListProps = RequestPaging & {
kbId: string;
searchText: string;
};
export const getKbDataList = (data: GetKbDataListProps) =>
POST(`/plugins/kb/data/getDataList`, data);
/**
* 获取导出数据（不分页）
*/
export const getExportDataList = (kbId: string) =>
GET<[string, string][]>(`/plugins/kb/data/exportModelData?kbId=${kbId}`);
/**
* 获取模型正在拆分数据的数量
*/
export const getTrainingData = (kbId: string) =>
GET<{
splitDataQueue: number;
embeddingQueue: number;
}>(`/plugins/kb/data/getTrainingData?kbId=${kbId}`);
/**
* 获取 web 页面内容
*/
export const getWebContent = (url: string) => POST<string>(`/model/data/fetchingUrlData`, { url });
/**
* 直接push数据
*/
export const postKbDataFromList = (data: {
kbId: string;
data: { a: KbDataItemType['a']; q: KbDataItemType['q'] }[];
}) => POST(`/openapi/kb/pushData`, data);
/**
* 更新一条数据
*/
export const putKbDataById = (data: { dataId: string; a: string; q?: string }) =>
PUT('/openapi/kb/updateData', data);
/**
* 删除一条知识库数据
*/
export const delOneKbDataByDataId = (dataId: string) =>
DELETE(`/openapi/kb/delDataById?dataId=${dataId}`);
/**
* 拆分数据
*/
export const postSplitData = (data: {
kbId: string;
chunks: string[];
prompt: string;
mode: `${SplitTextTypEnum}`;
}) => POST(`/openapi/text/splitText`, data);
src/components/Icon/icons/kb.svg 0 → 100644
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1684163814302" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="3451" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M512 384c-229.8 0-416-57.3-416-128v256c0 70.7 186.2 128 416 128s416-57.3 416-128V256c0 70.7-186.2 128-416 128z" p-id="3452"></path><path d="M512 704c-229.8 0-416-57.3-416-128v256c0 70.7 186.2 128 416 128s416-57.3 416-128V576c0 70.7-186.2 128-416 128zM512 320c229.8 0 416-57.3 416-128S741.8 64 512 64 96 121.3 96 192s186.2 128 416 128z" p-id="3453"></path></svg>
\ No newline at end of file
src/components/Icon/index.tsx
......@@ -26,7 +26,8 @@ const map = {
closeSolid: require('./icons/closeSolid.svg').default,
wx: require('./icons/wx.svg').default,
out: require('./icons/out.svg').default,
git: require('./icons/git.svg').default
git: require('./icons/git.svg').default,
kb: require('./icons/kb.svg').default
};
export type IconName = keyof typeof map;
......
src/components/Layout/index.tsx
import React, { useEffect, useMemo } from 'react';
import { Box, useColorMode, Flex } from '@chakra-ui/react';
import { useRouter } from 'next/router';
import { useScreen } from '@/hooks/useScreen';
import { useLoading } from '@/hooks/useLoading';
import { useGlobalStore } from '@/store/global';
import { throttle } from 'lodash';
import Auth from './auth';
import Navbar from './navbar';
import NavbarPhone from './navbarPhone';
......@@ -19,12 +19,11 @@ const phoneUnShowLayoutRoute: Record<string, boolean> = {
'/chat/share': true
};
const Layout = ({ children, isPcDevice }: { children: JSX.Element; isPcDevice: boolean }) => {
const { isPc } = useScreen({ defaultIsPc: isPcDevice });
const Layout = ({ children }: { children: JSX.Element }) => {
const router = useRouter();
const { colorMode, setColorMode } = useColorMode();
const { Loading } = useLoading();
const { loading } = useGlobalStore();
const { loading, setScreenWidth, isPc } = useGlobalStore();
const isChatPage = useMemo(
() => router.pathname === '/chat' && Object.values(router.query).join('').length !== 0,
......@@ -37,6 +36,19 @@ const Layout = ({ children, isPcDevice }: { children: JSX.Element; isPcDevice: b
}
}, [colorMode, router.pathname, setColorMode]);
useEffect(() => {
const resize = throttle(() => {
setScreenWidth(document.documentElement.clientWidth);
}, 300);
resize();
window.addEventListener('resize', resize);
return () => {
window.removeEventListener('resize', resize);
};
}, [setScreenWidth]);
return (
<>
<Box
......@@ -75,9 +87,3 @@ const Layout = ({ children, isPcDevice }: { children: JSX.Element; isPcDevice: b
};
export default Layout;
Layout.getInitialProps = ({ req }: any) => {
return {
isPcDevice: !/Mobile/.test(req?.headers?.['user-agent'])
};
};
src/components/Layout/navbar.tsx
......@@ -22,7 +22,6 @@ const Navbar = () => {
link: `/chat?modelId=${lastChatModelId}&chatId=${lastChatId}`,
activeLink: ['/chat']
},
{
label: 'AI助手',
icon: 'model',
......@@ -30,6 +29,12 @@ const Navbar = () => {
activeLink: ['/model']
},
{
label: '知识库',
icon: 'kb',
link: `/kb`,
activeLink: ['/kb']
},
{
label: '共享',
icon: 'shareMarket',
link: '/model/share',
......
src/components/SideBar/index.tsx 0 → 100644
import React, { useState } from 'react';
import { Box, Flex } from '@chakra-ui/react';
import type { BoxProps } from '@chakra-ui/react';
import MyIcon from '../Icon';
interface Props extends BoxProps {}
const SideBar = (e?: Props) => {
const {
w = [1, '0 0 250px', '0 0 280px', '0 0 310px', '0 0 340px'],
children,
...props
} = e || {};
const [foldSideBar, setFoldSideBar] = useState(false);
return (
<Box
position={'relative'}
flex={foldSideBar ? '0 0 0' : w}
w={['100%', 0]}
h={'100%'}
zIndex={1}
transition={'0.2s'}
_hover={{
'& > div': { visibility: 'visible', opacity: 1 }
}}
{...props}
>
<Flex
position={'absolute'}
right={0}
top={'50%'}
transform={'translate(50%,-50%)'}
alignItems={'center'}
justifyContent={'flex-end'}
pr={1}
w={'36px'}
h={'50px'}
borderRadius={'10px'}
bg={'rgba(0,0,0,0.5)'}
cursor={'pointer'}
transition={'0.2s'}
{...(foldSideBar
? {
opacity: 0.6
}
: {
visibility: 'hidden',
opacity: 0
})}
onClick={() => setFoldSideBar(!foldSideBar)}
>
<MyIcon
name={'back'}
transform={foldSideBar ? 'rotate(180deg)' : ''}
w={'14px'}
color={'white'}
/>
</Flex>
<Box position={'relative'} h={'100%'} overflow={foldSideBar ? 'hidden' : 'visible'}>
{children}
</Box>
</Box>
);
};
export default SideBar;
src/constants/kb.ts 0 → 100644
import type { KbItemType } from '@/types/plugin';
export const defaultKbDetail: KbItemType = {
_id: '',
userId: '',
updateTime: new Date(),
avatar: '/icon/logo.png',
name: '',
tags: '',
totalData: 0
};
src/constants/plugin.ts 0 → 100644
export enum SplitTextTypEnum {
'qa' = 'qa',
'subsection' = 'subsection'
}
src/hooks/usePagination.tsx
......@@ -41,6 +41,7 @@ export const usePagination = <T = any,>({
});
console.log(error);
}
return null;
}
});
......
src/pages/api/model/data/fetchingUrlData.ts deleted 100644 → 0
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import axios from 'axios';
import { axiosConfig } from '@/service/utils/tools';
/**
* 读取网站的内容
*/
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { url } = req.body as { url: string };
if (!url) {
throw new Error('缺少 url');
}
await connectToDatabase();
await authToken(req);
const data = await axios
.get(url, {
httpsAgent: axiosConfig().httpsAgent
})
.then((res) => res.data as string);
jsonRes(res, { data });
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/model/del.ts
......@@ -2,7 +2,6 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { Chat, Model, connectToDatabase, Collection, ShareChat } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { authModel } from '@/service/utils/auth';
/* 获取我的模型 */
......@@ -25,11 +24,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
userId
});
// 删除 pg 中所有该模型的数据
await PgClient.delete('modelData', {
where: [['user_id', userId], 'AND', ['model_id', modelId]]
});
// 删除对应的聊天
await Chat.deleteMany({
modelId
......
src/pages/api/model/data/pushModelDataCsv.ts → src/pages/api/openapi/kb/pushData.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import type { KbDataItemType } from '@/types/plugin';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import { generateVector } from '@/service/events/generateVector';
import { ModelDataStatusEnum } from '@/constants/model';
import { PgClient } from '@/service/pg';
import { authModel } from '@/service/utils/auth';
import { authKb } from '@/service/utils/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { modelId, data } = req.body as {
modelId: string;
data: string[][];
const {
kbId,
data,
formatLineBreak = true
} = req.body as {
kbId: string;
formatLineBreak?: boolean;
data: { a: KbDataItemType['a']; q: KbDataItemType['q'] }[];
};
if (!modelId || !Array.isArray(data)) {
if (!kbId || !Array.isArray(data)) {
throw new Error('缺少参数');
}
......@@ -23,31 +28,27 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await connectToDatabase();
// 验证是否是该用户的 model
await authModel({
await authKb({
userId,
modelId
kbId
});
// 去重
// 过滤重复的内容
const searchRes = await Promise.allSettled(
data.map(async ([q, a = '']) => {
data.map(async ({ q, a = '' }) => {
if (!q) {
return Promise.reject('q为空');
}
try {
if (formatLineBreak) {
q = q.replace(/\\n/g, '\n');
a = a.replace(/\\n/g, '\n');
}
// Exactly the same data, not push
try {
const count = await PgClient.count('modelData', {
where: [
['user_id', userId],
'AND',
['model_id', modelId],
'AND',
['q', q],
'AND',
['a', a]
]
where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['q', q], 'AND', ['a', a]]
});
if (count > 0) {
return Promise.reject('已经存在');
......@@ -61,25 +62,25 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
})
);
// 过滤重复的内容
const filterData = searchRes
.filter((item) => item.status === 'fulfilled')
.map<{ q: string; a: string }>((item: any) => item.value);
// 插入 pg
// 插入记录
const insertRes = await PgClient.insert('modelData', {
values: filterData.map((item) => [
{ key: 'user_id', value: userId },
{ key: 'model_id', value: modelId },
{ key: 'kb_id', value: kbId },
{ key: 'q', value: item.q },
{ key: 'a', value: item.a },
{ key: 'status', value: ModelDataStatusEnum.waiting }
{ key: 'status', value: 'waiting' }
])
});
generateVector();
jsonRes(res, {
message: `共插入 ${insertRes.rowCount} 条数据`,
data: insertRes.rowCount
});
} catch (err) {
......@@ -89,11 +90,3 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
}
}
export const config = {
api: {
bodyParser: {
sizeLimit: '100mb'
}
}
};
src/pages/api/model/data/putModelData.ts → src/pages/api/openapi/kb/updateData.ts
......@@ -16,7 +16,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const userId = await authToken(req);
// 更新 pg 内容
// 更新 pg 内容.仅修改a，不需要更新向量。
await PgClient.update('modelData', {
where: [['id', dataId], 'AND', ['user_id', userId]],
values: [
......
src/pages/api/model/data/splitData.ts → src/pages/api/openapi/text/splitText.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, SplitData } from '@/service/mongo';
import { authModel, authToken } from '@/service/utils/auth';
import { authKb, authToken } from '@/service/utils/auth';
import { generateVector } from '@/service/events/generateVector';
import { generateQA } from '@/service/events/generateQA';
import { PgClient } from '@/service/pg';
import { SplitTextTypEnum } from '@/constants/plugin';
/* 拆分数据成QA */
/* split text */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { chunks, modelId, prompt, mode } = req.body as {
modelId: string;
const { chunks, kbId, prompt, mode } = req.body as {
kbId: string;
chunks: string[];
prompt: string;
mode: 'qa' | 'subsection';
mode: `${SplitTextTypEnum}`;
};
if (!chunks || !modelId || !prompt) {
if (!chunks || !kbId || !prompt) {
throw new Error('参数错误');
}
await connectToDatabase();
......@@ -23,27 +24,28 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const userId = await authToken(req);
// 验证是否是该用户的 model
await authModel({
modelId,
await authKb({
kbId,
userId
});
if (mode === 'qa') {
if (mode === SplitTextTypEnum.qa) {
// 批量QA拆分插入数据
await SplitData.create({
userId,
modelId,
kbId,
textList: chunks,
prompt
});
generateQA();
} else if (mode === 'subsection') {
} else if (mode === SplitTextTypEnum.subsection) {
// 待优化，直接调用另一个接口
// 插入记录
await PgClient.insert('modelData', {
values: chunks.map((item) => [
{ key: 'user_id', value: userId },
{ key: 'model_id', value: modelId },
{ key: 'kb_id', value: kbId },
{ key: 'q', value: item },
{ key: 'a', value: '' },
{ key: 'status', value: 'waiting' }
......
src/pages/api/model/data/pushModelDataInput.ts → src/pages/api/plugins/kb/create.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { connectToDatabase, KB } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import { ModelDataSchema } from '@/types/mongoSchema';
import { generateVector } from '@/service/events/generateVector';
import { PgClient } from '@/service/pg';
import { authModel } from '@/service/utils/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { modelId, data } = req.body as {
modelId: string;
data: { a: ModelDataSchema['a']; q: ModelDataSchema['q'] }[];
const { name, tags } = req.body as {
name: string;
tags: string[];
};
if (!modelId || !Array.isArray(data)) {
if (!name) {
throw new Error('缺少参数');
}
......@@ -23,28 +19,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await connectToDatabase();
// 验证是否是该用户的 model
await authModel({
const { _id } = await KB.create({
name,
userId,
modelId
tags
});
// 插入记录
await PgClient.insert('modelData', {
values: data.map((item) => [
{ key: 'user_id', value: userId },
{ key: 'model_id', value: modelId },
{ key: 'q', value: item.q },
{ key: 'a', value: item.a },
{ key: 'status', value: 'waiting' }
])
});
generateVector();
jsonRes(res, {
data: 0
});
jsonRes(res, { data: _id });
} catch (err) {
jsonRes(res, {
code: 500,
......
src/pages/api/model/data/exportModelData.ts → src/pages/api/plugins/kb/data/exportModelData.ts
......@@ -6,11 +6,11 @@ import { PgClient } from '@/service/pg';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
let { modelId } = req.query as {
modelId: string;
let { kbId } = req.query as {
kbId: string;
};
if (!modelId) {
if (!kbId) {
throw new Error('缺少参数');
}
......@@ -21,11 +21,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 统计数据
const count = await PgClient.count('modelData', {
where: [['model_id', modelId], 'AND', ['user_id', userId]]
where: [['kb_id', kbId], 'AND', ['user_id', userId]]
});
// 从 pg 中获取所有数据
const pgData = await PgClient.select<{ q: string; a: string }>('modelData', {
where: [['model_id', modelId], 'AND', ['user_id', userId]],
where: [['kb_id', kbId], 'AND', ['user_id', userId]],
fields: ['q', 'a'],
order: [{ field: 'id', mode: 'DESC' }],
limit: count
......
src/pages/api/model/data/getModelData.ts → src/pages/api/plugins/kb/data/getDataList.ts
......@@ -3,27 +3,22 @@ import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import type { PgModelDataItemType } from '@/types/pg';
import { authModel } from '@/service/utils/auth';
import type { PgKBDataItemType } from '@/types/pg';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
let {
modelId,
kbId,
pageNum = 1,
pageSize = 10,
searchText = ''
} = req.query as {
modelId: string;
pageNum: string;
pageSize: string;
} = req.body as {
kbId: string;
pageNum: number;
pageSize: number;
searchText: string;
};
pageNum = +pageNum;
pageSize = +pageSize;
if (!modelId) {
if (!kbId) {
throw new Error('缺少参数');
}
......@@ -32,19 +27,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await connectToDatabase();
const { model } = await authModel({
userId,
modelId,
authOwner: false
});
const where: any = [
...(model.share.isShareDetail ? [] : [['user_id', userId], 'AND']),
['model_id', modelId],
['user_id', userId],
'AND',
['kb_id', kbId],
...(searchText ? ['AND', `(q LIKE '%${searchText}%' OR a LIKE '%${searchText}%')`] : [])
];
const searchRes = await PgClient.select<PgModelDataItemType>('modelData', {
const searchRes = await PgClient.select<PgKBDataItemType>('modelData', {
fields: ['id', 'q', 'a', 'status'],
where,
order: [{ field: 'id', mode: 'DESC' }],
......
src/pages/api/model/data/getTrainingData.ts → src/pages/api/plugins/kb/data/getTrainingData.ts
......@@ -8,8 +8,8 @@ import { PgClient } from '@/service/pg';
/* 拆分数据成QA */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { modelId } = req.query as { modelId: string };
if (!modelId) {
const { kbId } = req.query as { kbId: string };
if (!kbId) {
throw new Error('参数错误');
}
await connectToDatabase();
......@@ -19,25 +19,25 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
// split queue data
const data = await SplitData.find({
userId,
modelId,
kbId,
textList: { $exists: true, $not: { $size: 0 } }
});
// embedding queue data
const where: any = [
['user_id', userId],
'AND',
['model_id', modelId],
'AND',
['status', ModelDataStatusEnum.waiting]
];
const embeddingData = await PgClient.count('modelData', {
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
'AND',
['status', ModelDataStatusEnum.waiting]
]
});
jsonRes(res, {
data: {
splitDataQueue: data.map((item) => item.textList).flat().length,
embeddingQueue: await PgClient.count('modelData', {
where
})
embeddingQueue: embeddingData
}
});
} catch (err) {
......
src/pages/api/plugins/kb/delete.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { id } = req.query as {
id: string;
};
if (!id) {
throw new Error('缺少参数');
}
// 凭证校验
const userId = await authToken(req);
await connectToDatabase();
// delete mongo data
await KB.findOneAndDelete({
_id: id,
userId
});
// delete all pg data
// 删除 pg 中所有该模型的数据
await PgClient.delete('modelData', {
where: [['user_id', userId], 'AND', ['kb_id', id]]
});
// delete related model
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/plugins/kb/list.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { KbItemType } from '@/types/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
// 凭证校验
const userId = await authToken(req);
await connectToDatabase();
const kbList = await KB.find({
userId
}).sort({ updateTime: -1 });
const data = await Promise.all(
kbList.map(async (item) => ({
_id: item._id,
avatar: item.avatar,
name: item.name,
userId: item.userId,
updateTime: item.updateTime,
tags: item.tags.join(' '),
totalData: await PgClient.count('modelData', {
where: [['user_id', userId], 'AND', ['kb_id', item._id]]
})
}))
);
jsonRes<KbItemType[]>(res, {
data
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/api/plugins/kb/update.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authToken } from '@/service/utils/auth';
import type { KbUpdateParams } from '@/api/plugins/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { id, name, tags, avatar } = req.body as KbUpdateParams;
if (!id || !name) {
throw new Error('缺少参数');
}
// 凭证校验
const userId = await authToken(req);
await connectToDatabase();
await KB.findOneAndUpdate(
{
_id: id,
userId
},
{
avatar,
name,
tags: tags.split(' ').filter((item) => item)
}
);
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
src/pages/chat/components/History.tsx
......@@ -20,24 +20,22 @@ import { formatTimeToChatTime } from '@/utils/tools';
import MyIcon from '@/components/Icon';
import type { HistoryItemType, ExportChatType } from '@/types/chat';
import { useChatStore } from '@/store/chat';
import { useScreen } from '@/hooks/useScreen';
import ModelList from './ModelList';
import { useGlobalStore } from '@/store/global';
import styles from '../index.module.scss';
const PcSliderBar = ({
isPcDevice,
onclickDelHistory,
onclickExportChat
}: {
isPcDevice: boolean;
onclickDelHistory: (historyId: string) => Promise<void>;
onclickExportChat: (type: ExportChatType) => void;
}) => {
const router = useRouter();
const { modelId = '', chatId = '' } = router.query as { modelId: string; chatId: string };
const theme = useTheme();
const { isPc } = useScreen({ defaultIsPc: isPcDevice });
const { isPc } = useGlobalStore();
const ContextMenuRef = useRef(null);
......
src/pages/chat/components/ShareHistory.tsx
......@@ -17,17 +17,15 @@ import { formatTimeToChatTime } from '@/utils/tools';
import MyIcon from '@/components/Icon';
import type { ShareChatHistoryItemType, ExportChatType } from '@/types/chat';
import { useChatStore } from '@/store/chat';
import { useScreen } from '@/hooks/useScreen';
import { useGlobalStore } from '@/store/global';
import styles from '../index.module.scss';
const PcSliderBar = ({
isPcDevice,
onclickDelHistory,
onclickExportChat,
onCloseSlider
}: {
isPcDevice: boolean;
onclickDelHistory: (historyId: string) => void;
onclickExportChat: (type: ExportChatType) => void;
onCloseSlider: () => void;
......@@ -35,7 +33,7 @@ const PcSliderBar = ({
const router = useRouter();
const { shareId = '', historyId = '' } = router.query as { shareId: string; historyId: string };
const theme = useTheme();
const { isPc } = useScreen({ defaultIsPc: isPcDevice });
const { isPc } = useGlobalStore();
const ContextMenuRef = useRef(null);
......
src/pages/chat/index.tsx
......@@ -33,7 +33,7 @@ import {
useTheme
} from '@chakra-ui/react';
import { useToast } from '@/hooks/useToast';
import { useScreen } from '@/hooks/useScreen';
import { useGlobalStore } from '@/store/global';
import { useQuery } from '@tanstack/react-query';
import dynamic from 'next/dynamic';
import { useCopyData, voiceBroadcast, hasVoiceApi } from '@/utils/tools';
......@@ -50,6 +50,7 @@ import { htmlTemplate } from '@/constants/common';
import { useUserStore } from '@/store/user';
import Loading from '@/components/Loading';
import Markdown from '@/components/Markdown';
import SideBar from '@/components/SideBar';
import Empty from './components/Empty';
const PhoneSliderBar = dynamic(() => import('./components/PhoneSliderBar'), {
......@@ -64,15 +65,7 @@ import styles from './index.module.scss';
const textareaMinH = '22px';
const Chat = ({
modelId,
chatId,
isPcDevice
}: {
modelId: string;
chatId: string;
isPcDevice: boolean;
}) => {
const Chat = ({ modelId, chatId }: { modelId: string; chatId: string }) => {
const router = useRouter();
const theme = useTheme();
......@@ -92,7 +85,6 @@ const Chat = ({
top: number;
message: ChatSiteItemType;
}>();
const [foldSliderBar, setFoldSlideBar] = useState(false);
const {
lastChatModelId,
......@@ -113,7 +105,7 @@ const Chat = ({
const { toast } = useToast();
const { copyData } = useCopyData();
const { isPc } = useScreen({ defaultIsPc: isPcDevice });
const { isPc } = useGlobalStore();
const { Loading, setIsLoading } = useLoading();
const { userInfo } = useUserStore();
const { isOpen: isOpenSlider, onClose: onCloseSlider, onOpen: onOpenSlider } = useDisclosure();
......@@ -481,7 +473,7 @@ const Chat = ({
navigator.vibrate?.(50); // 震动 50 毫秒
if (!isPcDevice) {
if (!isPc) {
PhoneContextShow.current = true;
}
......@@ -493,7 +485,7 @@ const Chat = ({
return false;
},
[isPcDevice]
[isPc]
);
// 获取对话信息
......@@ -636,61 +628,9 @@ const Chat = ({
>
{/* pc always show history. */}
{(isPc || !modelId) && (
<Box
position={'relative'}
flex={foldSliderBar ? '0 0 0' : [1, '0 0 250px', '0 0 280px', '0 0 310px', '0 0 340px']}
w={['100%', 0]}
h={'100%'}
zIndex={1}
transition={'0.2s'}
_hover={{
'& > div': { visibility: 'visible', opacity: 1 }
}}
>
<Flex
position={'absolute'}
right={0}
top={'50%'}
transform={'translate(50%,-50%)'}
alignItems={'center'}
justifyContent={'flex-end'}
pr={1}
w={'36px'}
h={'50px'}
borderRadius={'10px'}
bg={'rgba(0,0,0,0.5)'}
cursor={'pointer'}
transition={'0.2s'}
{...(foldSliderBar
? {
opacity: 0.6
}
: {
visibility: 'hidden',
opacity: 0
})}
onClick={() => setFoldSlideBar(!foldSliderBar)}
>
<MyIcon
name={'back'}
transform={foldSliderBar ? 'rotate(180deg)' : ''}
w={'14px'}
color={'white'}
/>
</Flex>
<Box
position={'relative'}
h={'100%'}
bg={'white'}
overflow={foldSliderBar ? 'hidden' : 'visible'}
>
<History
onclickDelHistory={onclickDelHistory}
onclickExportChat={onclickExportChat}
isPcDevice={isPcDevice}
/>
</Box>
</Box>
<SideBar>
<History onclickDelHistory={onclickDelHistory} onclickExportChat={onclickExportChat} />
</SideBar>
)}
{/* 聊天内容 */}
......@@ -906,7 +846,7 @@ const Chat = ({
}}
onKeyDown={(e) => {
// 触发快捷发送
if (isPcDevice && e.keyCode === 13 && !e.shiftKey) {
if (isPc && e.keyCode === 13 && !e.shiftKey) {
sendPrompt();
e.preventDefault();
}
......@@ -1008,8 +948,7 @@ const Chat = ({
Chat.getInitialProps = ({ query, req }: any) => {
return {
modelId: query?.modelId || '',
chatId: query?.chatId || '',
isPcDevice: !/Mobile/.test(req?.headers?.['user-agent'])
chatId: query?.chatId || ''
};
};
......
src/pages/chat/share.tsx
......@@ -31,7 +31,7 @@ import {
ModalHeader
} from '@chakra-ui/react';
import { useToast } from '@/hooks/useToast';
import { useScreen } from '@/hooks/useScreen';
import { useGlobalStore } from '@/store/global';
import { useQuery } from '@tanstack/react-query';
import dynamic from 'next/dynamic';
import { useCopyData, voiceBroadcast, hasVoiceApi } from '@/utils/tools';
......@@ -47,6 +47,7 @@ import { htmlTemplate } from '@/constants/common';
import { useUserStore } from '@/store/user';
import Loading from '@/components/Loading';
import Markdown from '@/components/Markdown';
import SideBar from '@/components/SideBar';
import Empty from './components/Empty';
const ShareHistory = dynamic(() => import('./components/ShareHistory'), {
......@@ -58,15 +59,7 @@ import styles from './index.module.scss';
const textareaMinH = '22px';
const Chat = ({
shareId,
historyId,
isPcDevice
}: {
shareId: string;
historyId: string;
isPcDevice: boolean;
}) => {
const Chat = ({ shareId, historyId }: { shareId: string; historyId: string }) => {
const router = useRouter();
const theme = useTheme();
......@@ -87,7 +80,6 @@ const Chat = ({
top: number;
message: ChatSiteItemType;
}>();
const [foldSliderBar, setFoldSlideBar] = useState(false);
const {
password,
......@@ -108,7 +100,7 @@ const Chat = ({
const { toast } = useToast();
const { copyData } = useCopyData();
const { isPc } = useScreen({ defaultIsPc: isPcDevice });
const { isPc } = useGlobalStore();
const { Loading, setIsLoading } = useLoading();
const { userInfo } = useUserStore();
const { isOpen: isOpenSlider, onClose: onCloseSlider, onOpen: onOpenSlider } = useDisclosure();
......@@ -426,7 +418,7 @@ const Chat = ({
navigator.vibrate?.(50); // 震动 50 毫秒
if (!isPcDevice) {
if (!isPc) {
PhoneContextShow.current = true;
}
......@@ -438,7 +430,7 @@ const Chat = ({
return false;
},
[isPcDevice]
[isPc]
);
// 获取对话信息
......@@ -543,57 +535,13 @@ const Chat = ({
>
{/* pc always show history. */}
{isPc && (
<Box
position={'relative'}
flex={foldSliderBar ? '0 0 0' : [1, '0 0 250px', '0 0 280px', '0 0 310px', '0 0 340px']}
w={['100%', 0]}
h={'100%'}
zIndex={1}
transition={'0.2s'}
_hover={{
'& > div': { visibility: 'visible', opacity: 1 }
}}
>
<Flex
position={'absolute'}
right={0}
top={'50%'}
transform={'translate(50%,-50%)'}
alignItems={'center'}
justifyContent={'flex-end'}
pr={1}
w={'36px'}
h={'50px'}
borderRadius={'10px'}
bg={'rgba(0,0,0,0.5)'}
cursor={'pointer'}
transition={'0.2s'}
{...(foldSliderBar
? {
opacity: 0.6
}
: {
visibility: 'hidden',
opacity: 0
})}
onClick={() => setFoldSlideBar(!foldSliderBar)}
>
<MyIcon
name={'back'}
transform={foldSliderBar ? 'rotate(180deg)' : ''}
w={'14px'}
color={'white'}
/>
</Flex>
<Box position={'relative'} h={'100%'} bg={'white'} overflow={'hidden'}>
<ShareHistory
onclickDelHistory={delShareHistoryById}
onclickExportChat={onclickExportChat}
onCloseSlider={onCloseSlider}
isPcDevice={isPcDevice}
/>
</Box>
</Box>
<SideBar>
<ShareHistory
onclickDelHistory={delShareHistoryById}
onclickExportChat={onclickExportChat}
onCloseSlider={onCloseSlider}
/>
</SideBar>
)}
{/* 聊天内容 */}
......@@ -623,13 +571,7 @@ const Chat = ({
onClick={onOpenSlider}
/>
)}
<Box
cursor={'pointer'}
lineHeight={1.2}
textAlign={'center'}
px={3}
fontSize={['sm', 'md']}
>
<Box lineHeight={1.2} textAlign={'center'} px={3} fontSize={['sm', 'md']}>
{shareChatData.model.name}
{shareChatData.history.length > 0 ? ` (${shareChatData.history.length})` : ''}
</Box>
......@@ -797,7 +739,7 @@ const Chat = ({
}}
onKeyDown={(e) => {
// 触发快捷发送
if (isPcDevice && e.keyCode === 13 && !e.shiftKey) {
if (isPc && e.keyCode === 13 && !e.shiftKey) {
sendPrompt();
e.preventDefault();
}
......@@ -854,7 +796,6 @@ const Chat = ({
onclickDelHistory={delShareHistoryById}
onclickExportChat={onclickExportChat}
onCloseSlider={onCloseSlider}
isPcDevice={isPcDevice}
/>
</DrawerContent>
</Drawer>
......@@ -924,8 +865,7 @@ const Chat = ({
Chat.getInitialProps = ({ query, req }: any) => {
return {
shareId: query?.shareId || '',
historyId: query?.historyId || '',
isPcDevice: !/Mobile/.test(req?.headers?.['user-agent'])
historyId: query?.historyId || ''
};
};
......
src/pages/index.tsx
......@@ -4,8 +4,8 @@ import Markdown from '@/components/Markdown';
import { useMarkdown } from '@/hooks/useMarkdown';
import { getFilling } from '@/api/system';
import { useQuery } from '@tanstack/react-query';
import { useScreen } from '@/hooks/useScreen';
import { useRouter } from 'next/router';
import { useGlobalStore } from '@/store/global';
import styles from './index.module.scss';
......@@ -13,7 +13,7 @@ const Home = () => {
const router = useRouter();
const { inviterId } = router.query as { inviterId: string };
const { data } = useMarkdown({ url: '/intro.md' });
const { isPc } = useScreen();
const { isPc } = useGlobalStore();
useEffect(() => {
if (inviterId) {
......
src/pages/kb/components/Detail.tsx 0 → 100644
import React, { useCallback, useState, useRef } from 'react';
import { useRouter } from 'next/router';
import {
Card,
Box,
Flex,
Button,
Tooltip,
Image,
FormControl,
Input,
Tag,
IconButton
} from '@chakra-ui/react';
import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons';
import { useToast } from '@/hooks/useToast';
import { useForm } from 'react-hook-form';
import { useQuery } from '@tanstack/react-query';
import { useUserStore } from '@/store/user';
import { delKbById, putKbById } from '@/api/plugins/kb';
import { useLoading } from '@/hooks/useLoading';
import { KbItemType } from '@/types/plugin';
import { useSelectFile } from '@/hooks/useSelectFile';
import { useConfirm } from '@/hooks/useConfirm';
import { compressImg } from '@/utils/file';
import DataCard from './DataCard';
const Detail = ({ kbId }: { kbId: string }) => {
const { toast } = useToast();
const router = useRouter();
const InputRef = useRef<HTMLInputElement>(null);
const { setLastKbId, KbDetail, getKbDetail, loadKbList, myKbList } = useUserStore();
const { Loading, setIsLoading } = useLoading();
const [btnLoading, setBtnLoading] = useState(false);
const [refresh, setRefresh] = useState(false);
const { getValues, formState, setValue, reset, register, handleSubmit } = useForm<KbItemType>({
defaultValues: KbDetail
});
const { openConfirm, ConfirmChild } = useConfirm({
content: '确认删除该知识库？数据将无法恢复，请确认！'
});
const { File, onOpen: onOpenSelectFile } = useSelectFile({
fileType: '.jpg,.png',
multiple: false
});
const { isLoading } = useQuery([kbId, myKbList], () => getKbDetail(kbId), {
onSuccess(res) {
kbId && setLastKbId(kbId);
if (res) {
reset(res);
if (InputRef.current) {
InputRef.current.value = res.tags;
}
}
},
onError(err: any) {
toast({
title: err?.message || '获取AI助手异常',
status: 'error'
});
setLastKbId('');
router.replace('/model');
}
});
/* 点击删除 */
const onclickDelKb = useCallback(async () => {
setIsLoading(true);
try {
await delKbById(kbId);
toast({
title: '删除成功',
status: 'success'
});
router.replace(`/kb?kbId=${myKbList.find((item) => item._id !== kbId)?._id || ''}`);
await loadKbList(true);
} catch (err: any) {
toast({
title: err?.message || '删除失败',
status: 'error'
});
}
setIsLoading(false);
}, [setIsLoading, kbId, toast, router, myKbList, loadKbList]);
const saveSubmitSuccess = useCallback(
async (data: KbItemType) => {
setBtnLoading(true);
try {
await putKbById({
id: kbId,
...data
});
toast({
title: '更新成功',
status: 'success'
});
loadKbList(true);
} catch (err: any) {
toast({
title: err?.message || '更新失败',
status: 'error'
});
}
setBtnLoading(false);
},
[kbId, loadKbList, toast]
);
const saveSubmitError = useCallback(() => {
// deep search message
const deepSearch = (obj: any): string => {
if (!obj) return '提交表单错误';
if (!!obj.message) {
return obj.message;
}
return deepSearch(Object.values(obj)[0]);
};
toast({
title: deepSearch(formState.errors),
status: 'error',
duration: 4000,
isClosable: true
});
}, [formState.errors, toast]);
const onSelectFile = useCallback(
async (e: File[]) => {
const file = e[0];
if (!file) return;
try {
const base64 = await compressImg({
file,
maxW: 100,
maxH: 100
});
setValue('avatar', base64);
loadKbList(true);
} catch (err: any) {
toast({
title: typeof err === 'string' ? err : '头像选择异常',
status: 'warning'
});
}
},
[loadKbList, setValue, toast]
);
return (
<Box h={'100%'} p={5} overflow={'overlay'} position={'relative'}>
<Card p={6}>
<Flex>
<Box fontWeight={'bold'} fontSize={'2xl'} flex={1}>
知识库信息
</Box>
{KbDetail._id && (
<>
<Button
isLoading={btnLoading}
mr={3}
onClick={handleSubmit(saveSubmitSuccess, saveSubmitError)}
>
保存
</Button>
<IconButton
isLoading={btnLoading}
icon={<DeleteIcon />}
aria-label={''}
variant={'solid'}
colorScheme={'red'}
onClick={openConfirm(onclickDelKb)}
/>
</>
)}
</Flex>
<Flex mt={5} alignItems={'center'}>
<Box flex={'0 0 60px'} w={0}>
头像
</Box>
<Image
src={getValues('avatar') || '/icon/logo.png'}
alt={'avatar'}
w={['28px', '36px']}
h={['28px', '36px']}
objectFit={'cover'}
cursor={'pointer'}
title={'点击切换头像'}
onClick={onOpenSelectFile}
/>
</Flex>
<FormControl mt={5}>
<Flex alignItems={'center'} maxW={'350px'}>
<Box flex={'0 0 60px'} w={0}>
名称
</Box>
<Input
{...register('name', {
required: '知识库名称不能为空'
})}
/>
</Flex>
</FormControl>
<Box>
<Flex mt={5} alignItems={'center'} maxW={'350px'} flexWrap={'wrap'}>
<Box flex={'0 0 60px'} w={0}>
标签
<Tooltip label={'仅用于记忆，用空格隔开多个标签'}>
<QuestionOutlineIcon ml={1} />
</Tooltip>
</Box>
<Input
flex={1}
ref={InputRef}
placeholder={'标签,使用空格分割。'}
onChange={(e) => {
setValue('tags', e.target.value);
setRefresh(!refresh);
}}
/>
<Box pl={'60px'} mt={2} w="100%">
{getValues('tags')
.split(' ')
.filter((item) => item)
.map((item, i) => (
<Tag mr={2} mb={2} key={i} variant={'outline'} colorScheme={'blue'}>
{item}
</Tag>
))}
</Box>
</Flex>
</Box>
</Card>
<Card p={6} mt={5}>
<DataCard kbId={kbId} />
</Card>
<File onSelect={onSelectFile} />
<ConfirmChild />
<Loading loading={isLoading} fixed={false} />
</Box>
);
};
export default Detail;
src/pages/model/components/detail/components/InputDataModal.tsx → src/pages/kb/components/InputDataModal.tsx
......@@ -11,17 +11,15 @@ import {
Textarea
} from '@chakra-ui/react';
import { useForm } from 'react-hook-form';
import { postModelDataInput, putModelDataById } from '@/api/model';
import { postKbDataFromList, putKbDataById } from '@/api/plugins/kb';
import { useToast } from '@/hooks/useToast';
import { customAlphabet } from 'nanoid';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
export type FormData = { dataId?: string; a: string; q: string };
const InputDataModal = ({
onClose,
onSuccess,
modelId,
kbId,
defaultValues = {
a: '',
q: ''
......@@ -29,7 +27,7 @@ const InputDataModal = ({
}: {
onClose: () => void;
onSuccess: () => void;
modelId: string;
kbId: string;
defaultValues?: FormData;
}) => {
const [importing, setImporting] = useState(false);
......@@ -54,8 +52,8 @@ const InputDataModal = ({
setImporting(true);
try {
const res = await postModelDataInput({
modelId: modelId,
const res = await postKbDataFromList({
kbId,
data: [
{
a: e.a,
......@@ -65,8 +63,8 @@ const InputDataModal = ({
});
toast({
title: res === 0 ? '导入数据成功,需要一段时间训练' : '数据导入异常',
status: res === 0 ? 'success' : 'warning'
title: res === 0 ? '可能已存在完全一致的数据' : '导入数据成功,需要一段时间训练',
status: 'success'
});
reset({
a: '',
......@@ -82,7 +80,7 @@ const InputDataModal = ({
}
setImporting(false);
},
[modelId, onSuccess, reset, toast]
[kbId, onSuccess, reset, toast]
);
const updateData = useCallback(
......@@ -90,7 +88,7 @@ const InputDataModal = ({
if (!e.dataId) return;
if (e.a !== defaultValues.a || e.q !== defaultValues.q) {
await putModelDataById({
await putKbDataById({
dataId: e.dataId,
a: e.a,
q: e.q === defaultValues.q ? '' : e.q
......
src/pages/kb/components/KbList.tsx 0 → 100644
import React, { useCallback, useState } from 'react';
import { Box, Flex, useTheme, Input, IconButton, Tooltip, Image, Tag } from '@chakra-ui/react';
import { AddIcon } from '@chakra-ui/icons';
import { useRouter } from 'next/router';
import { postCreateKb } from '@/api/plugins/kb';
import { useLoading } from '@/hooks/useLoading';
import { useToast } from '@/hooks/useToast';
import { useQuery } from '@tanstack/react-query';
import { useUserStore } from '@/store/user';
import MyIcon from '@/components/Icon';
const KbList = ({ kbId }: { kbId: string }) => {
const theme = useTheme();
const router = useRouter();
const { toast } = useToast();
const { Loading, setIsLoading } = useLoading();
const { myKbList, loadKbList } = useUserStore();
const [searchText, setSearchText] = useState('');
/* 加载模型 */
const { isLoading } = useQuery(['loadModels'], () => loadKbList(false));
const handleCreateModel = useCallback(async () => {
setIsLoading(true);
try {
const name = `知识库${myKbList.length + 1}`;
const id = await postCreateKb({ name });
await loadKbList(true);
toast({
title: '创建成功',
status: 'success'
});
router.replace(`/kb?kbId=${id}`);
} catch (err: any) {
toast({
title: typeof err === 'string' ? err : err.message || '出现了意外',
status: 'error'
});
}
setIsLoading(false);
}, [loadKbList, myKbList.length, router, setIsLoading, toast]);
return (
<Flex
position={'relative'}
flexDirection={'column'}
w={'100%'}
h={'100%'}
bg={'white'}
borderRight={['', theme.borders.base]}
>
<Flex w={'90%'} my={5} mx={'auto'}>
<Flex flex={1} mr={2} position={'relative'} alignItems={'center'}>
<Input
h={'32px'}
placeholder="搜索知识库"
value={searchText}
onChange={(e) => setSearchText(e.target.value)}
/>
{searchText && (
<MyIcon
zIndex={10}
position={'absolute'}
right={3}
name={'closeSolid'}
w={'16px'}
h={'16px'}
color={'myGray.500'}
cursor={'pointer'}
onClick={() => setSearchText('')}
/>
)}
</Flex>
<Tooltip label={'新建一个知识库'}>
<IconButton
h={'32px'}
icon={<AddIcon />}
aria-label={''}
variant={'outline'}
onClick={handleCreateModel}
/>
</Tooltip>
</Flex>
<Box flex={'1 0 0'} h={0} overflow={'overlay'}>
{myKbList.map((item) => (
<Flex
key={item._id}
position={'relative'}
alignItems={['flex-start', 'center']}
p={3}
mb={[2, 0]}
cursor={'pointer'}
transition={'background-color .2s ease-in'}
borderLeft={['', '5px solid transparent']}
_hover={{
backgroundColor: ['', '#dee0e3']
}}
{...(kbId === item._id
? {
backgroundColor: '#eff0f1',
borderLeftColor: 'myBlue.600 !important'
}
: {})}
onClick={() => {
if (item._id === kbId) return;
router.push(`/kb?kbId=${item._id}`);
}}
>
<Image
src={item.avatar || '/icon/logo.png'}
alt=""
w={'34px'}
maxH={'50px'}
objectFit={'contain'}
/>
<Box flex={'1 0 0'} w={0} ml={3}>
<Box className="textEllipsis" color={'myGray.1000'}>
{item.name}
</Box>
{/* tags */}
<Box className="textEllipsis" color={'myGray.400'} mt={1} fontSize={'sm'}>
{!item.tags ? (
<>{item.tags || '你还没设置标签~'}</>
) : (
item.tags.split(' ').map((item, i) => (
<Tag key={i} mr={2} mb={2} variant={'outline'} colorScheme={'blue'} size={'sm'}>
{item}
</Tag>
))
)}
</Box>
</Box>
</Flex>
))}
{!isLoading && myKbList.length === 0 && (
<Flex h={'100%'} flexDirection={'column'} alignItems={'center'} pt={'30vh'}>
<MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
<Box mt={2} color={'myGray.500'}>
知识库空空如也~
</Box>
</Flex>
)}
</Box>
<Loading loading={isLoading} fixed={false} />
</Flex>
);
};
export default KbList;
src/pages/model/components/detail/components/SelectCsvModal.tsx → src/pages/kb/components/SelectCsvModal.tsx
......@@ -15,7 +15,7 @@ import { useSelectFile } from '@/hooks/useSelectFile';
import { useConfirm } from '@/hooks/useConfirm';
import { readCsvContent } from '@/utils/file';
import { useMutation } from '@tanstack/react-query';
import { postModelDataCsvData } from '@/api/model';
import { postKbDataFromList } from '@/api/plugins/kb';
import Markdown from '@/components/Markdown';
import { useMarkdown } from '@/hooks/useMarkdown';
import { fileDownload } from '@/utils/file';
......@@ -25,16 +25,16 @@ const csvTemplate = `question,answer\n"什么是 laf","laf 是一个云函数开
const SelectJsonModal = ({
onClose,
onSuccess,
modelId
kbId
}: {
onClose: () => void;
onSuccess: () => void;
modelId: string;
kbId: string;
}) => {
const [selecting, setSelecting] = useState(false);
const { toast } = useToast();
const { File, onOpen } = useSelectFile({ fileType: '.csv', multiple: false });
const [fileData, setFileData] = useState<string[][]>([]);
const [fileData, setFileData] = useState<{ q: string; a: string }[]>([]);
const { openConfirm, ConfirmChild } = useConfirm({
content: '确认导入该数据集?'
});
......@@ -48,7 +48,12 @@ const SelectJsonModal = ({
if (header[0] !== 'question' || header[1] !== 'answer') {
throw new Error('csv 文件格式有误');
}
setFileData(data);
setFileData(
data.map((item) => ({
q: item[0] || '',
a: item[1] || ''
}))
);
} catch (error: any) {
console.log(error);
toast({
......@@ -63,8 +68,13 @@ const SelectJsonModal = ({
const { mutate, isLoading } = useMutation({
mutationFn: async () => {
if (!fileData) return;
const res = await postModelDataCsvData(modelId, fileData);
if (!fileData || fileData.length === 0) return;
const res = await postKbDataFromList({
kbId,
data: fileData
});
toast({
title: `导入数据成功，最终导入: ${res || 0} 条数据。需要一段时间训练`,
status: 'success',
......@@ -120,10 +130,10 @@ const SelectJsonModal = ({
{fileData.map((item, index) => (
<Box key={index}>
<Box>
Q{index + 1}. {item[0]}
Q{index + 1}. {item.q}
</Box>
<Box>
A{index + 1}. {item[1]}
A{index + 1}. {item.a}
</Box>
</Box>
))}
......
src/pages/model/components/detail/components/SelectFileModal.tsx → src/pages/kb/components/SelectFileModal.tsx
......@@ -17,10 +17,10 @@ import { useSelectFile } from '@/hooks/useSelectFile';
import { useConfirm } from '@/hooks/useConfirm';
import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
import { useMutation } from '@tanstack/react-query';
import { postModelDataSplitData } from '@/api/model';
import { formatPrice } from '@/utils/user';
import { postSplitData } from '@/api/plugins/kb';
import Radio from '@/components/Radio';
import { splitText_token } from '@/utils/file';
import { SplitTextTypEnum } from '@/constants/plugin';
const fileExtension = '.txt,.doc,.docx,.pdf,.md';
......@@ -42,17 +42,17 @@ const modeMap = {
const SelectFileModal = ({
onClose,
onSuccess,
modelId
kbId
}: {
onClose: () => void;
onSuccess: () => void;
modelId: string;
kbId: string;
}) => {
const [btnLoading, setBtnLoading] = useState(false);
const { toast } = useToast();
const [prompt, setPrompt] = useState('');
const { File, onOpen } = useSelectFile({ fileType: fileExtension, multiple: true });
const [mode, setMode] = useState<'qa' | 'subsection'>('qa');
const [mode, setMode] = useState<`${SplitTextTypEnum}`>(SplitTextTypEnum.subsection);
const [fileTextArr, setFileTextArr] = useState<string[]>(['']);
const [splitRes, setSplitRes] = useState<{ tokens: number; chunks: string[] }>({
tokens: 0,
......@@ -107,8 +107,8 @@ const SelectFileModal = ({
mutationFn: async () => {
if (splitRes.chunks.length === 0) return;
await postModelDataSplitData({
modelId,
await postSplitData({
kbId,
chunks: splitRes.chunks,
prompt: `下面是"${prompt || '一段长文本'}"`,
mode
......
src/pages/kb/index.tsx 0 → 100644
import React, { useEffect } from 'react';
import { Box, Flex } from '@chakra-ui/react';
import { useGlobalStore } from '@/store/global';
import { useRouter } from 'next/router';
import { useUserStore } from '@/store/user';
import SideBar from '@/components/SideBar';
import KbList from './components/KbList';
import KbDetail from './components/Detail';
const Kb = ({ kbId }: { kbId: string }) => {
const router = useRouter();
const { isPc } = useGlobalStore();
const { lastKbId } = useUserStore();
// redirect
useEffect(() => {
if (isPc && !kbId && lastKbId) {
router.replace(`/kb?kbId=${lastKbId}`);
}
}, [isPc, kbId, lastKbId, router]);
return (
<Flex h={'100%'} position={'relative'} overflow={'hidden'}>
{/* 模型列表 */}
{(isPc || !kbId) && (
<SideBar w={[1, '0 0 250px', '0 0 270px', '0 0 290px']}>
<KbList kbId={kbId} />
</SideBar>
)}
<Box flex={1} h={'100%'} position={'relative'}>
{kbId && <KbDetail kbId={kbId} />}
</Box>
</Flex>
);
};
export default Kb;
Kb.getInitialProps = ({ query, req }: any) => {
return {
kbId: query?.kbId || ''
};
};
src/pages/login/components/ForgetPasswordForm.tsx
......@@ -5,7 +5,6 @@ import { PageTypeEnum } from '../../../constants/user';
import { postFindPassword } from '@/api/user';
import { useSendCode } from '@/hooks/useSendCode';
import type { ResLogin } from '@/api/response/user';
import { useScreen } from '@/hooks/useScreen';
import { useToast } from '@/hooks/useToast';
interface Props {
......@@ -22,7 +21,6 @@ interface RegisterType {
const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
const { toast } = useToast();
const { mediaLgMd } = useScreen();
const {
register,
handleSubmit,
......@@ -81,7 +79,7 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
<FormControl mt={5} isInvalid={!!errors.username}>
<Input
placeholder="邮箱/手机号"
size={mediaLgMd}
size={['md', 'lg']}
{...register('username', {
required: '邮箱/手机号不能为空',
pattern: {
......@@ -100,7 +98,7 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
<Input
flex={1}
placeholder="验证码"
size={mediaLgMd}
size={['md', 'lg']}
{...register('code', {
required: '验证码不能为空'
})}
......@@ -109,7 +107,7 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
ml={5}
w={'145px'}
maxW={'50%'}
size={mediaLgMd}
size={['md', 'lg']}
onClick={onclickSendCode}
isDisabled={codeCountDown > 0}
isLoading={codeSending}
......@@ -125,7 +123,7 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
<Input
type={'password'}
placeholder="新密码"
size={mediaLgMd}
size={['md', 'lg']}
{...register('password', {
required: '密码不能为空',
minLength: {
......@@ -146,7 +144,7 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
<Input
type={'password'}
placeholder="确认密码"
size={mediaLgMd}
size={['md', 'lg']}
{...register('password2', {
validate: (val) => (getValues('password') === val ? true : '两次密码不一致')
})}
......@@ -170,7 +168,7 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
type="submit"
mt={5}
w={'100%'}
size={mediaLgMd}
size={['md', 'lg']}
colorScheme="blue"
isLoading={requesting}
>
......
src/pages/login/components/LoginForm.tsx
......@@ -5,7 +5,6 @@ import { PageTypeEnum } from '@/constants/user';
import { postLogin } from '@/api/user';
import type { ResLogin } from '@/api/response/user';
import { useToast } from '@/hooks/useToast';
import { useScreen } from '@/hooks/useScreen';
interface Props {
setPageType: Dispatch<`${PageTypeEnum}`>;
......@@ -19,7 +18,6 @@ interface LoginFormType {
const LoginForm = ({ setPageType, loginSuccess }: Props) => {
const { toast } = useToast();
const { mediaLgMd } = useScreen();
const {
register,
handleSubmit,
......@@ -62,7 +60,7 @@ const LoginForm = ({ setPageType, loginSuccess }: Props) => {
<FormControl mt={8} isInvalid={!!errors.username}>
<Input
placeholder="邮箱/手机号"
size={mediaLgMd}
size={['md', 'lg']}
{...register('username', {
required: '邮箱/手机号不能为空',
pattern: {
......@@ -79,7 +77,7 @@ const LoginForm = ({ setPageType, loginSuccess }: Props) => {
<FormControl mt={8} isInvalid={!!errors.password}>
<Input
type={'password'}
size={mediaLgMd}
size={['md', 'lg']}
placeholder="密码"
{...register('password', {
required: '密码不能为空',
......@@ -119,7 +117,7 @@ const LoginForm = ({ setPageType, loginSuccess }: Props) => {
type="submit"
mt={8}
w={'100%'}
size={mediaLgMd}
size={['md', 'lg']}
colorScheme="blue"
isLoading={requesting}
>
......
src/pages/login/components/RegisterForm.tsx
......@@ -5,7 +5,6 @@ import { PageTypeEnum } from '@/constants/user';
import { postRegister } from '@/api/user';
import { useSendCode } from '@/hooks/useSendCode';
import type { ResLogin } from '@/api/response/user';
import { useScreen } from '@/hooks/useScreen';
import { useToast } from '@/hooks/useToast';
import { useRouter } from 'next/router';
import { postCreateModel } from '@/api/model';
......@@ -25,7 +24,6 @@ interface RegisterType {
const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
const { inviterId = '' } = useRouter().query as { inviterId: string };
const { toast } = useToast();
const { mediaLgMd } = useScreen();
const {
register,
handleSubmit,
......@@ -89,7 +87,7 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
<FormControl mt={5} isInvalid={!!errors.username}>
<Input
placeholder="邮箱/手机号"
size={mediaLgMd}
size={['md', 'lg']}
{...register('username', {
required: '邮箱/手机号不能为空',
pattern: {
......@@ -107,7 +105,7 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
<Flex>
<Input
flex={1}
size={mediaLgMd}
size={['md', 'lg']}
placeholder="验证码"
{...register('code', {
required: '验证码不能为空'
......@@ -117,7 +115,7 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
ml={5}
w={'145px'}
maxW={'50%'}
size={mediaLgMd}
size={['md', 'lg']}
onClick={onclickSendCode}
isDisabled={codeCountDown > 0}
isLoading={codeSending}
......@@ -133,7 +131,7 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
<Input
type={'password'}
placeholder="密码"
size={mediaLgMd}
size={['md', 'lg']}
{...register('password', {
required: '密码不能为空',
minLength: {
......@@ -154,7 +152,7 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
<Input
type={'password'}
placeholder="确认密码"
size={mediaLgMd}
size={['md', 'lg']}
{...register('password2', {
validate: (val) => (getValues('password') === val ? true : '两次密码不一致')
})}
......@@ -178,7 +176,7 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
type="submit"
mt={5}
w={'100%'}
size={mediaLgMd}
size={['md', 'lg']}
colorScheme="blue"
isLoading={requesting}
>
......
src/pages/login/index.tsx
......@@ -2,7 +2,7 @@ import React, { useState, useCallback, useEffect } from 'react';
import styles from './index.module.scss';
import { Box, Flex, Image } from '@chakra-ui/react';
import { PageTypeEnum } from '@/constants/user';
import { useScreen } from '@/hooks/useScreen';
import { useGlobalStore } from '@/store/global';
import type { ResLogin } from '@/api/response/user';
import { useRouter } from 'next/router';
import { useUserStore } from '@/store/user';
......@@ -12,10 +12,10 @@ import dynamic from 'next/dynamic';
const RegisterForm = dynamic(() => import('./components/RegisterForm'));
const ForgetPasswordForm = dynamic(() => import('./components/ForgetPasswordForm'));
const Login = ({ isPcDevice }: { isPcDevice: boolean }) => {
const Login = () => {
const router = useRouter();
const { lastRoute = '' } = router.query as { lastRoute: string };
const { isPc } = useScreen({ defaultIsPc: isPcDevice });
const { isPc } = useGlobalStore();
const [pageType, setPageType] = useState<`${PageTypeEnum}`>(PageTypeEnum.login);
const { setUserInfo, setLastModelId, loadMyModels } = useUserStore();
const { setLastChatId, setLastChatModelId, loadHistory } = useChatStore();
......@@ -114,9 +114,3 @@ const Login = ({ isPcDevice }: { isPcDevice: boolean }) => {
};
export default Login;
Login.getInitialProps = ({ query, req }: any) => {
return {
isPcDevice: !/Mobile/.test(req?.headers?.['user-agent'])
};
};
src/pages/model/components/ModelList.tsx
......@@ -126,7 +126,7 @@ const ModelList = ({ modelId }: { modelId: string }) => {
{...(modelId === item._id
? {
backgroundColor: '#eff0f1',
borderLeftColor: 'myBlue.600'
borderLeftColor: 'myBlue.600 !important'
}
: {})}
onClick={() => {
......
src/pages/model/components/detail/components/SelectUrlModal.tsx deleted 100644 → 0
import React, { useState } from 'react';
import {
Box,
Flex,
Button,
Modal,
ModalOverlay,
ModalContent,
ModalHeader,
ModalCloseButton,
ModalBody,
Input,
Textarea
} from '@chakra-ui/react';
import { useToast } from '@/hooks/useToast';
import { useConfirm } from '@/hooks/useConfirm';
import { useMutation } from '@tanstack/react-query';
import { postModelDataSplitData, getWebContent } from '@/api/model';
import { formatPrice } from '@/utils/user';
const SelectUrlModal = ({
onClose,
onSuccess,
modelId
}: {
onClose: () => void;
onSuccess: () => void;
modelId: string;
}) => {
const { toast } = useToast();
const [webUrl, setWebUrl] = useState('');
const [webText, setWebText] = useState('');
const [prompt, setPrompt] = useState(''); // 提示词
const { openConfirm, ConfirmChild } = useConfirm({
content: '确认导入该文件，需要一定时间进行拆解，该任务无法终止！如果余额不足，任务讲被终止。'
});
const { mutate: onclickImport, isLoading: isImporting } = useMutation({
mutationFn: async () => {
if (!webText) return;
await postModelDataSplitData({
modelId,
chunks: [],
prompt: `下面是"${prompt || '一段长文本'}"`,
mode: 'qa'
});
toast({
title: '导入数据成功,需要一段拆解和训练',
status: 'success'
});
onClose();
onSuccess();
},
onError(error) {
console.log(error);
toast({
title: '导入数据失败',
status: 'error'
});
}
});
const { mutate: onclickFetchingUrl, isLoading: isFetching } = useMutation({
mutationFn: async () => {
if (!webUrl) return;
const res = await getWebContent(webUrl);
const parser = new DOMParser();
const htmlDoc = parser.parseFromString(res, 'text/html');
const data = htmlDoc?.body?.innerText || '';
if (!data) {
throw new Error('获取不到数据');
}
setWebText(data.replace(/\s+/g, ' '));
},
onError(error) {
console.log(error);
toast({
status: 'error',
title: '获取网站内容失败'
});
}
});
return (
<Modal isOpen={true} onClose={onClose} isCentered>
<ModalOverlay />
<ModalContent maxW={'min(900px, 90vw)'} m={0} position={'relative'} h={'90vh'}>
<ModalHeader>静态网站内容导入</ModalHeader>
<ModalCloseButton />
<ModalBody
display={'flex'}
flexDirection={'column'}
p={4}
h={'100%'}
alignItems={'center'}
justifyContent={'center'}
fontSize={'sm'}
>
<Box mt={2} maxW={['100%', '70%']}>
根据网站地址，获取网站文本内容（请注意仅能获取静态网站文本，注意看下获取后的内容是否正确）。Gpt会对文本进行
QA 拆分，需要较长训练时间，拆分需要消耗 tokens，账号余额不足时，未拆分的数据会被删除。
</Box>
<Flex w={'100%'} alignItems={'center'} my={4}>
<Box flex={'0 0 70px'}>网站地址</Box>
<Input
mx={2}
placeholder="需要获取内容的地址。例如：https://fastgpt.ahapocket.cn"
value={webUrl}
onChange={(e) => setWebUrl(e.target.value)}
size={'sm'}
/>
<Button isLoading={isFetching} onClick={() => onclickFetchingUrl()}>
获取
</Button>
</Flex>
<Flex w={'100%'} alignItems={'center'} my={4}>
<Box flex={'0 0 70px'} mr={2}>
下面是
</Box>
<Input
placeholder="内容提示词。例如: Laf的介绍/关于gpt4的论文/一段长文本"
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
size={'sm'}
/>
</Flex>
<Textarea
flex={'1 0 0'}
h={0}
w={'100%'}
placeholder="网站的内容"
maxLength={-1}
resize={'none'}
fontSize={'xs'}
whiteSpace={'pre-wrap'}
value={webText}
onChange={(e) => setWebText(e.target.value)}
/>
</ModalBody>
<Flex px={6} pt={2} pb={4}>
<Box flex={1}></Box>
<Button variant={'outline'} mr={3} onClick={onClose}>
取消
</Button>
<Button
isLoading={isImporting}
isDisabled={webText === ''}
onClick={openConfirm(onclickImport)}
>
确认导入
</Button>
</Flex>
</ModalContent>
<ConfirmChild />
</Modal>
);
};
export default SelectUrlModal;
src/pages/model/components/detail/index.tsx
......@@ -10,7 +10,6 @@ import { useQuery } from '@tanstack/react-query';
import { useUserStore } from '@/store/user';
import { useLoading } from '@/hooks/useLoading';
import ModelEditForm from './components/ModelEditForm';
import ModelDataCard from './components/ModelDataCard';
const ModelDetail = ({ modelId, isPc }: { modelId: string; isPc: boolean }) => {
const { toast } = useToast();
......@@ -194,10 +193,6 @@ const ModelDetail = ({ modelId, isPc }: { modelId: string; isPc: boolean }) => {
</Card>
<Grid mt={5} gridTemplateColumns={['1fr', '1fr 1fr']} gridGap={5}>
<ModelEditForm formHooks={formHooks} handleDelModel={handleDelModel} isOwner={isOwner} />
<Card p={4} gridColumnStart={[1, 1]} gridColumnEnd={[2, 3]}>
<ModelDataCard modelId={modelId} isOwner={isOwner} />
</Card>
</Grid>
<Loading loading={isLoading} fixed={false} />
</Box>
......
src/pages/model/index.tsx
import React, { useEffect } from 'react';
import { Box, Flex } from '@chakra-ui/react';
import { useScreen } from '@/hooks/useScreen';
import { useRouter } from 'next/router';
import ModelList from './components/ModelList';
import dynamic from 'next/dynamic';
import { useUserStore } from '@/store/user';
import { useGlobalStore } from '@/store/global';
import Loading from '@/components/Loading';
import SideBar from '@/components/SideBar';
const ModelDetail = dynamic(() => import('./components/detail/index'), {
loading: () => <Loading fixed={false} />,
ssr: false
});
const Model = ({ modelId, isPcDevice }: { modelId: string; isPcDevice: boolean }) => {
const Model = ({ modelId }: { modelId: string }) => {
const router = useRouter();
const { isPc } = useScreen({
defaultIsPc: isPcDevice
});
const { isPc } = useGlobalStore();
const { lastModelId } = useUserStore();
// redirect modelId
......@@ -27,12 +26,12 @@ const Model = ({ modelId, isPcDevice }: { modelId: string; isPcDevice: boolean }
}, [isPc, lastModelId, modelId, router]);
return (
<Flex h={'100%'} position={'relative'}>
<Flex h={'100%'} position={'relative'} overflow={'hidden'}>
{/* 模型列表 */}
{(isPc || !modelId) && (
<Box w={['100%', '250px']}>
<SideBar w={[1, '0 0 250px', '0 0 270px', '0 0 290px']}>
<ModelList modelId={modelId} />
</Box>
</SideBar>
)}
<Box flex={1} h={'100%'} position={'relative'}>
{modelId && <ModelDetail modelId={modelId} isPc={isPc} />}
......@@ -45,7 +44,6 @@ export default Model;
Model.getInitialProps = ({ query, req }: any) => {
return {
modelId: query?.modelId || '',
isPcDevice: !/Mobile/.test(req?.headers?.['user-agent'])
modelId: query?.modelId || ''
};
};
src/service/errorCode.ts
......@@ -37,7 +37,8 @@ export const proxyError: Record<string, boolean> = {
export enum ERROR_ENUM {
unAuthorization = 'unAuthorization',
insufficientQuota = 'insufficientQuota',
unAuthModel = 'unAuthModel'
unAuthModel = 'unAuthModel',
unAuthKb = 'unAuthKb'
}
export const ERROR_RESPONSE: Record<
any,
......@@ -65,5 +66,11 @@ export const ERROR_RESPONSE: Record<
statusText: ERROR_ENUM.unAuthModel,
message: '无权使用该模型',
data: null
},
[ERROR_ENUM.unAuthKb]: {
code: 512,
statusText: ERROR_ENUM.unAuthKb,
message: '无权使用该知识库',
data: null
}
};
src/service/events/generateQA.ts
......@@ -5,7 +5,7 @@ import { pushSplitDataBill } from '@/service/events/pushBill';
import { generateVector } from './generateVector';
import { openaiError2 } from '../errorCode';
import { PgClient } from '@/service/pg';
import { ModelSplitDataSchema } from '@/types/mongoSchema';
import { SplitDataSchema } from '@/types/mongoSchema';
import { modelServiceToolMap } from '../utils/chat';
import { ChatRoleEnum } from '@/constants/chat';
import { getErrMessage } from '../utils/tools';
......@@ -32,7 +32,7 @@ export async function generateQA(next = false): Promise<any> {
{ $sample: { size: 1 } }
]);
const dataItem: ModelSplitDataSchema = data[0];
const dataItem: SplitDataSchema = data[0];
if (!dataItem) {
console.log('没有需要生成 QA 的数据');
......@@ -127,14 +127,15 @@ A2:
const resultList = successResponse.map((item) => item.result).flat();
await Promise.allSettled([
// 删掉后5个数据
SplitData.findByIdAndUpdate(dataItem._id, {
textList: dataItem.textList.slice(0, -5)
}), // 删掉后5个数据
}),
// 生成的内容插入 pg
PgClient.insert('modelData', {
values: resultList.map((item) => [
{ key: 'user_id', value: dataItem.userId },
{ key: 'model_id', value: dataItem.modelId },
{ key: 'kb_id', value: dataItem.kbId },
{ key: 'q', value: item.q },
{ key: 'a', value: item.a },
{ key: 'status', value: 'waiting' }
......
src/service/models/kb.ts 0 → 100644
import { Schema, model, models, Model } from 'mongoose';
import { kbSchema as SchemaType } from '@/types/mongoSchema';
const kbSchema = new Schema({
userId: {
type: Schema.Types.ObjectId,
ref: 'user',
required: true
},
updateTime: {
type: Date,
default: () => new Date()
},
avatar: {
type: String,
default: '/icon/logo.png'
},
name: {
type: String,
required: true
},
tags: {
type: [String],
default: []
}
});
export const KB: Model<SchemaType> = models['kb'] || model('kb', kbSchema);
src/service/models/splitData.ts
/* 模型的知识库 */
import { Schema, model, models, Model as MongoModel } from 'mongoose';
import { ModelSplitDataSchema as SplitDataType } from '@/types/mongoSchema';
import { SplitDataSchema as SplitDataType } from '@/types/mongoSchema';
const SplitDataSchema = new Schema({
userId: {
......@@ -13,9 +13,9 @@ const SplitDataSchema = new Schema({
type: String,
required: true
},
modelId: {
kbId: {
type: Schema.Types.ObjectId,
ref: 'model',
ref: 'kb',
required: true
},
textList: {
......
src/service/mongo.ts
......@@ -52,3 +52,4 @@ export * from './models/openapi';
export * from './models/promotionRecord';
export * from './models/collection';
export * from './models/shareChat';
export * from './models/kb';
src/service/utils/auth.ts
import type { NextApiRequest } from 'next';
import jwt from 'jsonwebtoken';
import cookie from 'cookie';
import { Chat, Model, OpenApi, User, ShareChat } from '../mongo';
import { Chat, Model, OpenApi, User, ShareChat, KB } from '../mongo';
import type { ModelSchema } from '@/types/mongoSchema';
import type { ChatItemSimpleType } from '@/types/chat';
import mongoose from 'mongoose';
......@@ -129,6 +129,18 @@ export const authModel = async ({
return { model, showModelDetail: model.share.isShareDetail || userId === String(model.userId) };
};
// 知识库操作权限
export const authKb = async ({ kbId, userId }: { kbId: string; userId: string }) => {
const kb = await KB.findOne({
_id: kbId,
userId
});
if (kb) {
return kb;
}
return Promise.reject(ERROR_ENUM.unAuthKb);
};
// 获取对话校验
export const authChat = async ({
modelId,
......
src/store/global.ts
......@@ -5,6 +5,9 @@ import { immer } from 'zustand/middleware/immer';
type State = {
loading: boolean;
setLoading: (val: boolean) => null;
screenWidth: number;
setScreenWidth: (val: number) => void;
isPc: boolean;
};
export const useGlobalStore = create<State>()(
......@@ -16,7 +19,15 @@ export const useGlobalStore = create<State>()(
state.loading = val;
});
return null;
}
},
screenWidth: 600,
setScreenWidth(val: number) {
set((state) => {
state.screenWidth = val;
state.isPc = val < 900 ? false : true;
});
},
isPc: false
}))
)
);
src/store/user.ts
......@@ -8,12 +8,16 @@ import { formatPrice } from '@/utils/user';
import { getTokenLogin } from '@/api/user';
import { defaultModel } from '@/constants/model';
import { ModelListItemType } from '@/types/model';
import { KbItemType } from '@/types/plugin';
import { getKbList } from '@/api/plugins/kb';
import { defaultKbDetail } from '@/constants/kb';
type State = {
userInfo: UserType | null;
initUserInfo: () => Promise<null>;
setUserInfo: (user: UserType | null) => void;
updateUserInfo: (user: UserUpdateParams) => void;
// model
lastModelId: string;
setLastModelId: (id: string) => void;
myModels: ModelListItemType[];
......@@ -26,6 +30,13 @@ type State = {
updateModelDetail(model: ModelSchema): void;
removeModelDetail(modelId: string): void;
};
// kb
lastKbId: string;
setLastKbId: (id: string) => void;
myKbList: KbItemType[];
loadKbList: (init?: boolean) => Promise<null>;
KbDetail: KbItemType;
getKbDetail: (id: string) => KbItemType;
};
export const useUserStore = create<State>()(
......@@ -103,12 +114,38 @@ export const useUserStore = create<State>()(
}
get().loadMyModels(true);
}
},
lastKbId: '',
setLastKbId(id: string) {
set((state) => {
state.lastKbId = id;
});
},
myKbList: [],
async loadKbList(init = false) {
if (get().myKbList.length > 0 && !init) return null;
const res = await getKbList();
set((state) => {
state.myKbList = res;
});
return null;
},
KbDetail: defaultKbDetail,
getKbDetail(id: string) {
const data = get().myKbList.find((item) => item._id === id) || defaultKbDetail;
set((state) => {
state.KbDetail = data;
});
return data;
}
})),
{
name: 'userStore',
partialize: (state) => ({
lastModelId: state.lastModelId
lastModelId: state.lastModelId,
lastKbId: state.lastKbId
})
}
)
......
src/types/model.d.ts
......@@ -16,15 +16,6 @@ export interface ModelUpdateParams {
security: ModelSchema['security'];
}
export interface ModelDataItemType {
id: string;
status: 'waiting' | 'ready';
q: string; // 提问词
a: string; // 原文
modelId: string;
userId: string;
}
export interface ShareModelItem {
_id: string;
avatar: string;
......
src/types/mongoSchema.d.ts
......@@ -69,19 +69,11 @@ export interface CollectionSchema {
}
export type ModelDataType = 0 | 1;
export interface ModelDataSchema {
_id: string;
modelId: string;
userId: string;
a: string;
q: string;
status: ModelDataType;
}
export interface ModelSplitDataSchema {
export interface SplitDataSchema {
_id: string;
userId: string;
modelId: string;
kbId: string;
prompt: string;
errorText: string;
textList: string[];
......@@ -148,3 +140,12 @@ export interface ShareChatSchema {
maxContext: number;
lastTime: Date;
}
export interface kbSchema {
_id: string;
userId: string;
updateTime: Date;
avatar: string;
name: string;
tags: string[];
}
src/types/pg.d.ts
import { ModelDataStatusEnum } from '@/constants/model';
export interface PgModelDataItemType {
export interface PgKBDataItemType {
id: string;
q: string;
a: string;
status: `${ModelDataStatusEnum}`;
model_id: string;
user_id: string;
kb_id: string;
}
src/types/plugin.d.ts 0 → 100644
import type { kbSchema } from './mongoSchema';
/* kb type */
export interface KbItemType extends kbSchema {
totalData: number;
tags: string;
}
export interface KbDataItemType {
id: string;
status: 'waiting' | 'ready';
q: string; // 提问词
a: string; // 原文
kbId: string;
userId: string;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment