Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit a19afca1 by Archer Committed by GitHub
Options

v4.4.1 (#294) 

* move file

* perf: dataset file manage

* v441 description

* fix: qa csv update file

* feat: rename file

* frontend show system-version
parent be3b680b
Showing with 570 additions and 301 deletions
client/package.json
{ {
"name": "fastgpt", "name": "fastgpt",
"version": "3.7", "version": "4.2.1",
"private": true, "private": false,
"scripts": { "scripts": {
"dev": "next dev", "dev": "next dev",
"build": "next build", "build": "next build",
......
client/public/locales/en/common.json
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
"Confirm": "Yes", "Confirm": "Yes",
"Create New": "Create", "Create New": "Create",
"Dataset": "Dataset", "Dataset": "Dataset",
"Export": "Export",
"Folder": "Folder", "Folder": "Folder",
"Move": "Move", "Move": "Move",
"Name": "Name", "Name": "Name",
...@@ -215,6 +216,9 @@ ...@@ -215,6 +216,9 @@
"Response Detail": "Detail", "Response Detail": "Detail",
"Response Detail tips": "Whether detailed data such as references and full context need to be returned" "Response Detail tips": "Whether detailed data such as references and full context need to be returned"
}, },
"system": {
"Help Document": "Document"
},
"user": { "user": {
"Account": "Account", "Account": "Account",
"Amount of earnings": "Earnings", "Amount of earnings": "Earnings",
......
client/public/locales/zh/common.json
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
"Confirm": "确认", "Confirm": "确认",
"Create New": "新建", "Create New": "新建",
"Dataset": "知识库", "Dataset": "知识库",
"Export": "导出",
"Folder": "文件夹", "Folder": "文件夹",
"Move": "移动", "Move": "移动",
"Name": "名称", "Name": "名称",
...@@ -215,6 +216,9 @@ ...@@ -215,6 +216,9 @@
"Response Detail": "返回详情", "Response Detail": "返回详情",
"Response Detail tips": "是否需要返回引用、完整上下文等详细数据" "Response Detail tips": "是否需要返回引用、完整上下文等详细数据"
}, },
"system": {
"Help Document": "帮助文档"
},
"user": { "user": {
"Account": "账号", "Account": "账号",
"Amount of earnings": "收益（￥）", "Amount of earnings": "收益（￥）",
......
client/src/api/core/dataset/file.d.ts 0 → 100644
import { RequestPaging } from '../../../types/index';
export type GetFileListProps = RequestPaging & {
kbId: string;
searchText: string;
};
export type UpdateFileProps = { id: string; name?: string; datasetUsed?: boolean };
client/src/api/core/dataset/file.ts 0 → 100644
import { GET, POST, PUT, DELETE } from '@/api/request';
import type { FileInfo, KbFileItemType } from '@/types/plugin';
import type { GetFileListProps, UpdateFileProps } from './file.d';
export const getDatasetFiles = (data: GetFileListProps) =>
POST<KbFileItemType[]>(`/core/dataset/file/list`, data);
export const delDatasetFileById = (params: { fileId: string; kbId: string }) =>
DELETE(`/core/dataset/file/delById`, params);
export const getFileInfoById = (fileId: string) =>
GET<FileInfo>(`/core/dataset/file/detail`, { fileId });
export const delDatasetEmptyFiles = (kbId: string) =>
DELETE(`/core/dataset/file/delEmptyFiles`, { kbId });
export const updateDatasetFile = (data: UpdateFileProps) => PUT(`/core/dataset/file/update`, data);
client/src/api/plugins/kb.ts
import { GET, POST, PUT, DELETE } from '../request'; import { GET, POST, PUT, DELETE } from '../request';
import type { import type { DatasetItemType, KbItemType, KbListItemType, KbPathItemType } from '@/types/plugin';
DatasetItemType,
FileInfo,
KbFileItemType,
KbItemType,
KbListItemType,
KbPathItemType
} from '@/types/plugin';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import { import {
Props as PushDataProps, Props as PushDataProps,
...@@ -17,12 +10,7 @@ import { ...@@ -17,12 +10,7 @@ import {
Response as SearchTestResponse Response as SearchTestResponse
} from '@/pages/api/openapi/kb/searchTest'; } from '@/pages/api/openapi/kb/searchTest';
import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData'; import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData';
import type { import type { KbUpdateParams, CreateKbParams, GetKbDataListProps } from '../request/kb';
KbUpdateParams,
CreateKbParams,
GetKbDataListProps,
GetFileListProps
} from '../request/kb';
import { QuoteItemType } from '@/types/chat'; import { QuoteItemType } from '@/types/chat';
import { KbTypeEnum } from '@/constants/kb'; import { KbTypeEnum } from '@/constants/kb';
...@@ -42,16 +30,6 @@ export const putKbById = (data: KbUpdateParams) => PUT(`/plugins/kb/update`, dat ...@@ -42,16 +30,6 @@ export const putKbById = (data: KbUpdateParams) => PUT(`/plugins/kb/update`, dat
export const delKbById = (id: string) => DELETE(`/plugins/kb/delete?id=${id}`); export const delKbById = (id: string) => DELETE(`/plugins/kb/delete?id=${id}`);
/* kb file */
export const getKbFiles = (data: GetFileListProps) =>
POST<KbFileItemType[]>(`/plugins/kb/file/list`, data);
export const deleteKbFileById = (params: { fileId: string; kbId: string }) =>
DELETE(`/plugins/kb/file/delFileByFileId`, params);
export const getFileInfoById = (fileId: string) =>
GET<FileInfo>(`/plugins/kb/file/getFileInfo`, { fileId });
export const delEmptyFiles = (kbId: string) =>
DELETE(`/plugins/kb/file/deleteEmptyFiles`, { kbId });
/* kb data */ /* kb data */
export const getKbDataList = (data: GetKbDataListProps) => export const getKbDataList = (data: GetKbDataListProps) =>
POST(`/plugins/kb/data/getDataList`, data); POST(`/plugins/kb/data/getDataList`, data);
...@@ -59,7 +37,7 @@ export const getKbDataList = (data: GetKbDataListProps) => ...@@ -59,7 +37,7 @@ export const getKbDataList = (data: GetKbDataListProps) =>
/** /**
* 获取导出数据（不分页） * 获取导出数据（不分页）
*/ */
export const getExportDataList = (data: { kbId: string; fileId: string }) => export const getExportDataList = (data: { kbId: string }) =>
GET<[string, string, string][]>(`/plugins/kb/data/exportModelData`, data, { GET<[string, string, string][]>(`/plugins/kb/data/exportModelData`, data, {
timeout: 600000 timeout: 600000
}); });
......
client/src/api/request/kb.d.ts
...@@ -17,11 +17,6 @@ export type CreateKbParams = { ...@@ -17,11 +17,6 @@ export type CreateKbParams = {
type: `${KbTypeEnum}`; type: `${KbTypeEnum}`;
}; };
export type GetFileListProps = RequestPaging & {
kbId: string;
searchText: string;
};
export type GetKbDataListProps = RequestPaging & { export type GetKbDataListProps = RequestPaging & {
kbId: string; kbId: string;
searchText: string; searchText: string;
......
client/src/api/support/file.ts 0 → 100644
import { GET, POST } from '../request';
import { AxiosProgressEvent } from 'axios';
export const uploadImg = (base64Img: string) => POST<string>('/system/uploadImage', { base64Img });
export const postUploadFiles = (
data: FormData,
onUploadProgress: (progressEvent: AxiosProgressEvent) => void
) =>
POST<string[]>('/support/file/upload', data, {
onUploadProgress,
headers: {
'Content-Type': 'multipart/form-data; charset=utf-8'
}
});
export const getFileViewUrl = (fileId: string) => GET<string>('/support/file/readUrl', { fileId });
client/src/api/system.ts
import { GET, POST, PUT } from './request'; import { GET, POST, PUT } from './request';
import type { InitDateResponse } from '@/pages/api/system/getInitData'; import type { InitDateResponse } from '@/pages/api/system/getInitData';
import { AxiosProgressEvent } from 'axios';
export const getInitData = () => GET<InitDateResponse>('/system/getInitData'); export const getInitData = () => GET<InitDateResponse>('/system/getInitData');
export const uploadImg = (base64Img: string) => POST<string>('/system/uploadImage', { base64Img });
export const postUploadFiles = (
data: FormData,
onUploadProgress: (progressEvent: AxiosProgressEvent) => void
) =>
POST<string[]>('/plugins/file/upload', data, {
onUploadProgress,
headers: {
'Content-Type': 'multipart/form-data; charset=utf-8'
}
});
export const getFileViewUrl = (fileId: string) => GET<string>('/plugins/file/readUrl', { fileId });
client/src/components/Layout/navbar.tsx
...@@ -167,20 +167,7 @@ const Navbar = ({ unread }: { unread: number }) => { ...@@ -167,20 +167,7 @@ const Navbar = ({ unread }: { unread: number }) => {
</Link> </Link>
</Box> </Box>
)} )}
{feConfigs?.show_doc && (
<MyTooltip label={t('home.Docs')} placement={'right-end'}>
<Box
{...itemStyles}
mb={0}
color={'#9096a5'}
onClick={() => {
window.open(`https://doc.fastgpt.run/docs/intro`);
}}
>
<MyIcon name={'courseLight'} width={'26px'} height={'26px'} />
</Box>
</MyTooltip>
)}
<Language {...itemStyles} /> <Language {...itemStyles} />
{feConfigs?.show_git && ( {feConfigs?.show_git && (
<MyTooltip label={`Git Star: ${gitStar}`} placement={'right-end'}> <MyTooltip label={`Git Star: ${gitStar}`} placement={'right-end'}>
......
client/src/components/MyInput/index.tsx
...@@ -8,7 +8,7 @@ interface Props extends InputProps { ...@@ -8,7 +8,7 @@ interface Props extends InputProps {
const MyInput = ({ leftIcon, ...props }: Props) => { const MyInput = ({ leftIcon, ...props }: Props) => {
return ( return (
<Flex position={'relative'} alignItems={'center'}> <Flex position={'relative'} alignItems={'center'}>
<Input w={'100%'} pl={leftIcon ? '30px' : 3} {...props} /> <Input w={'100%'} pl={leftIcon ? '30px !important' : 3} {...props} />
{leftIcon && ( {leftIcon && (
<Flex <Flex
alignItems={'center'} alignItems={'center'}
......
client/src/constants/plugin.ts
...@@ -7,4 +7,4 @@ export const TrainingTypeMap = { ...@@ -7,4 +7,4 @@ export const TrainingTypeMap = {
[TrainingModeEnum.index]: 'index' [TrainingModeEnum.index]: 'index'
}; };
export const PgTrainingTableName = 'modeldata'; export const PgDatasetTableName = 'modeldata';
client/src/hooks/useEditInfo.tsx → client/src/hooks/useEditTitle.tsx
...@@ -2,7 +2,7 @@ import React, { useCallback, useRef } from 'react'; ...@@ -2,7 +2,7 @@ import React, { useCallback, useRef } from 'react';
import { ModalFooter, ModalBody, Input, useDisclosure, Button } from '@chakra-ui/react'; import { ModalFooter, ModalBody, Input, useDisclosure, Button } from '@chakra-ui/react';
import MyModal from '@/components/MyModal'; import MyModal from '@/components/MyModal';
export const useEditInfo = ({ export const useEditTitle = ({
title, title,
placeholder = '' placeholder = ''
}: { }: {
......
client/src/pages/account/components/Info.tsx
...@@ -9,7 +9,7 @@ import { useQuery } from '@tanstack/react-query'; ...@@ -9,7 +9,7 @@ import { useQuery } from '@tanstack/react-query';
import dynamic from 'next/dynamic'; import dynamic from 'next/dynamic';
import { useSelectFile } from '@/hooks/useSelectFile'; import { useSelectFile } from '@/hooks/useSelectFile';
import { compressImg } from '@/utils/file'; import { compressImg } from '@/utils/file';
import { feConfigs } from '@/store/static'; import { feConfigs, systemVersion } from '@/store/static';
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
import { timezoneList } from '@/utils/user'; import { timezoneList } from '@/utils/user';
import Loading from '@/components/Loading'; import Loading from '@/components/Loading';
...@@ -173,19 +173,48 @@ const UserInfo = () => { ...@@ -173,19 +173,48 @@ const UserInfo = () => {
</Button> </Button>
</Flex> </Flex>
{feConfigs?.show_userDetail && ( {feConfigs?.show_userDetail && (
<Box mt={6} whiteSpace={'nowrap'} w={['85%', '300px']}>
<Flex alignItems={'center'}>
<Box flex={'0 0 80px'}>{t('user.Balance')}:&nbsp;</Box>
<Box flex={1}>
<strong>{userInfo?.balance.toFixed(3)}</strong> 元
</Box>
<Button size={['sm', 'md']} ml={5} onClick={onOpenPayModal}>
{t('user.Pay')}
</Button>
</Flex>
</Box>
)}
{feConfigs?.show_doc && (
<>
<Flex
mt={4}
w={['85%', '300px']}
py={3}
px={6}
border={theme.borders.sm}
borderWidth={'1.5px'}
borderRadius={'md'}
alignItems={'center'}
cursor={'pointer'}
userSelect={'none'}
onClick={() => {
window.open(`https://doc.fastgpt.run/docs/intro`);
}}
>
<MyIcon name={'courseLight'} w={'18px'} />
<Box ml={2} flex={1}>
{t('system.Help Document')}
</Box>
<Box w={'8px'} h={'8px'} borderRadius={'50%'} bg={'#67c13b'} />
<Box fontSize={'md'} ml={2}>
V{systemVersion}
</Box>
</Flex>
</>
)}
{feConfigs?.show_userDetail && (
<> <>
<Box mt={6} whiteSpace={'nowrap'} w={['85%', '300px']}>
<Flex alignItems={'center'}>
<Box flex={'0 0 80px'}>{t('user.Balance')}:&nbsp;</Box>
<Box flex={1}>
<strong>{userInfo?.balance.toFixed(3)}</strong> 元
</Box>
<Button size={['sm', 'md']} ml={5} onClick={onOpenPayModal}>
{t('user.Pay')}
</Button>
</Flex>
</Box>
<Divider my={3} /> <Divider my={3} />
<MyTooltip label={'点击配置账号'}> <MyTooltip label={'点击配置账号'}>
......
client/src/pages/api/admin/initv43.ts
...@@ -3,7 +3,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; ...@@ -3,7 +3,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
...@@ -12,7 +12,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -12,7 +12,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { rowCount } = await PgClient.query(`SELECT 1 const { rowCount } = await PgClient.query(`SELECT 1
FROM information_schema.columns FROM information_schema.columns
WHERE table_schema = 'public' WHERE table_schema = 'public'
AND table_name = '${PgTrainingTableName}' AND table_name = '${PgDatasetTableName}'
AND column_name = 'file_id'`); AND column_name = 'file_id'`);
if (rowCount > 0) { if (rowCount > 0) {
...@@ -23,7 +23,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -23,7 +23,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
jsonRes(res, { jsonRes(res, {
data: await PgClient.query( data: await PgClient.query(
`ALTER TABLE ${PgTrainingTableName} ADD COLUMN file_id VARCHAR(100)` `ALTER TABLE ${PgDatasetTableName} ADD COLUMN file_id VARCHAR(100)`
) )
}); });
} catch (error) { } catch (error) {
......
client/src/pages/api/admin/initv44.ts
...@@ -5,7 +5,7 @@ import { authUser } from '@/service/utils/auth'; ...@@ -5,7 +5,7 @@ import { authUser } from '@/service/utils/auth';
import { connectToDatabase, KB } from '@/service/mongo'; import { connectToDatabase, KB } from '@/service/mongo';
import { KbTypeEnum } from '@/constants/kb'; import { KbTypeEnum } from '@/constants/kb';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
...@@ -24,7 +24,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -24,7 +24,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
} }
); );
const response = await PgClient.update(PgTrainingTableName, { const response = await PgClient.update(PgDatasetTableName, {
where: [['file_id', 'undefined']], where: [['file_id', 'undefined']],
values: [{ key: 'file_id', value: '' }] values: [{ key: 'file_id', value: '' }]
}); });
......
client/src/pages/api/admin/initv441.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase } from '@/service/mongo';
import mongoose from 'mongoose';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authRoot: true });
const data = await mongoose.connection.db
.collection('dataset.files')
.updateMany({}, { $set: { 'metadata.datasetUsed': true } });
// update pg data
const pg = await PgClient.query(`UPDATE ${PgDatasetTableName}
SET file_id = ''
WHERE (file_id = 'undefined' OR LENGTH(file_id) < 20) AND file_id != '';`);
jsonRes(res, {
data: {
data,
pg
}
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}
client/src/pages/api/plugins/kb/file/delFileByFileId.ts → client/src/pages/api/core/dataset/file/delById.ts
...@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo'; ...@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { Types } from 'mongoose'; import { Types } from 'mongoose';
import { OtherFileId } from '@/constants/kb'; import { OtherFileId } from '@/constants/kb';
...@@ -22,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -22,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
if (fileId === OtherFileId) { if (fileId === OtherFileId) {
await PgClient.delete(PgTrainingTableName, { await PgClient.delete(PgDatasetTableName, {
where: [ where: [
['user_id', userId], ['user_id', userId],
'AND', 'AND',
...@@ -37,7 +37,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -37,7 +37,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await gridFs.findAndAuthFile(fileId); await gridFs.findAndAuthFile(fileId);
// delete all pg data // delete all pg data
await PgClient.delete(PgTrainingTableName, { await PgClient.delete(PgDatasetTableName, {
where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['file_id', fileId]] where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['file_id', fileId]]
}); });
......
client/src/pages/api/core/dataset/file/delEmptyFiles.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { kbId } = req.query as { kbId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const gridFs = new GridFSStorage('dataset', userId);
const collection = gridFs.Collection();
const files = await collection.deleteMany({
uploadDate: { $lte: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) },
['metadata.kbId']: kbId,
['metadata.userId']: userId,
['metadata.datasetUsed']: { $ne: true }
});
jsonRes(res, {
data: files
});
} catch (err) {
jsonRes(res);
}
}
client/src/pages/api/plugins/kb/file/list.ts → client/src/pages/api/core/dataset/file/list.ts
...@@ -4,9 +4,8 @@ import { connectToDatabase, TrainingData } from '@/service/mongo'; ...@@ -4,9 +4,8 @@ import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { FileStatusEnum, OtherFileId } from '@/constants/kb'; import { FileStatusEnum, OtherFileId } from '@/constants/kb';
import mongoose from 'mongoose';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
...@@ -16,28 +15,37 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -16,28 +15,37 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
pageNum = 1, pageNum = 1,
pageSize = 10, pageSize = 10,
kbId, kbId,
searchText searchText = ''
} = req.body as { pageNum: number; pageSize: number; kbId: string; searchText: string }; } = req.body as { pageNum: number; pageSize: number; kbId: string; searchText: string };
searchText = searchText.replace(/'/g, ''); searchText = searchText?.replace(/'/g, '');
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
// find files
const gridFs = new GridFSStorage('dataset', userId); const gridFs = new GridFSStorage('dataset', userId);
const bucket = gridFs.GridFSBucket(); const collection = gridFs.Collection();
const mongoWhere = { const mongoWhere = {
['metadata.kbId']: kbId, ['metadata.kbId']: kbId,
['metadata.userId']: userId,
['metadata.datasetUsed']: true,
...(searchText && { filename: { $regex: searchText } }) ...(searchText && { filename: { $regex: searchText } })
}; };
const [files, total] = await Promise.all([ const [files, total] = await Promise.all([
bucket collection
.find(mongoWhere) .find(mongoWhere, {
.sort({ _id: -1 }) projection: {
_id: 1,
filename: 1,
uploadDate: 1,
length: 1
}
})
.skip((pageNum - 1) * pageSize) .skip((pageNum - 1) * pageSize)
.limit(pageSize) .limit(pageSize)
.toArray(), .toArray(),
mongoose.connection.db.collection('dataset.files').countDocuments(mongoWhere) collection.countDocuments(mongoWhere)
]); ]);
async function GetOtherData() { async function GetOtherData() {
...@@ -49,7 +57,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -49,7 +57,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
status: (await TrainingData.findOne({ userId, kbId, file_id: '' })) status: (await TrainingData.findOne({ userId, kbId, file_id: '' }))
? FileStatusEnum.embedding ? FileStatusEnum.embedding
: FileStatusEnum.ready, : FileStatusEnum.ready,
chunkLength: await PgClient.count(PgTrainingTableName, { chunkLength: await PgClient.count(PgDatasetTableName, {
fields: ['id'], fields: ['id'],
where: [ where: [
['user_id', userId], ['user_id', userId],
...@@ -72,7 +80,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -72,7 +80,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
status: (await TrainingData.findOne({ userId, kbId, file_id: file._id })) status: (await TrainingData.findOne({ userId, kbId, file_id: file._id }))
? FileStatusEnum.embedding ? FileStatusEnum.embedding
: FileStatusEnum.ready, : FileStatusEnum.ready,
chunkLength: await PgClient.count(PgTrainingTableName, { chunkLength: await PgClient.count(PgDatasetTableName, {
fields: ['id'], fields: ['id'],
where: [ where: [
['user_id', userId], ['user_id', userId],
...@@ -90,7 +98,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -90,7 +98,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
data: { data: {
pageNum, pageNum,
pageSize, pageSize,
data: data.flat().filter((item) => item.chunkLength > 0), data: data.flat(),
total total
} }
}); });
......
client/src/pages/api/core/dataset/file/update.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { UpdateFileProps } from '@/api/core/dataset/file.d';
import { Types } from 'mongoose';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id, name, datasetUsed } = req.body as UpdateFileProps;
const { userId } = await authUser({ req, authToken: true });
const gridFs = new GridFSStorage('dataset', userId);
const collection = gridFs.Collection();
await collection.findOneAndUpdate(
{
_id: new Types.ObjectId(id)
},
{
$set: {
...(name && { filename: name }),
...(datasetUsed && { ['metadata.datasetUsed']: datasetUsed })
}
}
);
// data source
updateDatasetSource({
fileId: id,
userId,
name
});
jsonRes(res, {});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
async function updateDatasetSource(data: { fileId: string; userId: string; name?: string }) {
const { fileId, userId, name } = data;
if (!fileId || !name || !userId) return;
try {
await PgClient.update(PgDatasetTableName, {
where: [['user_id', userId], 'AND', ['file_id', fileId]],
values: [
{
key: 'source',
value: name
}
]
});
} catch (error) {
setTimeout(() => {
updateDatasetSource(data);
}, 2000);
}
}
client/src/pages/api/openapi/kb/delDataById.ts
...@@ -3,7 +3,7 @@ import { jsonRes } from '@/service/response'; ...@@ -3,7 +3,7 @@ import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
...@@ -18,7 +18,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex ...@@ -18,7 +18,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req }); const { userId } = await authUser({ req });
await PgClient.delete(PgTrainingTableName, { await PgClient.delete(PgDatasetTableName, {
where: [['user_id', userId], 'AND', ['id', dataId]] where: [['user_id', userId], 'AND', ['id', dataId]]
}); });
......
client/src/pages/api/openapi/kb/pushData.ts
...@@ -4,7 +4,7 @@ import { connectToDatabase, TrainingData, KB } from '@/service/mongo'; ...@@ -4,7 +4,7 @@ import { connectToDatabase, TrainingData, KB } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { authKb } from '@/service/utils/auth'; import { authKb } from '@/service/utils/auth';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { PgTrainingTableName, TrainingModeEnum } from '@/constants/plugin'; import { PgDatasetTableName, TrainingModeEnum } from '@/constants/plugin';
import { startQueue } from '@/service/utils/tools'; import { startQueue } from '@/service/utils/tools';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { modelToolMap } from '@/utils/plugin'; import { modelToolMap } from '@/utils/plugin';
...@@ -136,7 +136,7 @@ export async function pushDataToKb({ ...@@ -136,7 +136,7 @@ export async function pushDataToKb({
try { try {
const { rows } = await PgClient.query(` const { rows } = await PgClient.query(`
SELECT COUNT(*) > 0 AS exists SELECT COUNT(*) > 0 AS exists
FROM ${PgTrainingTableName} FROM ${PgDatasetTableName}
WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND kb_id='${kbId}' WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND kb_id='${kbId}'
`); `);
const exists = rows[0]?.exists || false; const exists = rows[0]?.exists || false;
......
client/src/pages/api/openapi/kb/searchTest.ts
...@@ -5,7 +5,7 @@ import { PgClient } from '@/service/pg'; ...@@ -5,7 +5,7 @@ import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { getVector } from '../plugin/vector'; import { getVector } from '../plugin/vector';
import type { KbTestItemType } from '@/types/plugin'; import type { KbTestItemType } from '@/types/plugin';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { KB } from '@/service/mongo'; import { KB } from '@/service/mongo';
export type Props = { export type Props = {
...@@ -43,7 +43,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex ...@@ -43,7 +43,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10}; SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10};
select id, q, a, source, file_id, (vector <#> '[${ select id, q, a, source, file_id, (vector <#> '[${
vectors[0] vectors[0]
}]') * -1 AS score from ${PgTrainingTableName} where kb_id='${kbId}' AND user_id='${userId}' order by vector <#> '[${ }]') * -1 AS score from ${PgDatasetTableName} where kb_id='${kbId}' AND user_id='${userId}' order by vector <#> '[${
vectors[0] vectors[0]
}]' limit 12; }]' limit 12;
COMMIT;` COMMIT;`
......
client/src/pages/api/openapi/kb/updateData.ts
...@@ -5,7 +5,7 @@ import { PgClient } from '@/service/pg'; ...@@ -5,7 +5,7 @@ import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { KB, connectToDatabase } from '@/service/mongo'; import { KB, connectToDatabase } from '@/service/mongo';
import { getVector } from '../plugin/vector'; import { getVector } from '../plugin/vector';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
export type Props = { export type Props = {
dataId: string; dataId: string;
...@@ -47,7 +47,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex ...@@ -47,7 +47,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
})(); })();
// 更新 pg 内容.仅修改a，不需要更新向量。 // 更新 pg 内容.仅修改a，不需要更新向量。
await PgClient.update(PgTrainingTableName, { await PgClient.update(PgDatasetTableName, {
where: [['id', dataId], 'AND', ['user_id', userId]], where: [['id', dataId], 'AND', ['user_id', userId]],
values: [ values: [
{ key: 'a', value: a.replace(/'/g, '"') }, { key: 'a', value: a.replace(/'/g, '"') },
......
client/src/pages/api/plugins/kb/data/exportModelData.ts
...@@ -3,14 +3,13 @@ import { jsonRes } from '@/service/response'; ...@@ -3,14 +3,13 @@ import { jsonRes } from '@/service/response';
import { connectToDatabase, User } from '@/service/mongo'; import { connectToDatabase, User } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { OtherFileId } from '@/constants/kb'; import { findAllChildrenIds } from '../delete';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
let { kbId, fileId } = req.query as { let { kbId } = req.query as {
kbId: string; kbId: string;
fileId: string;
}; };
if (!kbId) { if (!kbId) {
...@@ -22,6 +21,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -22,6 +21,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
const exportIds = [kbId, ...(await findAllChildrenIds(kbId))];
console.log(exportIds);
const thirtyMinutesAgo = new Date( const thirtyMinutesAgo = new Date(
Date.now() - (global.feConfigs?.limit?.exportLimitMinutes || 0) * 60 * 1000 Date.now() - (global.feConfigs?.limit?.exportLimitMinutes || 0) * 60 * 1000
); );
...@@ -43,10 +45,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -43,10 +45,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error(`上次导出未到 ${minutes}，每 ${minutes}仅可导出一次。`); throw new Error(`上次导出未到 ${minutes}，每 ${minutes}仅可导出一次。`);
} }
const where: any = [['kb_id', kbId], 'AND', ['user_id', userId]]; const where: any = [
['user_id', userId],
'AND',
`kb_id IN (${exportIds.map((id) => `'${id}'`).join(',')})`
];
// 从 pg 中获取所有数据 // 从 pg 中获取所有数据
const pgData = await PgClient.select<{ q: string; a: string; source: string }>( const pgData = await PgClient.select<{ q: string; a: string; source: string }>(
PgTrainingTableName, PgDatasetTableName,
{ {
where, where,
fields: ['q', 'a', 'source'], fields: ['q', 'a', 'source'],
......
client/src/pages/api/plugins/kb/data/getDataById.ts
...@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo'; ...@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import type { KbDataItemType } from '@/types/plugin'; import type { KbDataItemType } from '@/types/plugin';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
export type Response = { export type Response = {
id: string; id: string;
...@@ -29,7 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -29,7 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const where: any = [['user_id', userId], 'AND', ['id', dataId]]; const where: any = [['user_id', userId], 'AND', ['id', dataId]];
const searchRes = await PgClient.select<KbDataItemType>(PgTrainingTableName, { const searchRes = await PgClient.select<KbDataItemType>(PgDatasetTableName, {
fields: ['kb_id', 'id', 'q', 'a', 'source', 'file_id'], fields: ['kb_id', 'id', 'q', 'a', 'source', 'file_id'],
where, where,
limit: 1 limit: 1
......
client/src/pages/api/plugins/kb/data/getDataList.ts
...@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo'; ...@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import type { KbDataItemType } from '@/types/plugin'; import type { KbDataItemType } from '@/types/plugin';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { OtherFileId } from '@/constants/kb'; import { OtherFileId } from '@/constants/kb';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
...@@ -50,14 +50,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -50,14 +50,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
]; ];
const [searchRes, total] = await Promise.all([ const [searchRes, total] = await Promise.all([
PgClient.select<KbDataItemType>(PgTrainingTableName, { PgClient.select<KbDataItemType>(PgDatasetTableName, {
fields: ['id', 'q', 'a', 'source', 'file_id'], fields: ['id', 'q', 'a', 'source', 'file_id'],
where, where,
order: [{ field: 'id', mode: 'DESC' }], order: [{ field: 'id', mode: 'DESC' }],
limit: pageSize, limit: pageSize,
offset: pageSize * (pageNum - 1) offset: pageSize * (pageNum - 1)
}), }),
PgClient.count(PgTrainingTableName, { PgClient.count(PgDatasetTableName, {
fields: ['id'], fields: ['id'],
where where
}) })
......
client/src/pages/api/plugins/kb/data/insertData.ts
...@@ -3,7 +3,7 @@ import { jsonRes } from '@/service/response'; ...@@ -3,7 +3,7 @@ import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo'; import { connectToDatabase, KB } from '@/service/mongo';
import { authKb, authUser } from '@/service/utils/auth'; import { authKb, authUser } from '@/service/utils/auth';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { insertKbItem, PgClient } from '@/service/pg'; import { insertKbItem, PgClient } from '@/service/pg';
import { modelToolMap } from '@/utils/plugin'; import { modelToolMap } from '@/utils/plugin';
import { getVectorModel } from '@/service/utils/data'; import { getVectorModel } from '@/service/utils/data';
...@@ -45,7 +45,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex ...@@ -45,7 +45,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
const { rows: existsRows } = await PgClient.query(` const { rows: existsRows } = await PgClient.query(`
SELECT COUNT(*) > 0 AS exists SELECT COUNT(*) > 0 AS exists
FROM ${PgTrainingTableName} FROM ${PgDatasetTableName}
WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND kb_id='${kbId}' WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND kb_id='${kbId}'
`); `);
const exists = existsRows[0]?.exists || false; const exists = existsRows[0]?.exists || false;
......
client/src/pages/api/plugins/kb/delete.ts
...@@ -3,7 +3,7 @@ import { jsonRes } from '@/service/response'; ...@@ -3,7 +3,7 @@ import { jsonRes } from '@/service/response';
import { connectToDatabase, KB, App, TrainingData } from '@/service/mongo'; import { connectToDatabase, KB, App, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
...@@ -29,7 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -29,7 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}); });
// delete all pg data // delete all pg data
await PgClient.delete(PgTrainingTableName, { await PgClient.delete(PgDatasetTableName, {
where: [ where: [
['user_id', userId], ['user_id', userId],
'AND', 'AND',
...@@ -56,7 +56,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -56,7 +56,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
} }
} }
async function findAllChildrenIds(id: string) { export async function findAllChildrenIds(id: string) {
// find children // find children
const children = await KB.find({ parentId: id }); const children = await KB.find({ parentId: id });
......
client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts deleted 100644 → 0
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg';
import { PgTrainingTableName } from '@/constants/plugin';
import { Types } from 'mongoose';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { kbId } = req.query as { kbId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const gridFs = new GridFSStorage('dataset', userId);
const bucket = gridFs.GridFSBucket();
const files = await bucket
// 1 hours expired
.find({
uploadDate: { $lte: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) },
['metadata.kbId']: kbId,
['metadata.userId']: userId
})
.sort({ _id: -1 })
.toArray();
const data = await Promise.all(
files.map(async (file) => {
return {
id: file._id,
chunkLength: await PgClient.count(PgTrainingTableName, {
fields: ['id'],
where: [
['user_id', userId],
'AND',
['kb_id', kbId],
'AND',
['file_id', String(file._id)]
]
})
};
})
);
await Promise.all(
data
.filter((item) => item.chunkLength === 0)
.map((file) => bucket.delete(new Types.ObjectId(file.id)))
);
jsonRes(res);
} catch (err) {
jsonRes(res);
}
}
client/src/pages/api/plugins/file/readUrl.ts → client/src/pages/api/support/file/readUrl.ts
...@@ -23,7 +23,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -23,7 +23,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}); });
jsonRes(res, { jsonRes(res, {
data: `/api/plugins/file/read?token=${token}` data: `/api/support/file/read?token=${token}`
}); });
} catch (error) { } catch (error) {
jsonRes(res, { jsonRes(res, {
......
client/src/pages/api/system/getInitData.ts
...@@ -13,6 +13,7 @@ export type InitDateResponse = { ...@@ -13,6 +13,7 @@ export type InitDateResponse = {
qaModel: QAModelItemType; qaModel: QAModelItemType;
vectorModels: VectorModelItemType[]; vectorModels: VectorModelItemType[];
feConfigs: FeConfigsType; feConfigs: FeConfigsType;
systemVersion: string;
}; };
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
...@@ -24,7 +25,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -24,7 +25,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
feConfigs: global.feConfigs, feConfigs: global.feConfigs,
chatModels: global.chatModels, chatModels: global.chatModels,
qaModel: global.qaModel, qaModel: global.qaModel,
vectorModels: global.vectorModels vectorModels: global.vectorModels,
systemVersion: process.env.npm_package_version || '0.0.0'
} }
}); });
} }
......
client/src/pages/chat/components/ChatHistorySlider.tsx
...@@ -11,7 +11,7 @@ import { ...@@ -11,7 +11,7 @@ import {
IconButton IconButton
} from '@chakra-ui/react'; } from '@chakra-ui/react';
import { useGlobalStore } from '@/store/global'; import { useGlobalStore } from '@/store/global';
import { useEditInfo } from '@/hooks/useEditInfo'; import { useEditTitle } from '@/hooks/useEditTitle';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import Avatar from '@/components/Avatar'; import Avatar from '@/components/Avatar';
import MyTooltip from '@/components/MyTooltip'; import MyTooltip from '@/components/MyTooltip';
...@@ -70,7 +70,7 @@ const ChatHistorySlider = ({ ...@@ -70,7 +70,7 @@ const ChatHistorySlider = ({
const isShare = useMemo(() => !appId || !userInfo, [appId, userInfo]); const isShare = useMemo(() => !appId || !userInfo, [appId, userInfo]);
// custom title edit // custom title edit
const { onOpenModal, EditModal: EditTitleModal } = useEditInfo({ const { onOpenModal, EditModal: EditTitleModal } = useEditTitle({
title: '自定义历史记录标题', title: '自定义历史记录标题',
placeholder: '如果设置为空，会自动跟随聊天记录。' placeholder: '如果设置为空，会自动跟随聊天记录。'
}); });
......
client/src/pages/kb/detail/components/DataCard.tsx
import React, { useCallback, useState, useRef, useMemo } from 'react'; import React, { useCallback, useState, useRef, useMemo } from 'react';
import { Box, Card, IconButton, Flex, Button, Grid, Image } from '@chakra-ui/react'; import { Box, Card, IconButton, Flex, Grid, Image } from '@chakra-ui/react';
import type { KbDataItemType } from '@/types/plugin'; import type { KbDataItemType } from '@/types/plugin';
import { usePagination } from '@/hooks/usePagination'; import { usePagination } from '@/hooks/usePagination';
import { import { getKbDataList, delOneKbDataByDataId, getTrainingData } from '@/api/plugins/kb';
getKbDataList, import { getFileInfoById } from '@/api/core/dataset/file';
getExportDataList,
delOneKbDataByDataId,
getTrainingData,
getFileInfoById
} from '@/api/plugins/kb';
import { DeleteIcon, RepeatIcon } from '@chakra-ui/icons'; import { DeleteIcon, RepeatIcon } from '@chakra-ui/icons';
import { fileDownload } from '@/utils/file'; import { useQuery } from '@tanstack/react-query';
import { useMutation, useQuery } from '@tanstack/react-query';
import { useToast } from '@/hooks/useToast'; import { useToast } from '@/hooks/useToast';
import Papa from 'papaparse';
import InputModal, { FormData as InputDataType } from './InputDataModal'; import InputModal, { FormData as InputDataType } from './InputDataModal';
import { debounce } from 'lodash'; import { debounce } from 'lodash';
import { getErrText } from '@/utils/tools'; import { getErrText } from '@/utils/tools';
...@@ -24,8 +17,6 @@ import MyIcon from '@/components/Icon'; ...@@ -24,8 +17,6 @@ import MyIcon from '@/components/Icon';
import MyTooltip from '@/components/MyTooltip'; import MyTooltip from '@/components/MyTooltip';
import MyInput from '@/components/MyInput'; import MyInput from '@/components/MyInput';
import { fileImgs } from '@/constants/common'; import { fileImgs } from '@/constants/common';
import { useRequest } from '@/hooks/useRequest';
import { feConfigs } from '@/store/static';
const DataCard = ({ kbId }: { kbId: string }) => { const DataCard = ({ kbId }: { kbId: string }) => {
const BoxRef = useRef<HTMLDivElement>(null); const BoxRef = useRef<HTMLDivElement>(null);
...@@ -82,31 +73,27 @@ const DataCard = ({ kbId }: { kbId: string }) => { ...@@ -82,31 +73,27 @@ const DataCard = ({ kbId }: { kbId: string }) => {
[fileInfo?.filename] [fileInfo?.filename]
); );
// get al data and export csv
const { mutate: onclickExport, isLoading: isLoadingExport = false } = useRequest({
mutationFn: () => getExportDataList({ kbId, fileId }),
onSuccess(res) {
const text = Papa.unparse({
fields: ['question', 'answer', 'source'],
data: res
});
const filenameSplit = fileInfo?.filename?.split('.') || [];
const filename = filenameSplit?.length <= 1 ? 'data' : filenameSplit.slice(0, -1).join('.');
fileDownload({
text,
type: 'text/csv',
filename
});
},
successToast: `导出成功，下次导出需要 ${feConfigs?.limit?.exportLimitMinutes} 分钟后`,
errorToast: '导出异常'
});
return ( return (
<Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}> <Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}>
<Flex alignItems={'center'}> <Flex alignItems={'center'}>
<IconButton
mr={3}
icon={<MyIcon name={'backFill'} w={'18px'} color={'myBlue.600'} />}
bg={'white'}
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
h={'28px'}
size={'sm'}
borderRadius={'50%'}
aria-label={''}
onClick={() =>
router.replace({
query: {
kbId,
currentTab: 'dataset'
}
})
}
/>
<Flex <Flex
className="textEllipsis" className="textEllipsis"
flex={'1 0 0'} flex={'1 0 0'}
...@@ -117,18 +104,6 @@ const DataCard = ({ kbId }: { kbId: string }) => { ...@@ -117,18 +104,6 @@ const DataCard = ({ kbId }: { kbId: string }) => {
<Image src={fileIcon || '/imgs/files/file.svg'} w={'16px'} mr={2} alt={''} /> <Image src={fileIcon || '/imgs/files/file.svg'} w={'16px'} mr={2} alt={''} />
{t(fileInfo?.filename || 'Filename')} {t(fileInfo?.filename || 'Filename')}
</Flex> </Flex>
<Button
mr={2}
size={['sm', 'md']}
variant={'base'}
borderColor={'myBlue.600'}
color={'myBlue.600'}
isLoading={isLoadingExport || isLoading}
title={`${feConfigs} 分钟能导出 1 次`}
onClick={onclickExport}
>
{t('dataset.Export')}
</Button>
<Box> <Box>
<MyTooltip label={'刷新'}> <MyTooltip label={'刷新'}>
<IconButton <IconButton
......
client/src/pages/kb/detail/components/FileCard.tsx
...@@ -9,9 +9,14 @@ import { ...@@ -9,9 +9,14 @@ import {
Th, Th,
Td, Td,
Tbody, Tbody,
Image Image,
MenuButton,
Menu,
MenuList,
MenuItem
} from '@chakra-ui/react'; } from '@chakra-ui/react';
import { getKbFiles, deleteKbFileById, getTrainingData } from '@/api/plugins/kb'; import { getTrainingData } from '@/api/plugins/kb';
import { getDatasetFiles, delDatasetFileById, updateDatasetFile } from '@/api/core/dataset/file';
import { useQuery } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query';
import { debounce } from 'lodash'; import { debounce } from 'lodash';
import { formatFileSize } from '@/utils/tools'; import { formatFileSize } from '@/utils/tools';
...@@ -23,11 +28,13 @@ import dayjs from 'dayjs'; ...@@ -23,11 +28,13 @@ import dayjs from 'dayjs';
import { fileImgs } from '@/constants/common'; import { fileImgs } from '@/constants/common';
import { useRequest } from '@/hooks/useRequest'; import { useRequest } from '@/hooks/useRequest';
import { useLoading } from '@/hooks/useLoading'; import { useLoading } from '@/hooks/useLoading';
import { FileStatusEnum } from '@/constants/kb'; import { FileStatusEnum, OtherFileId } from '@/constants/kb';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { usePagination } from '@/hooks/usePagination'; import { usePagination } from '@/hooks/usePagination';
import { KbFileItemType } from '@/types/plugin'; import { KbFileItemType } from '@/types/plugin';
import { useGlobalStore } from '@/store/global'; import { useGlobalStore } from '@/store/global';
import MyMenu from '@/components/MyMenu';
import { useEditTitle } from '@/hooks/useEditTitle';
const FileCard = ({ kbId }: { kbId: string }) => { const FileCard = ({ kbId }: { kbId: string }) => {
const BoxRef = useRef<HTMLDivElement>(null); const BoxRef = useRef<HTMLDivElement>(null);
...@@ -45,13 +52,13 @@ const FileCard = ({ kbId }: { kbId: string }) => { ...@@ -45,13 +52,13 @@ const FileCard = ({ kbId }: { kbId: string }) => {
data: files, data: files,
Pagination, Pagination,
total, total,
isLoading,
getData, getData,
isLoading,
pageNum, pageNum,
pageSize pageSize
} = usePagination<KbFileItemType>({ } = usePagination<KbFileItemType>({
api: getKbFiles, api: getDatasetFiles,
pageSize: 40, pageSize: 20,
params: { params: {
kbId, kbId,
searchText searchText
...@@ -63,6 +70,7 @@ const FileCard = ({ kbId }: { kbId: string }) => { ...@@ -63,6 +70,7 @@ const FileCard = ({ kbId }: { kbId: string }) => {
} }
}); });
// change search
const debounceRefetch = useCallback( const debounceRefetch = useCallback(
debounce(() => { debounce(() => {
getData(1); getData(1);
...@@ -71,6 +79,7 @@ const FileCard = ({ kbId }: { kbId: string }) => { ...@@ -71,6 +79,7 @@ const FileCard = ({ kbId }: { kbId: string }) => {
[] []
); );
// add file icon
const formatFiles = useMemo( const formatFiles = useMemo(
() => () =>
files.map((file) => ({ files.map((file) => ({
...@@ -79,15 +88,11 @@ const FileCard = ({ kbId }: { kbId: string }) => { ...@@ -79,15 +88,11 @@ const FileCard = ({ kbId }: { kbId: string }) => {
})), })),
[files] [files]
); );
const totalDataLength = useMemo(
() => files.reduce((sum, item) => sum + item.chunkLength, 0),
[files]
);
const { mutate: onDeleteFile } = useRequest({ const { mutate: onDeleteFile } = useRequest({
mutationFn: (fileId: string) => { mutationFn: (fileId: string) => {
setLoading(true); setLoading(true);
return deleteKbFileById({ return delDatasetFileById({
fileId, fileId,
kbId kbId
}); });
...@@ -101,6 +106,24 @@ const FileCard = ({ kbId }: { kbId: string }) => { ...@@ -101,6 +106,24 @@ const FileCard = ({ kbId }: { kbId: string }) => {
successToast: t('common.Delete Success'), successToast: t('common.Delete Success'),
errorToast: t('common.Delete Failed') errorToast: t('common.Delete Failed')
}); });
const { mutate: onUpdateFilename } = useRequest({
mutationFn: (data: { id: string; name: string }) => {
setLoading(true);
return updateDatasetFile(data);
},
onSuccess() {
getData(pageNum);
},
onSettled() {
setLoading(false);
},
successToast: t('common.Delete Success'),
errorToast: t('common.Delete Failed')
});
const { onOpenModal, EditModal: EditTitleModal } = useEditTitle({
title: t('Rename')
});
const statusMap = { const statusMap = {
[FileStatusEnum.embedding]: { [FileStatusEnum.embedding]: {
...@@ -121,17 +144,21 @@ const FileCard = ({ kbId }: { kbId: string }) => { ...@@ -121,17 +144,21 @@ const FileCard = ({ kbId }: { kbId: string }) => {
} }
}); });
useQuery(['refetchTrainingData'], refetchTrainingData, { useQuery(
refetchInterval: 8000, ['refetchTrainingData', kbId],
enabled: qaListLen > 0 || vectorListLen > 0 () => Promise.all([refetchTrainingData(), getData(pageNum)]),
}); {
refetchInterval: 8000,
enabled: qaListLen > 0 || vectorListLen > 0
}
);
return ( return (
<Box ref={BoxRef} position={'relative'} py={[1, 5]} h={'100%'} overflow={'overlay'}> <Box ref={BoxRef} py={[1, 5]} h={'100%'} overflow={'overlay'}>
<Flex justifyContent={'space-between'} px={5}> <Flex justifyContent={'space-between'} px={[2, 5]}>
<Box> <Box>
<Box fontWeight={'bold'} fontSize={'lg'} mr={2}> <Box fontWeight={'bold'} fontSize={['md', 'lg']} mr={2}>
{t('kb.Files', { total: files.length })} {t('kb.Files', { total })}
</Box> </Box>
<Box as={'span'} fontSize={'sm'}> <Box as={'span'} fontSize={'sm'}>
{(qaListLen > 0 || vectorListLen > 0) && ( {(qaListLen > 0 || vectorListLen > 0) && (
...@@ -149,7 +176,8 @@ const FileCard = ({ kbId }: { kbId: string }) => { ...@@ -149,7 +176,8 @@ const FileCard = ({ kbId }: { kbId: string }) => {
leftIcon={ leftIcon={
<MyIcon name="searchLight" position={'absolute'} w={'14px'} color={'myGray.500'} /> <MyIcon name="searchLight" position={'absolute'} w={'14px'} color={'myGray.500'} />
} }
w={['100%', '200px']} w={['100%', '250px']}
size={['sm', 'md']}
placeholder={t('common.Search') || ''} placeholder={t('common.Search') || ''}
value={searchText} value={searchText}
onChange={(e) => { onChange={(e) => {
...@@ -169,14 +197,12 @@ const FileCard = ({ kbId }: { kbId: string }) => { ...@@ -169,14 +197,12 @@ const FileCard = ({ kbId }: { kbId: string }) => {
/> />
</Flex> </Flex>
</Flex> </Flex>
<TableContainer mt={[0, 3]}> <TableContainer mt={[0, 3]} position={'relative'} minH={'70vh'}>
<Table variant={'simple'} fontSize={'sm'}> <Table variant={'simple'} fontSize={'sm'}>
<Thead> <Thead>
<Tr> <Tr>
<Th>{t('kb.Filename')}</Th> <Th>{t('kb.Filename')}</Th>
<Th> <Th>{t('kb.Chunk Length')}</Th>
{t('kb.Chunk Length')}({totalDataLength})
</Th>
<Th>{t('kb.Upload Time')}</Th> <Th>{t('kb.Upload Time')}</Th>
<Th>{t('kb.File Size')}</Th> <Th>{t('kb.File Size')}</Th>
<Th>{t('common.Status')}</Th> <Th>{t('common.Status')}</Th>
...@@ -213,45 +239,103 @@ const FileCard = ({ kbId }: { kbId: string }) => { ...@@ -213,45 +239,103 @@ const FileCard = ({ kbId }: { kbId: string }) => {
</Td> </Td>
<Td>{dayjs(file.uploadTime).format('YYYY/MM/DD HH:mm')}</Td> <Td>{dayjs(file.uploadTime).format('YYYY/MM/DD HH:mm')}</Td>
<Td>{formatFileSize(file.size)}</Td> <Td>{formatFileSize(file.size)}</Td>
<Td <Td>
display={'flex'} <Flex
alignItems={'center'} alignItems={'center'}
_before={{ _before={{
content: '""', content: '""',
w: '10px', w: '10px',
h: '10px', h: '10px',
mr: 2, mr: 2,
borderRadius: 'lg', borderRadius: 'lg',
bg: statusMap[file.status].color bg: statusMap[file.status].color
}} }}
> >
{statusMap[file.status].text} {statusMap[file.status].text}
</Flex>
</Td> </Td>
<Td onClick={(e) => e.stopPropagation()}> <Td onClick={(e) => e.stopPropagation()}>
<MyIcon <MyMenu
name={'delete'} width={100}
w={'14px'} Button={
_hover={{ color: 'red.600' }} <MenuButton
onClick={() => w={'22px'}
openConfirm(() => { h={'22px'}
onDeleteFile(file.id); borderRadius={'md'}
})() _hover={{
color: 'myBlue.600',
'& .icon': {
bg: 'myGray.100'
}
}}
>
<MyIcon
className="icon"
name={'more'}
h={'16px'}
w={'16px'}
px={1}
py={1}
borderRadius={'md'}
cursor={'pointer'}
/>
</MenuButton>
} }
menuList={[
...(file.id !== OtherFileId
? [
{
child: (
<Flex alignItems={'center'}>
<MyIcon name={'edit'} w={'14px'} mr={2} />
{t('Rename')}
</Flex>
),
onClick: () =>
onOpenModal({
defaultVal: file.filename,
onSuccess: (newName) => {
onUpdateFilename({
id: file.id,
name: newName
});
}
})
}
]
: []),
{
child: (
<Flex alignItems={'center'}>
<MyIcon
mr={1}
name={'delete'}
w={'14px'}
_hover={{ color: 'red.600' }}
/>
<Box>{t('common.Delete')}</Box>
</Flex>
),
onClick: openConfirm(() => {
onDeleteFile(file.id);
})
}
]}
/> />
</Td> </Td>
</Tr> </Tr>
))} ))}
</Tbody> </Tbody>
</Table> </Table>
<Loading loading={isLoading && files.length === 0} fixed={false} />
{total > pageSize && (
<Flex mt={2} justifyContent={'center'}>
<Pagination />
</Flex>
)}
</TableContainer> </TableContainer>
{total > pageSize && (
<Flex mt={2} justifyContent={'center'}>
<Pagination />
</Flex>
)}
<ConfirmModal /> <ConfirmModal />
<Loading loading={isLoading} /> <EditTitleModal />
</Box> </Box>
); );
}; };
......
client/src/pages/kb/detail/components/Import/Chunk.tsx
...@@ -27,6 +27,7 @@ import { QuestionOutlineIcon } from '@chakra-ui/icons'; ...@@ -27,6 +27,7 @@ import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import FileSelect, { type FileItemType } from './FileSelect'; import FileSelect, { type FileItemType } from './FileSelect';
import { useDatasetStore } from '@/store/dataset'; import { useDatasetStore } from '@/store/dataset';
import { updateDatasetFile } from '@/api/core/dataset/file';
const fileExtension = '.txt, .doc, .docx, .pdf, .md'; const fileExtension = '.txt, .doc, .docx, .pdf, .md';
...@@ -64,6 +65,16 @@ const ChunkImport = ({ kbId }: { kbId: string }) => { ...@@ -64,6 +65,16 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
mutationFn: async () => { mutationFn: async () => {
const chunks = files.map((file) => file.chunks).flat(); const chunks = files.map((file) => file.chunks).flat();
// mark the file is used
await Promise.all(
files.map((file) =>
updateDatasetFile({
id: file.id,
datasetUsed: true
})
)
);
// subsection import // subsection import
let success = 0; let success = 0;
const step = 300; const step = 300;
......
client/src/pages/kb/detail/components/Import/Csv.tsx
...@@ -11,6 +11,7 @@ import { TrainingModeEnum } from '@/constants/plugin'; ...@@ -11,6 +11,7 @@ import { TrainingModeEnum } from '@/constants/plugin';
import FileSelect, { type FileItemType } from './FileSelect'; import FileSelect, { type FileItemType } from './FileSelect';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { useDatasetStore } from '@/store/dataset'; import { useDatasetStore } from '@/store/dataset';
import { updateDatasetFile } from '@/api/core/dataset/file';
const fileExtension = '.csv'; const fileExtension = '.csv';
...@@ -37,6 +38,16 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -37,6 +38,16 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
const { mutate: onclickUpload, isLoading: uploading } = useMutation({ const { mutate: onclickUpload, isLoading: uploading } = useMutation({
mutationFn: async () => { mutationFn: async () => {
// mark the file is used
await Promise.all(
files.map((file) =>
updateDatasetFile({
id: file.id,
datasetUsed: true
})
)
);
const chunks = files const chunks = files
.map((file) => file.chunks) .map((file) => file.chunks)
.flat() .flat()
......
client/src/pages/kb/detail/components/Import/FileSelect.tsx
...@@ -152,7 +152,7 @@ const FileSelect = ({ ...@@ -152,7 +152,7 @@ const FileSelect = ({
throw new Error('csv 文件格式有误,请确保 question 和 answer 两列'); throw new Error('csv 文件格式有误,请确保 question 和 answer 两列');
} }
const fileItem: FileItemType = { const fileItem: FileItemType = {
id: nanoid(), id: filesId[0],
filename: file.name, filename: file.name,
icon, icon,
tokens: 0, tokens: 0,
......
client/src/pages/kb/detail/components/Import/QA.tsx
...@@ -16,6 +16,7 @@ import { QuestionOutlineIcon } from '@chakra-ui/icons'; ...@@ -16,6 +16,7 @@ import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import FileSelect, { type FileItemType } from './FileSelect'; import FileSelect, { type FileItemType } from './FileSelect';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { updateDatasetFile } from '@/api/core/dataset/file';
const fileExtension = '.txt, .doc, .docx, .pdf, .md'; const fileExtension = '.txt, .doc, .docx, .pdf, .md';
...@@ -55,6 +56,16 @@ const QAImport = ({ kbId }: { kbId: string }) => { ...@@ -55,6 +56,16 @@ const QAImport = ({ kbId }: { kbId: string }) => {
mutationFn: async () => { mutationFn: async () => {
const chunks = files.map((file) => file.chunks).flat(); const chunks = files.map((file) => file.chunks).flat();
// mark the file is used
await Promise.all(
files.map((file) =>
updateDatasetFile({
id: file.id,
datasetUsed: true
})
)
);
// subsection import // subsection import
let success = 0; let success = 0;
const step = 200; const step = 200;
......
client/src/pages/kb/detail/components/InputDataModal.tsx
...@@ -2,7 +2,7 @@ import React, { useState, useCallback } from 'react'; ...@@ -2,7 +2,7 @@ import React, { useState, useCallback } from 'react';
import { Box, Flex, Button, Textarea, IconButton, BoxProps } from '@chakra-ui/react'; import { Box, Flex, Button, Textarea, IconButton, BoxProps } from '@chakra-ui/react';
import { useForm } from 'react-hook-form'; import { useForm } from 'react-hook-form';
import { insertData2Kb, putKbDataById, delOneKbDataByDataId } from '@/api/plugins/kb'; import { insertData2Kb, putKbDataById, delOneKbDataByDataId } from '@/api/plugins/kb';
import { getFileViewUrl } from '@/api/system'; import { getFileViewUrl } from '@/api/support/file';
import { useToast } from '@/hooks/useToast'; import { useToast } from '@/hooks/useToast';
import { getErrText } from '@/utils/tools'; import { getErrText } from '@/utils/tools';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
......
client/src/pages/kb/detail/index.tsx
...@@ -17,7 +17,8 @@ import Avatar from '@/components/Avatar'; ...@@ -17,7 +17,8 @@ import Avatar from '@/components/Avatar';
import Info from './components/Info'; import Info from './components/Info';
import { serviceSideProps } from '@/utils/i18n'; import { serviceSideProps } from '@/utils/i18n';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { delEmptyFiles, getTrainingQueueLen } from '@/api/plugins/kb'; import { getTrainingQueueLen } from '@/api/plugins/kb';
import { delDatasetEmptyFiles } from '@/api/core/dataset/file';
import MyTooltip from '@/components/MyTooltip'; import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { feConfigs } from '@/store/static'; import { feConfigs } from '@/store/static';
...@@ -96,7 +97,7 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` } ...@@ -96,7 +97,7 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` }
useEffect(() => { useEffect(() => {
return () => { return () => {
try { try {
delEmptyFiles(kbId); delDatasetEmptyFiles(kbId);
} catch (error) {} } catch (error) {}
}; };
}, [kbId]); }, [kbId]);
......
client/src/pages/kb/list/index.tsx
...@@ -15,7 +15,7 @@ import PageContainer from '@/components/PageContainer'; ...@@ -15,7 +15,7 @@ import PageContainer from '@/components/PageContainer';
import { useConfirm } from '@/hooks/useConfirm'; import { useConfirm } from '@/hooks/useConfirm';
import { AddIcon } from '@chakra-ui/icons'; import { AddIcon } from '@chakra-ui/icons';
import { useQuery } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query';
import { delKbById, getKbPaths, putKbById } from '@/api/plugins/kb'; import { delKbById, getExportDataList, getKbPaths, putKbById } from '@/api/plugins/kb';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import Avatar from '@/components/Avatar'; import Avatar from '@/components/Avatar';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
...@@ -26,7 +26,10 @@ import Tag from '@/components/Tag'; ...@@ -26,7 +26,10 @@ import Tag from '@/components/Tag';
import MyMenu from '@/components/MyMenu'; import MyMenu from '@/components/MyMenu';
import { useRequest } from '@/hooks/useRequest'; import { useRequest } from '@/hooks/useRequest';
import { useGlobalStore } from '@/store/global'; import { useGlobalStore } from '@/store/global';
import { useEditInfo } from '@/hooks/useEditInfo'; import { useEditTitle } from '@/hooks/useEditTitle';
import Papa from 'papaparse';
import { fileDownload } from '@/utils/file';
import { feConfigs } from '@/store/static';
const CreateModal = dynamic(() => import('./component/CreateModal'), { ssr: false }); const CreateModal = dynamic(() => import('./component/CreateModal'), { ssr: false });
const EditFolderModal = dynamic(() => import('./component/EditFolderModal'), { ssr: false }); const EditFolderModal = dynamic(() => import('./component/EditFolderModal'), { ssr: false });
...@@ -49,7 +52,7 @@ const Kb = () => { ...@@ -49,7 +52,7 @@ const Kb = () => {
content: '' content: ''
}); });
const { myKbList, loadKbList, setKbList, updateDataset } = useDatasetStore(); const { myKbList, loadKbList, setKbList, updateDataset } = useDatasetStore();
const { onOpenModal: onOpenTitleModal, EditModal: EditTitleModal } = useEditInfo({ const { onOpenModal: onOpenTitleModal, EditModal: EditTitleModal } = useEditTitle({
title: t('Rename') title: t('Rename')
}); });
const [moveDataId, setMoveDataId] = useState<string>(); const [moveDataId, setMoveDataId] = useState<string>();
...@@ -83,7 +86,32 @@ const Kb = () => { ...@@ -83,7 +86,32 @@ const Kb = () => {
errorToast: t('kb.Delete Dataset Error') errorToast: t('kb.Delete Dataset Error')
}); });
const { data, refetch } = useQuery(['loadKbList', parentId], () => { // export dataset to csv
const { mutate: onclickExport } = useRequest({
mutationFn: (kbId: string) => {
setLoading(true);
return getExportDataList({ kbId });
},
onSuccess(res) {
const text = Papa.unparse({
fields: ['question', 'answer', 'source'],
data: res
});
fileDownload({
text,
type: 'text/csv',
filename: 'dataset.csv'
});
},
onSettled() {
setLoading(false);
},
successToast: `导出成功，下次导出需要 ${feConfigs?.limit?.exportLimitMinutes} 分钟后`,
errorToast: '导出异常'
});
const { data, refetch } = useQuery(['loadDataset', parentId], () => {
return Promise.all([loadKbList(parentId), getKbPaths(parentId)]); return Promise.all([loadKbList(parentId), getKbPaths(parentId)]);
}); });
...@@ -321,6 +349,15 @@ const Kb = () => { ...@@ -321,6 +349,15 @@ const Kb = () => {
{ {
child: ( child: (
<Flex alignItems={'center'}> <Flex alignItems={'center'}>
<MyIcon name={'export'} w={'14px'} mr={2} />
{t('Export')}
</Flex>
),
onClick: () => onclickExport(kb._id)
},
{
child: (
<Flex alignItems={'center'}>
<MyIcon name={'delete'} w={'14px'} mr={2} /> <MyIcon name={'delete'} w={'14px'} mr={2} />
{t('common.Delete')} {t('common.Delete')}
</Flex> </Flex>
......
client/src/service/lib/gridfs.ts
...@@ -17,6 +17,9 @@ export class GridFSStorage { ...@@ -17,6 +17,9 @@ export class GridFSStorage {
this.bucket = bucket; this.bucket = bucket;
this.uid = String(uid); this.uid = String(uid);
} }
Collection() {
return mongoose.connection.db.collection(`${this.bucket}.files`);
}
GridFSBucket() { GridFSBucket() {
return new mongoose.mongo.GridFSBucket(mongoose.connection.db, { return new mongoose.mongo.GridFSBucket(mongoose.connection.db, {
bucketName: this.bucket bucketName: this.bucket
......
client/src/service/moduleDispatch/kb/search.ts
...@@ -5,7 +5,7 @@ import { getVector } from '@/pages/api/openapi/plugin/vector'; ...@@ -5,7 +5,7 @@ import { getVector } from '@/pages/api/openapi/plugin/vector';
import { countModelPrice } from '@/service/events/pushBill'; import { countModelPrice } from '@/service/events/pushBill';
import type { SelectedKbType } from '@/types/plugin'; import type { SelectedKbType } from '@/types/plugin';
import type { QuoteItemType } from '@/types/chat'; import type { QuoteItemType } from '@/types/chat';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
type KBSearchProps = { type KBSearchProps = {
kbList: SelectedKbType; kbList: SelectedKbType;
...@@ -42,7 +42,7 @@ export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSe ...@@ -42,7 +42,7 @@ export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSe
const res: any = await PgClient.query( const res: any = await PgClient.query(
`BEGIN; `BEGIN;
SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10}; SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10};
select kb_id,id,q,a,source,file_id from ${PgTrainingTableName} where kb_id IN (${kbList select kb_id,id,q,a,source,file_id from ${PgDatasetTableName} where kb_id IN (${kbList
.map((item) => `'${item.kbId}'`) .map((item) => `'${item.kbId}'`)
.join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${ .join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${
vectors[0] vectors[0]
......
client/src/service/pg.ts
import { Pool } from 'pg'; import { Pool } from 'pg';
import type { QueryResultRow } from 'pg'; import type { QueryResultRow } from 'pg';
import { PgTrainingTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { addLog } from './utils/tools'; import { addLog } from './utils/tools';
import { DatasetItemType } from '@/types/plugin'; import { DatasetItemType } from '@/types/plugin';
...@@ -174,7 +174,7 @@ export const insertKbItem = ({ ...@@ -174,7 +174,7 @@ export const insertKbItem = ({
vector: number[]; vector: number[];
})[]; })[];
}) => { }) => {
return PgClient.insert(PgTrainingTableName, { return PgClient.insert(PgDatasetTableName, {
values: data.map((item) => [ values: data.map((item) => [
{ key: 'user_id', value: userId }, { key: 'user_id', value: userId },
{ key: 'kb_id', value: kbId }, { key: 'kb_id', value: kbId },
...@@ -192,7 +192,7 @@ export async function initPg() { ...@@ -192,7 +192,7 @@ export async function initPg() {
await connectPg(); await connectPg();
await PgClient.query(` await PgClient.query(`
CREATE EXTENSION IF NOT EXISTS vector; CREATE EXTENSION IF NOT EXISTS vector;
CREATE TABLE IF NOT EXISTS ${PgTrainingTableName} ( CREATE TABLE IF NOT EXISTS ${PgDatasetTableName} (
id BIGSERIAL PRIMARY KEY, id BIGSERIAL PRIMARY KEY,
vector VECTOR(1536) NOT NULL, vector VECTOR(1536) NOT NULL,
user_id VARCHAR(50) NOT NULL, user_id VARCHAR(50) NOT NULL,
...@@ -202,9 +202,9 @@ export async function initPg() { ...@@ -202,9 +202,9 @@ export async function initPg() {
q TEXT NOT NULL, q TEXT NOT NULL,
a TEXT a TEXT
); );
CREATE INDEX IF NOT EXISTS modelData_userId_index ON ${PgTrainingTableName} USING HASH (user_id); CREATE INDEX IF NOT EXISTS modelData_userId_index ON ${PgDatasetTableName} USING HASH (user_id);
CREATE INDEX IF NOT EXISTS modelData_kbId_index ON ${PgTrainingTableName} USING HASH (kb_id); CREATE INDEX IF NOT EXISTS modelData_kbId_index ON ${PgDatasetTableName} USING HASH (kb_id);
CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON ${PgTrainingTableName} (md5(q), md5(a), user_id, kb_id); CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON ${PgDatasetTableName} (md5(q), md5(a), user_id, kb_id);
`); `);
console.log('init pg successful'); console.log('init pg successful');
} catch (error) { } catch (error) {
......
client/src/store/static.ts
...@@ -8,6 +8,7 @@ import { getInitData } from '@/api/system'; ...@@ -8,6 +8,7 @@ import { getInitData } from '@/api/system';
import { delay } from '@/utils/tools'; import { delay } from '@/utils/tools';
import { FeConfigsType } from '@/types'; import { FeConfigsType } from '@/types';
export let systemVersion = '0.0.0';
export let chatModelList: ChatModelItemType[] = []; export let chatModelList: ChatModelItemType[] = [];
export let qaModel: QAModelItemType = { export let qaModel: QAModelItemType = {
model: 'gpt-3.5-turbo-16k', model: 'gpt-3.5-turbo-16k',
...@@ -28,6 +29,7 @@ export const clientInitData = async (): Promise<InitDateResponse> => { ...@@ -28,6 +29,7 @@ export const clientInitData = async (): Promise<InitDateResponse> => {
qaModel = res.qaModel; qaModel = res.qaModel;
vectorModelList = res.vectorModels; vectorModelList = res.vectorModels;
feConfigs = res.feConfigs; feConfigs = res.feConfigs;
systemVersion = res.systemVersion;
return res; return res;
} catch (error) { } catch (error) {
......
client/src/utils/file.ts
...@@ -2,7 +2,7 @@ import mammoth from 'mammoth'; ...@@ -2,7 +2,7 @@ import mammoth from 'mammoth';
import Papa from 'papaparse'; import Papa from 'papaparse';
import { getOpenAiEncMap } from './plugin/openai'; import { getOpenAiEncMap } from './plugin/openai';
import { getErrText } from './tools'; import { getErrText } from './tools';
import { uploadImg, postUploadFiles } from '@/api/system'; import { uploadImg, postUploadFiles } from '@/api/support/file';
/** /**
* upload file to mongo gridfs * upload file to mongo gridfs
......
docSite/content/docs/installation/upgrading/44 copy.md 0 → 100644
---
title: '升级到 V4.4.1'
description: 'FastGPT 从旧版本升级到 V4.4.1 操作指南'
icon: 'upgrade'
draft: false
toc: true
weight: 994
---
## 执行初始化 API
发起 1 个 HTTP 请求（记得携带 `headers.rootkey`，这个值是环境变量里的）
1. https://xxxxx/api/admin/initv441
```bash
curl --location --request POST 'https://{{host}}/api/admin/initv441' \
--header 'rootkey: {{rootkey}}' \
--header 'Content-Type: application/json'
```
会给初始化 Mongo 的 dataset.files，将所有数据设置为可用。
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment