Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit b8ea546b by Archer Committed by GitHub
Options

v4.2.2 (#312)

parent 0bb31b98
Showing with 232 additions and 180 deletions
client/package.json
...@@ -23,6 +23,7 @@ ...@@ -23,6 +23,7 @@
"crypto": "^1.0.1", "crypto": "^1.0.1",
"date-fns": "^2.30.0", "date-fns": "^2.30.0",
"dayjs": "^1.11.7", "dayjs": "^1.11.7",
"downloadjs": "^1.4.7",
"echarts": "^5.4.1", "echarts": "^5.4.1",
"echarts-gl": "^2.0.9", "echarts-gl": "^2.0.9",
"formidable": "^2.1.1", "formidable": "^2.1.1",
...@@ -48,6 +49,7 @@ ...@@ -48,6 +49,7 @@
"openai": "^3.3.0", "openai": "^3.3.0",
"papaparse": "^5.4.1", "papaparse": "^5.4.1",
"pg": "^8.10.0", "pg": "^8.10.0",
"pg-query-stream": "^4.5.3",
"react": "18.2.0", "react": "18.2.0",
"react-day-picker": "^8.7.1", "react-day-picker": "^8.7.1",
"react-dom": "18.2.0", "react-dom": "18.2.0",
...@@ -71,6 +73,7 @@ ...@@ -71,6 +73,7 @@
"devDependencies": { "devDependencies": {
"@svgr/webpack": "^6.5.1", "@svgr/webpack": "^6.5.1",
"@types/cookie": "^0.5.1", "@types/cookie": "^0.5.1",
"@types/downloadjs": "^1.4.3",
"@types/formidable": "^2.0.5", "@types/formidable": "^2.0.5",
"@types/js-cookie": "^3.0.3", "@types/js-cookie": "^3.0.3",
"@types/jsdom": "^21.1.1", "@types/jsdom": "^21.1.1",
......
client/src/api/plugins/kb.ts
...@@ -13,6 +13,8 @@ import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData'; ...@@ -13,6 +13,8 @@ import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData';
import type { KbUpdateParams, CreateKbParams, GetKbDataListProps } from '../request/kb'; import type { KbUpdateParams, CreateKbParams, GetKbDataListProps } from '../request/kb';
import { QuoteItemType } from '@/types/chat'; import { QuoteItemType } from '@/types/chat';
import { KbTypeEnum } from '@/constants/kb'; import { KbTypeEnum } from '@/constants/kb';
import { getToken } from '@/utils/user';
import download from 'downloadjs';
/* knowledge base */ /* knowledge base */
export const getKbList = (data: { parentId?: string; type?: `${KbTypeEnum}` }) => export const getKbList = (data: { parentId?: string; type?: `${KbTypeEnum}` }) =>
...@@ -35,12 +37,23 @@ export const getKbDataList = (data: GetKbDataListProps) => ...@@ -35,12 +37,23 @@ export const getKbDataList = (data: GetKbDataListProps) =>
POST(`/plugins/kb/data/getDataList`, data); POST(`/plugins/kb/data/getDataList`, data);
/** /**
* 获取导出数据（不分页） * export and download data
*/ */
export const getExportDataList = (data: { kbId: string }) => export const exportDataset = (data: { kbId: string }) =>
GET<[string, string, string][]>(`/plugins/kb/data/exportModelData`, data, { fetch(`/api/plugins/kb/data/exportAll?kbId=${data.kbId}`, {
timeout: 600000 method: 'GET',
}); headers: {
token: getToken()
}
})
.then(async (res) => {
if (!res.ok) {
const data = await res.json();
throw new Error(data?.message || 'Export failed');
}
return res.blob();
})
.then((blob) => download(blob, 'dataset.csv', 'text/csv'));
/** /**
* 获取模型正在拆分数据的数量 * 获取模型正在拆分数据的数量
......
client/src/components/ChatBox/index.tsx
...@@ -452,29 +452,22 @@ const ChatBox = ( ...@@ -452,29 +452,22 @@ const ChatBox = (
border: theme.borders.base, border: theme.borders.base,
mr: 3 mr: 3
}; };
const controlContainerStyle = useCallback((status: ChatSiteItemType['status']) => { const controlContainerStyle = {
return { className: 'control',
className: 'control', color: 'myGray.400',
color: 'myGray.400', display: 'flex',
display: pl: 1,
status === 'finish' mt: 2
? feedbackType === FeedbackTypeEnum.admin };
? 'flex'
: ['flex', 'none']
: 'none',
pl: 1,
mt: 2
};
}, []);
const MessageCardStyle: BoxProps = { const MessageCardStyle: BoxProps = {
px: 4, px: 4,
py: 3, py: 3,
borderRadius: '0 8px 8px 8px', borderRadius: '0 8px 8px 8px',
boxShadow: '0 0 8px rgba(0,0,0,0.15)' boxShadow: '0 0 8px rgba(0,0,0,0.15)',
display: 'inline-block',
maxW: ['calc(100% - 25px)', 'calc(100% - 40px)']
}; };
const messageCardMaxW = ['calc(100% - 25px)', 'calc(100% - 40px)'];
const showEmpty = useMemo( const showEmpty = useMemo(
() => () =>
feConfigs?.show_emptyChat && feConfigs?.show_emptyChat &&
...@@ -542,84 +535,80 @@ const ChatBox = ( ...@@ -542,84 +535,80 @@ const ChatBox = (
{showEmpty && <Empty />} {showEmpty && <Empty />}
{!!welcomeText && ( {!!welcomeText && (
<Flex flexDirection={'column'} alignItems={'flex-start'} py={2}> <Box py={3}>
{/* avatar */} {/* avatar */}
<ChatAvatar src={appAvatar} type={'AI'} /> <ChatAvatar src={appAvatar} type={'AI'} />
{/* message */} {/* message */}
<Card order={2} mt={2} {...MessageCardStyle} bg={'white'} maxW={messageCardMaxW}> <Box textAlign={'left'}>
<Markdown source={`~~~guide \n${welcomeText}`} isChatting={false} /> <Card order={2} mt={2} {...MessageCardStyle} bg={'white'}>
</Card> <Markdown source={`~~~guide \n${welcomeText}`} isChatting={false} />
</Flex> </Card>
</Box>
</Box>
)} )}
{/* variable input */} {/* variable input */}
{!!variableModules?.length && ( {!!variableModules?.length && (
<Flex flexDirection={'column'} alignItems={'flex-start'} py={2}> <Box py={3}>
{/* avatar */} {/* avatar */}
<ChatAvatar src={appAvatar} type={'AI'} /> <ChatAvatar src={appAvatar} type={'AI'} />
{/* message */} {/* message */}
<Card <Box textAlign={'left'}>
order={2} <Card order={2} mt={2} bg={'white'} w={'400px'} {...MessageCardStyle}>
mt={2} {variableModules.map((item) => (
bg={'white'} <Box key={item.id} mb={4}>
w={'400px'} <VariableLabel required={item.required}>{item.label}</VariableLabel>
maxW={messageCardMaxW} {item.type === VariableInputEnum.input && (
{...MessageCardStyle} <Input
> isDisabled={variableIsFinish}
{variableModules.map((item) => ( {...register(item.key, {
<Box key={item.id} mb={4}> required: item.required
<VariableLabel required={item.required}>{item.label}</VariableLabel> })}
{item.type === VariableInputEnum.input && ( />
<Input )}
isDisabled={variableIsFinish} {item.type === VariableInputEnum.select && (
{...register(item.key, { <MySelect
required: item.required width={'100%'}
})} isDisabled={variableIsFinish}
/> list={(item.enums || []).map((item) => ({
)} label: item.value,
{item.type === VariableInputEnum.select && ( value: item.value
<MySelect }))}
width={'100%'} {...register(item.key, {
isDisabled={variableIsFinish} required: item.required
list={(item.enums || []).map((item) => ({ })}
label: item.value, value={getValues(item.key)}
value: item.value onchange={(e) => {
}))} setValue(item.key, e);
{...register(item.key, { setRefresh(!refresh);
required: item.required }}
})} />
value={getValues(item.key)} )}
onchange={(e) => { </Box>
setValue(item.key, e); ))}
setRefresh(!refresh); {!variableIsFinish && (
}} <Button
/> leftIcon={<MyIcon name={'chatFill'} w={'16px'} />}
)} size={'sm'}
</Box> maxW={'100px'}
))} borderRadius={'lg'}
{!variableIsFinish && ( onClick={handleSubmit((data) => {
<Button onUpdateVariable?.(data);
leftIcon={<MyIcon name={'chatFill'} w={'16px'} />} setVariables(data);
size={'sm'} setVariableInputFinish(true);
maxW={'100px'} })}
borderRadius={'lg'} >
onClick={handleSubmit((data) => { {'开始对话'}
onUpdateVariable?.(data); </Button>
setVariables(data); )}
setVariableInputFinish(true); </Card>
})} </Box>
> </Box>
{'开始对话'}
</Button>
)}
</Card>
</Flex>
)} )}
{/* chat history */} {/* chat history */}
<Box id={'history'}> <Box id={'history'}>
{chatHistory.map((item, index) => ( {chatHistory.map((item, index) => (
<Flex <Box
position={'relative'}
key={item.dataId} key={item.dataId}
flexDirection={'column'} flexDirection={'column'}
alignItems={item.obj === 'Human' ? 'flex-end' : 'flex-start'} alignItems={item.obj === 'Human' ? 'flex-end' : 'flex-start'}
...@@ -633,11 +622,7 @@ const ChatBox = ( ...@@ -633,11 +622,7 @@ const ChatBox = (
{item.obj === 'Human' && ( {item.obj === 'Human' && (
<> <>
<Flex w={'100%'} alignItems={'center'} justifyContent={'flex-end'}> <Flex w={'100%'} alignItems={'center'} justifyContent={'flex-end'}>
<Flex <Flex {...controlContainerStyle} justifyContent={'flex-end'} mr={3}>
{...controlContainerStyle(item.status)}
justifyContent={'flex-end'}
mr={3}
>
<MyTooltip label={t('common.Copy')}> <MyTooltip label={t('common.Copy')}>
<MyIcon <MyIcon
{...controlIconStyle} {...controlIconStyle}
...@@ -678,7 +663,7 @@ const ChatBox = ( ...@@ -678,7 +663,7 @@ const ChatBox = (
</Flex> </Flex>
<ChatAvatar src={userAvatar} type={'Human'} /> <ChatAvatar src={userAvatar} type={'Human'} />
</Flex> </Flex>
<Box position={'relative'} maxW={messageCardMaxW} mt={['6px', 2]}> <Box mt={['6px', 2]} textAlign={'right'}>
<Card <Card
className="markdown" className="markdown"
whiteSpace={'pre-wrap'} whiteSpace={'pre-wrap'}
...@@ -695,7 +680,7 @@ const ChatBox = ( ...@@ -695,7 +680,7 @@ const ChatBox = (
<> <>
<Flex w={'100%'} alignItems={'flex-end'}> <Flex w={'100%'} alignItems={'flex-end'}>
<ChatAvatar src={appAvatar} type={'AI'} /> <ChatAvatar src={appAvatar} type={'AI'} />
<Flex {...controlContainerStyle(item.status)} ml={3}> <Flex {...controlContainerStyle} ml={3}>
<MyTooltip label={'复制'}> <MyTooltip label={'复制'}>
<MyIcon <MyIcon
{...controlIconStyle} {...controlIconStyle}
...@@ -841,7 +826,7 @@ const ChatBox = ( ...@@ -841,7 +826,7 @@ const ChatBox = (
</Flex> </Flex>
)} )}
</Flex> </Flex>
<Box position={'relative'} maxW={messageCardMaxW} mt={['6px', 2]}> <Box textAlign={'left'} mt={['6px', 2]}>
<Card bg={'white'} {...MessageCardStyle}> <Card bg={'white'} {...MessageCardStyle}>
<Markdown <Markdown
source={item.value} source={item.value}
...@@ -869,7 +854,7 @@ const ChatBox = ( ...@@ -869,7 +854,7 @@ const ChatBox = (
</Box> </Box>
</> </>
)} )}
</Flex> </Box>
))} ))}
</Box> </Box>
</Box> </Box>
......
client/src/constants/user.ts
...@@ -14,7 +14,7 @@ export enum PageTypeEnum { ...@@ -14,7 +14,7 @@ export enum PageTypeEnum {
} }
export const BillSourceMap: Record<`${BillSourceEnum}`, string> = { export const BillSourceMap: Record<`${BillSourceEnum}`, string> = {
[BillSourceEnum.fastgpt]: 'FastGPT 平台', [BillSourceEnum.fastgpt]: '在线使用',
[BillSourceEnum.api]: 'Api', [BillSourceEnum.api]: 'Api',
[BillSourceEnum.shareLink]: '免登录链接' [BillSourceEnum.shareLink]: '免登录链接'
}; };
......
client/src/pages/account/components/Info.tsx
...@@ -251,7 +251,7 @@ const UserInfo = () => { ...@@ -251,7 +251,7 @@ const UserInfo = () => {
</Flex> </Flex>
</> </>
)} )}
{feConfigs?.show_userDetail && ( {feConfigs?.show_openai_account && (
<> <>
<Divider my={3} /> <Divider my={3} />
......
client/src/pages/account/components/PayModal.tsx
...@@ -6,8 +6,10 @@ import { useQuery } from '@tanstack/react-query'; ...@@ -6,8 +6,10 @@ import { useQuery } from '@tanstack/react-query';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { getErrText } from '@/utils/tools'; import { getErrText } from '@/utils/tools';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { formatPrice } from '@/utils/user';
import Markdown from '@/components/Markdown'; import Markdown from '@/components/Markdown';
import MyModal from '@/components/MyModal'; import MyModal from '@/components/MyModal';
import { vectorModelList, chatModelList, qaModel } from '@/store/static';
const PayModal = ({ onClose }: { onClose: () => void }) => { const PayModal = ({ onClose }: { onClose: () => void }) => {
const router = useRouter(); const router = useRouter();
...@@ -69,6 +71,7 @@ const PayModal = ({ onClose }: { onClose: () => void }) => { ...@@ -69,6 +71,7 @@ const PayModal = ({ onClose }: { onClose: () => void }) => {
onClose(); onClose();
}} }}
title={t('user.Pay')} title={t('user.Pay')}
isCentered
showCloseBtn={!payId} showCloseBtn={!payId}
> >
<ModalBody py={0}> <ModalBody py={0}>
...@@ -100,11 +103,13 @@ const PayModal = ({ onClose }: { onClose: () => void }) => { ...@@ -100,11 +103,13 @@ const PayModal = ({ onClose }: { onClose: () => void }) => {
source={` source={`
| 计费项 | 价格: 元/ 1K tokens(包含上下文)| | 计费项 | 价格: 元/ 1K tokens(包含上下文)|
| --- | --- | | --- | --- |
| 知识库 - 索引 | 0.002 | ${vectorModelList
| FastAI4k - 对话 | 0.015 | .map((item) => `| 索引-${item.name} | ${formatPrice(item.price, 1000)} |`)
| FastAI16k - 对话 | 0.03 | .join('\n')}
| FastAI-Plus - 对话 | 0.45 | ${chatModelList
| 文件QA拆分 | 0.03 |`} .map((item) => `| 对话-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
| 文件QA拆分 | ${formatPrice(qaModel.price, 1000)} |`}
/> />
</> </>
)} )}
......
client/src/pages/account/index.tsx
...@@ -31,7 +31,7 @@ enum TabEnum { ...@@ -31,7 +31,7 @@ enum TabEnum {
const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => { const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
const { t } = useTranslation(); const { t } = useTranslation();
const tabList = useRef([ const tabList = [
{ {
icon: 'meLight', icon: 'meLight',
label: t('user.Personal Information'), label: t('user.Personal Information'),
...@@ -67,7 +67,7 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => { ...@@ -67,7 +67,7 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
label: t('user.Sign Out'), label: t('user.Sign Out'),
id: TabEnum.loginout id: TabEnum.loginout
} }
]); ];
const { openConfirm, ConfirmModal } = useConfirm({ const { openConfirm, ConfirmModal } = useConfirm({
content: '确认退出登录？' content: '确认退出登录？'
...@@ -115,7 +115,7 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => { ...@@ -115,7 +115,7 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
mx={'auto'} mx={'auto'}
mt={2} mt={2}
w={'100%'} w={'100%'}
list={tabList.current} list={tabList}
activeId={currentTab} activeId={currentTab}
onChange={setCurrentTab} onChange={setCurrentTab}
/> />
...@@ -125,7 +125,7 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => { ...@@ -125,7 +125,7 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
<Tabs <Tabs
m={'auto'} m={'auto'}
size={isPc ? 'md' : 'sm'} size={isPc ? 'md' : 'sm'}
list={tabList.current.map((item) => ({ list={tabList.map((item) => ({
id: item.id, id: item.id,
label: item.label label: item.label
}))} }))}
......
client/src/pages/api/admin/initv442.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase, Bill } from '@/service/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authRoot: true });
try {
await Bill.collection.dropIndex('time_1');
} catch (error) {}
try {
await Bill.collection.createIndex({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 * 60 });
} catch (error) {}
jsonRes(res, {
data: {}
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}
client/src/pages/api/core/dataset/file/delById.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
...@@ -21,6 +21,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -21,6 +21,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
// other data. Delete only vector data
if (fileId === OtherFileId) { if (fileId === OtherFileId) {
await PgClient.delete(PgDatasetTableName, { await PgClient.delete(PgDatasetTableName, {
where: [ where: [
...@@ -31,6 +32,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -31,6 +32,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
] ]
}); });
} else { } else {
// auth file
const gridFs = new GridFSStorage('dataset', userId); const gridFs = new GridFSStorage('dataset', userId);
const bucket = gridFs.GridFSBucket(); const bucket = gridFs.GridFSBucket();
...@@ -40,6 +42,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -40,6 +42,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await PgClient.delete(PgDatasetTableName, { await PgClient.delete(PgDatasetTableName, {
where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['file_id', fileId]] where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['file_id', fileId]]
}); });
// delete all training data
await TrainingData.deleteMany({
userId,
file_id: fileId
});
// delete file // delete file
await bucket.delete(new Types.ObjectId(fileId)); await bucket.delete(new Types.ObjectId(fileId));
......
client/src/pages/api/openapi/postKey.ts
...@@ -18,7 +18,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -18,7 +18,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error('最多 10 组 API 秘钥'); throw new Error('最多 10 组 API 秘钥');
} }
const apiKey = `fastgpt-${nanoid()}`; const apiKey = `${global.systemEnv?.openapiPrefix || 'fastgpt'}-${nanoid()}`;
await OpenApi.create({ await OpenApi.create({
userId, userId,
......
client/src/pages/api/openapi/v1/chat/completions.ts
...@@ -116,7 +116,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex ...@@ -116,7 +116,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
appId, appId,
userId userId
}), }),
getChatHistory({ chatId, userId }) getChatHistory({ chatId, appId, userId })
]); ]);
const isOwner = !shareId && userId === String(app.userId); const isOwner = !shareId && userId === String(app.userId);
......
client/src/pages/api/openapi/v1/chat/getHistory.ts
...@@ -7,6 +7,7 @@ import { Types } from 'mongoose'; ...@@ -7,6 +7,7 @@ import { Types } from 'mongoose';
import type { ChatItemType } from '@/types/chat'; import type { ChatItemType } from '@/types/chat';
export type Props = { export type Props = {
appId?: string;
chatId?: string; chatId?: string;
limit?: number; limit?: number;
}; };
...@@ -36,9 +37,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -36,9 +37,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
export async function getChatHistory({ export async function getChatHistory({
chatId, chatId,
userId, userId,
appId,
limit = 30 limit = 30
}: Props & { userId: string }): Promise<Response> { }: Props & { userId: string }): Promise<Response> {
if (!chatId) { if (!chatId || !appId) {
return { history: [] }; return { history: [] };
} }
...@@ -46,6 +48,7 @@ export async function getChatHistory({ ...@@ -46,6 +48,7 @@ export async function getChatHistory({
{ {
$match: { $match: {
chatId, chatId,
appId: new Types.ObjectId(appId),
userId: new Types.ObjectId(userId) userId: new Types.ObjectId(userId)
} }
}, },
......
client/src/pages/api/plugins/kb/data/exportModelData.ts → client/src/pages/api/plugins/kb/data/exportAll.ts
...@@ -2,9 +2,10 @@ import type { NextApiRequest, NextApiResponse } from 'next'; ...@@ -2,9 +2,10 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, User } from '@/service/mongo'; import { connectToDatabase, User } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@/service/utils/auth';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { findAllChildrenIds } from '../delete'; import { findAllChildrenIds } from '../delete';
import QueryStream from 'pg-query-stream';
import Papa from 'papaparse';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
...@@ -12,7 +13,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -12,7 +13,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
kbId: string; kbId: string;
}; };
if (!kbId) { if (!kbId || !global.pgClient) {
throw new Error('缺少参数'); throw new Error('缺少参数');
} }
...@@ -22,7 +23,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -22,7 +23,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
const exportIds = [kbId, ...(await findAllChildrenIds(kbId))]; const exportIds = [kbId, ...(await findAllChildrenIds(kbId))];
console.log(exportIds);
const thirtyMinutesAgo = new Date( const thirtyMinutesAgo = new Date(
Date.now() - (global.feConfigs?.limit?.exportLimitMinutes || 0) * 60 * 1000 Date.now() - (global.feConfigs?.limit?.exportLimitMinutes || 0) * 60 * 1000
...@@ -45,37 +45,50 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -45,37 +45,50 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error(`上次导出未到 ${minutes}，每 ${minutes}仅可导出一次。`); throw new Error(`上次导出未到 ${minutes}，每 ${minutes}仅可导出一次。`);
} }
const where: any = [ // connect pg
['user_id', userId], global.pgClient.connect((err, client, done) => {
'AND', if (err) {
`kb_id IN (${exportIds.map((id) => `'${id}'`).join(',')})` console.error(err);
]; res.end('Error connecting to database');
// 从 pg 中获取所有数据 return;
const pgData = await PgClient.select<{ q: string; a: string; source: string }>(
PgDatasetTableName,
{
where,
fields: ['q', 'a', 'source'],
order: [{ field: 'id', mode: 'DESC' }],
limit: 1000000
} }
); // create pg select stream
const query = new QueryStream(
`SELECT q, a, source FROM ${PgDatasetTableName} where user_id='${userId}' AND kb_id IN (${exportIds
.map((id) => `'${id}'`)
.join(',')})`
);
const stream = client.query(query);
const data: [string, string, string][] = pgData.rows.map((item) => [ res.setHeader('Content-Disposition', 'attachment; filename=dataset.csv');
item.q.replace(/\n/g, '\\n'), res.setHeader('Content-Type', 'text/csv');
item.a.replace(/\n/g, '\\n'),
item.source
]);
// update export time res.write('index,content,source');
await User.findByIdAndUpdate(userId, {
'limit.exportKbTime': new Date()
});
jsonRes(res, { // parse data every row
data stream.on('data', (row: { q: string; a: string; source?: string }) => {
const csv = Papa.unparse([row], { header: false });
res.write(`\n${csv}`);
});
stream.on('end', async () => {
try {
// update export time
await User.findByIdAndUpdate(userId, {
'limit.exportKbTime': new Date()
});
} catch (error) {}
// close response
done();
res.end();
});
stream.on('error', (err) => {
done(err);
res.end('Error exporting data');
});
}); });
} catch (err) { } catch (err) {
res.status(500);
jsonRes(res, { jsonRes(res, {
code: 500, code: 500,
error: err error: err
......
client/src/pages/app/detail/components/API.tsx
...@@ -4,6 +4,7 @@ import { useCopyData } from '@/utils/tools'; ...@@ -4,6 +4,7 @@ import { useCopyData } from '@/utils/tools';
import dynamic from 'next/dynamic'; import dynamic from 'next/dynamic';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
import { useGlobalStore } from '@/store/global'; import { useGlobalStore } from '@/store/global';
import { feConfigs } from '@/store/static';
const APIKeyModal = dynamic(() => import('@/components/APIKeyModal'), { const APIKeyModal = dynamic(() => import('@/components/APIKeyModal'), {
ssr: false ssr: false
...@@ -77,7 +78,10 @@ const API = ({ appId }: { appId: string }) => { ...@@ -77,7 +78,10 @@ const API = ({ appId }: { appId: string }) => {
width: '100%', width: '100%',
height: '100%' height: '100%'
}} }}
src="https://kjqvjse66l.feishu.cn/docx/DmLedTWtUoNGX8xui9ocdUEjnNh" src={
feConfigs?.openAPIUrl ||
'https://kjqvjse66l.feishu.cn/docx/DmLedTWtUoNGX8xui9ocdUEjnNh'
}
frameBorder="0" frameBorder="0"
onLoad={() => setIsLoaded(true)} onLoad={() => setIsLoaded(true)}
onError={() => setIsLoaded(true)} onError={() => setIsLoaded(true)}
......
client/src/pages/app/detail/components/KBSelectModal.tsx
...@@ -27,6 +27,7 @@ import { KbTypeEnum } from '@/constants/kb'; ...@@ -27,6 +27,7 @@ import { KbTypeEnum } from '@/constants/kb';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { useQuery } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query';
import { useDatasetStore } from '@/store/dataset'; import { useDatasetStore } from '@/store/dataset';
import { feConfigs } from '@/store/static';
export type KbParamsType = { export type KbParamsType = {
searchSimilarity: number; searchSimilarity: number;
...@@ -345,9 +346,7 @@ export const KbParamsModal = ({ ...@@ -345,9 +346,7 @@ export const KbParamsModal = ({
<Textarea <Textarea
rows={5} rows={5}
maxLength={500} maxLength={500}
placeholder={ placeholder={`若填写该内容，没有搜索到对应内容时，将直接回复填写的内容。\n为了连贯上下文，${feConfigs?.systemTitle} 会取部分上一个聊天的搜索记录作为补充，因此在连续对话时，该功能可能会失效。`}
'若填写该内容，没有搜索到对应内容时，将直接回复填写的内容。\n为了连贯上下文，FastGPT 会取部分上一个聊天的搜索记录作为补充，因此在连续对话时，该功能可能会失效。'
}
{...register('searchEmptyText')} {...register('searchEmptyText')}
></Textarea> ></Textarea>
</Box> </Box>
......
client/src/pages/app/detail/index.tsx
...@@ -35,7 +35,8 @@ enum TabEnum { ...@@ -35,7 +35,8 @@ enum TabEnum {
'adEdit' = 'adEdit', 'adEdit' = 'adEdit',
'outLink' = 'outLink', 'outLink' = 'outLink',
'logs' = 'logs', 'logs' = 'logs',
'API' = 'API' 'API' = 'API',
'startChat' = 'startChat'
} }
const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => { const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
...@@ -64,7 +65,7 @@ const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => { ...@@ -64,7 +65,7 @@ const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
{ label: '外部使用', id: TabEnum.outLink, icon: 'shareLight' }, { label: '外部使用', id: TabEnum.outLink, icon: 'shareLight' },
{ label: 'API访问', id: TabEnum.API, icon: 'apiLight' }, { label: 'API访问', id: TabEnum.API, icon: 'apiLight' },
{ label: '对话日志', id: TabEnum.logs, icon: 'logsLight' }, { label: '对话日志', id: TabEnum.logs, icon: 'logsLight' },
{ label: '立即对话', id: 'startChat', icon: 'chat' } { label: '立即对话', id: TabEnum.startChat, icon: 'chat' }
], ],
[] []
); );
......
client/src/pages/chat/components/ChatHeader.tsx
...@@ -49,7 +49,7 @@ const ChatHeader = ({ ...@@ -49,7 +49,7 @@ const ChatHeader = ({
<MyIcon name={'history'} w={'14px'} /> <MyIcon name={'history'} w={'14px'} />
<Box ml={1}>{history.length === 0 ? '新的对话' : `${history.length}条记录`}</Box> <Box ml={1}>{history.length === 0 ? '新的对话' : `${history.length}条记录`}</Box>
</Tag> </Tag>
{!!chatModels && ( {!!chatModels && chatModels.length > 0 && (
<Tag ml={2} colorSchema={'green'}> <Tag ml={2} colorSchema={'green'}>
<MyIcon name={'chatModelTag'} w={'14px'} /> <MyIcon name={'chatModelTag'} w={'14px'} />
<Box ml={1}>{chatModels.join(',')}</Box> <Box ml={1}>{chatModels.join(',')}</Box>
......
client/src/pages/kb/detail/components/Import/Csv.tsx
...@@ -217,7 +217,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => { ...@@ -217,7 +217,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => {
/> />
</Flex> </Flex>
<Box px={4} fontSize={'sm'} whiteSpace={'pre-wrap'} wordBreak={'break-all'}> <Box px={4} fontSize={'sm'} whiteSpace={'pre-wrap'} wordBreak={'break-all'}>
{`q: ${item.q}\na: ${item.a}`} {`${item.q}\n${item.a}`}
</Box> </Box>
</Box> </Box>
)) ))
......
client/src/pages/kb/detail/components/Import/FileSelect.tsx
...@@ -25,7 +25,7 @@ const UrlFetchModal = dynamic(() => import('./UrlFetchModal')); ...@@ -25,7 +25,7 @@ const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
const CreateFileModal = dynamic(() => import('./CreateFileModal')); const CreateFileModal = dynamic(() => import('./CreateFileModal'));
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12); const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
const csvTemplate = `question,answer\n"什么是 laf","laf 是一个云函数开发平台……"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`; const csvTemplate = `index,content,source\n"被索引的内容","对应的答案。CSV 中请注意内容不能包含双引号，双引号是列分割符号","来源，可选。"\n"什么是 laf","laf 是一个云函数开发平台……",""\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……",""`;
export type FileItemType = { export type FileItemType = {
id: string; id: string;
...@@ -149,8 +149,8 @@ const FileSelect = ({ ...@@ -149,8 +149,8 @@ const FileSelect = ({
/* csv file */ /* csv file */
if (extension === 'csv') { if (extension === 'csv') {
const { header, data } = await readCsvContent(file); const { header, data } = await readCsvContent(file);
if (header[0] !== 'question' || header[1] !== 'answer') { if (header[0] !== 'index' || header[1] !== 'content') {
throw new Error('csv 文件格式有误,请确保 question 和 answer 两列'); throw new Error('csv 文件格式有误,请确保 index 和 content 两列');
} }
const fileItem: FileItemType = { const fileItem: FileItemType = {
id: filesId[0], id: filesId[0],
...@@ -158,12 +158,14 @@ const FileSelect = ({ ...@@ -158,12 +158,14 @@ const FileSelect = ({
icon, icon,
tokens: 0, tokens: 0,
text: '', text: '',
chunks: data.map((item) => ({ chunks: data
q: item[0], .filter((item) => item[0])
a: item[1], .map((item) => ({
source: item[2] || file.name, q: item[0] || '',
file_id: filesId[0] a: item[1] || '',
})) source: item[2] || file.name || '',
file_id: filesId[0]
}))
}; };
chunkFiles.unshift(fileItem); chunkFiles.unshift(fileItem);
......
client/src/pages/kb/list/index.tsx
...@@ -15,7 +15,7 @@ import PageContainer from '@/components/PageContainer'; ...@@ -15,7 +15,7 @@ import PageContainer from '@/components/PageContainer';
import { useConfirm } from '@/hooks/useConfirm'; import { useConfirm } from '@/hooks/useConfirm';
import { AddIcon } from '@chakra-ui/icons'; import { AddIcon } from '@chakra-ui/icons';
import { useQuery } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query';
import { delKbById, getExportDataList, getKbPaths, putKbById } from '@/api/plugins/kb'; import { delKbById, exportDataset, getKbPaths, putKbById } from '@/api/plugins/kb';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import Avatar from '@/components/Avatar'; import Avatar from '@/components/Avatar';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
...@@ -27,8 +27,6 @@ import MyMenu from '@/components/MyMenu'; ...@@ -27,8 +27,6 @@ import MyMenu from '@/components/MyMenu';
import { useRequest } from '@/hooks/useRequest'; import { useRequest } from '@/hooks/useRequest';
import { useGlobalStore } from '@/store/global'; import { useGlobalStore } from '@/store/global';
import { useEditTitle } from '@/hooks/useEditTitle'; import { useEditTitle } from '@/hooks/useEditTitle';
import Papa from 'papaparse';
import { fileDownload } from '@/utils/file';
import { feConfigs } from '@/store/static'; import { feConfigs } from '@/store/static';
const CreateModal = dynamic(() => import('./component/CreateModal'), { ssr: false }); const CreateModal = dynamic(() => import('./component/CreateModal'), { ssr: false });
...@@ -90,19 +88,7 @@ const Kb = () => { ...@@ -90,19 +88,7 @@ const Kb = () => {
const { mutate: onclickExport } = useRequest({ const { mutate: onclickExport } = useRequest({
mutationFn: (kbId: string) => { mutationFn: (kbId: string) => {
setLoading(true); setLoading(true);
return getExportDataList({ kbId }); return exportDataset({ kbId });
},
onSuccess(res) {
const text = Papa.unparse({
fields: ['question', 'answer', 'source'],
data: res
});
fileDownload({
text,
type: 'text/csv',
filename: 'dataset.csv'
});
}, },
onSettled() { onSettled() {
setLoading(false); setLoading(false);
......
client/src/pages/login/components/RegisterForm.tsx
...@@ -65,13 +65,15 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => { ...@@ -65,13 +65,15 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
status: 'success' status: 'success'
}); });
// auto register template app // auto register template app
appTemplates.forEach((template) => { setTimeout(() => {
postCreateApp({ appTemplates.forEach((template) => {
avatar: template.avatar, postCreateApp({
name: template.name, avatar: template.avatar,
modules: template.modules name: template.name,
modules: template.modules
});
}); });
}); }, 100);
} catch (error: any) { } catch (error: any) {
toast({ toast({
title: error.message || '注册异常', title: error.message || '注册异常',
......
client/src/pages/login/index.tsx
...@@ -36,7 +36,7 @@ const Login = () => { ...@@ -36,7 +36,7 @@ const Login = () => {
setToken(res.token); setToken(res.token);
setTimeout(() => { setTimeout(() => {
router.push(lastRoute ? decodeURIComponent(lastRoute) : '/app/list'); router.push(lastRoute ? decodeURIComponent(lastRoute) : '/app/list');
}, 100); }, 300);
}, },
[lastRoute, router, setLastChatId, setLastChatAppId, setUserInfo] [lastRoute, router, setLastChatId, setLastChatAppId, setUserInfo]
); );
......
client/src/service/models/bill.ts
...@@ -55,7 +55,7 @@ const BillSchema = new Schema({ ...@@ -55,7 +55,7 @@ const BillSchema = new Schema({
try { try {
BillSchema.index({ userId: 1 }); BillSchema.index({ userId: 1 });
BillSchema.index({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 }); BillSchema.index({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 * 60 });
} catch (error) { } catch (error) {
console.log(error); console.log(error);
} }
......
client/src/types/index.d.ts
...@@ -23,9 +23,10 @@ export type FeConfigsType = { ...@@ -23,9 +23,10 @@ export type FeConfigsType = {
show_contact?: boolean; show_contact?: boolean;
show_git?: boolean; show_git?: boolean;
show_doc?: boolean; show_doc?: boolean;
show_openai_account?: boolean;
openAPIUrl?: string;
systemTitle?: string; systemTitle?: string;
authorText?: string; authorText?: string;
beianText?: string;
googleClientVerKey?: string; googleClientVerKey?: string;
oauth?: { oauth?: {
github?: string; github?: string;
...@@ -38,6 +39,7 @@ export type FeConfigsType = { ...@@ -38,6 +39,7 @@ export type FeConfigsType = {
}; };
export type SystemEnvType = { export type SystemEnvType = {
pluginBaseUrl?: string; pluginBaseUrl?: string;
openapiPrefix?: string;
vectorMaxProcess: number; vectorMaxProcess: number;
qaMaxProcess: number; qaMaxProcess: number;
pgIvfflatProbe: number; pgIvfflatProbe: number;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment