Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit aa55f059 by Archer Committed by GitHub
Options

perf: chat history api;perf: full text error (#4852) 

* perf: chat history api

* perf: i18n

* perf: full text
parent 89c9a026
Showing with 192 additions and 124 deletions
docSite/content/zh-cn/docs/development/openapi/chat.md
...@@ -959,10 +959,16 @@ curl --location --request POST 'http://localhost:3000/api/core/chat/getHistories ...@@ -959,10 +959,16 @@ curl --location --request POST 'http://localhost:3000/api/core/chat/getHistories
{{< markdownify >}} {{< markdownify >}}
{{% alert icon=" " context="success" %}} {{% alert icon=" " context="success" %}}
目前仅能获取到当前 API key 的创建者的对话。
- appId - 应用 Id - appId - 应用 Id
- offset - 偏移量，即从第几条数据开始取 - offset - 偏移量，即从第几条数据开始取
- pageSize - 记录数量 - pageSize - 记录数量
- source - 对话源。source=api，表示获取通过 API 创建的对话（不会获取到页面上的对话记录） - source - 对话源。source=api，表示获取通过 API 创建的对话（不会获取到页面上的对话记录）
- startCreateTime - 开始创建时间（可选）
- endCreateTime - 结束创建时间（可选）
- startUpdateTime - 开始更新时间（可选）
- endUpdateTime - 结束更新时间（可选）
{{% /alert %}} {{% /alert %}}
{{< /markdownify >}} {{< /markdownify >}}
......
packages/global/core/chat/type.d.ts
...@@ -26,6 +26,7 @@ export type ChatSchema = { ...@@ -26,6 +26,7 @@ export type ChatSchema = {
teamId: string; teamId: string;
tmbId: string; tmbId: string;
appId: string; appId: string;
createTime: Date;
updateTime: Date; updateTime: Date;
title: string; title: string;
customTitle: string; customTitle: string;
......
packages/service/core/chat/chatSchema.ts
...@@ -34,6 +34,10 @@ const ChatSchema = new Schema({ ...@@ -34,6 +34,10 @@ const ChatSchema = new Schema({
ref: AppCollectionName, ref: AppCollectionName,
required: true required: true
}, },
createTime: {
type: Date,
default: () => new Date()
},
updateTime: { updateTime: {
type: Date, type: Date,
default: () => new Date() default: () => new Date()
......
packages/service/core/dataset/search/controller.ts
...@@ -27,6 +27,7 @@ import { type ChatItemType } from '@fastgpt/global/core/chat/type'; ...@@ -27,6 +27,7 @@ import { type ChatItemType } from '@fastgpt/global/core/chat/type';
import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { datasetSearchQueryExtension } from './utils'; import { datasetSearchQueryExtension } from './utils';
import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.d'; import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.d';
import { addLog } from '../../../common/system/log';
export type SearchDatasetDataProps = { export type SearchDatasetDataProps = {
histories: ChatItemType[]; histories: ChatItemType[];
...@@ -544,117 +545,125 @@ export async function searchDatasetData( ...@@ -544,117 +545,125 @@ export async function searchDatasetData(
}; };
} }
const searchResults = (await MongoDatasetDataText.aggregate( try {
[ const searchResults = (await MongoDatasetDataText.aggregate(
{ [
$match: { {
teamId: new Types.ObjectId(teamId), $match: {
$text: { $search: await jiebaSplit({ text: query }) }, teamId: new Types.ObjectId(teamId),
datasetId: { $in: datasetIds.map((id) => new Types.ObjectId(id)) }, $text: { $search: await jiebaSplit({ text: query }) },
...(filterCollectionIdList datasetId: { $in: datasetIds.map((id) => new Types.ObjectId(id)) },
? { ...(filterCollectionIdList
collectionId: { ? {
$in: filterCollectionIdList.map((id) => new Types.ObjectId(id)) collectionId: {
$in: filterCollectionIdList.map((id) => new Types.ObjectId(id))
}
} }
} : {}),
: {}), ...(forbidCollectionIdList && forbidCollectionIdList.length > 0
...(forbidCollectionIdList && forbidCollectionIdList.length > 0 ? {
? { collectionId: {
collectionId: { $nin: forbidCollectionIdList.map((id) => new Types.ObjectId(id))
$nin: forbidCollectionIdList.map((id) => new Types.ObjectId(id)) }
} }
} : {})
: {}) }
},
{
$sort: {
score: { $meta: 'textScore' }
}
},
{
$limit: limit
},
{
$project: {
_id: 1,
collectionId: 1,
dataId: 1,
score: { $meta: 'textScore' }
}
} }
}, ],
{ {
$sort: { ...readFromSecondary
score: { $meta: 'textScore' }
}
},
{
$limit: limit
},
{
$project: {
_id: 1,
collectionId: 1,
dataId: 1,
score: { $meta: 'textScore' }
}
} }
], )) as (DatasetDataTextSchemaType & { score: number })[];
{
...readFromSecondary
}
)) as (DatasetDataTextSchemaType & { score: number })[];
// Get data and collections // Get data and collections
const [dataList, collections] = await Promise.all([ const [dataList, collections] = await Promise.all([
MongoDatasetData.find( MongoDatasetData.find(
{ {
_id: { $in: searchResults.map((item) => item.dataId) } _id: { $in: searchResults.map((item) => item.dataId) }
}, },
'_id datasetId collectionId updateTime q a chunkIndex indexes', '_id datasetId collectionId updateTime q a chunkIndex indexes',
{ ...readFromSecondary } { ...readFromSecondary }
).lean(), ).lean(),
MongoDatasetCollection.find( MongoDatasetCollection.find(
{ {
_id: { $in: searchResults.map((item) => item.collectionId) } _id: { $in: searchResults.map((item) => item.collectionId) }
}, },
'_id name fileId rawLink apiFileId externalFileId externalFileUrl', '_id name fileId rawLink apiFileId externalFileId externalFileUrl',
{ ...readFromSecondary } { ...readFromSecondary }
).lean() ).lean()
]); ]);
return { return {
fullTextRecallResults: searchResults fullTextRecallResults: searchResults
.map((item, index) => { .map((item, index) => {
const collection = collections.find( const collection = collections.find(
(col) => String(col._id) === String(item.collectionId) (col) => String(col._id) === String(item.collectionId)
); );
if (!collection) { if (!collection) {
console.log('Collection is not found', item); console.log('Collection is not found', item);
return; return;
} }
const data = dataList.find((data) => String(data._id) === String(item.dataId)); const data = dataList.find((data) => String(data._id) === String(item.dataId));
if (!data) { if (!data) {
console.log('Data is not found', item); console.log('Data is not found', item);
return; return;
} }
return { return {
id: String(data._id), id: String(data._id),
datasetId: String(data.datasetId), datasetId: String(data.datasetId),
collectionId: String(data.collectionId), collectionId: String(data.collectionId),
updateTime: data.updateTime, updateTime: data.updateTime,
q: data.q, q: data.q,
a: data.a, a: data.a,
chunkIndex: data.chunkIndex, chunkIndex: data.chunkIndex,
indexes: data.indexes, indexes: data.indexes,
...getCollectionSourceData(collection), ...getCollectionSourceData(collection),
score: [ score: [
{ {
type: SearchScoreTypeEnum.fullText, type: SearchScoreTypeEnum.fullText,
value: item.score || 0, value: item.score || 0,
index index
} }
] ]
}; };
}) })
.filter((item) => { .filter((item) => {
if (!item) return false; if (!item) return false;
return true; return true;
}) })
.map((item, index) => { .map((item, index) => {
if (!item) return; if (!item) return;
return { return {
...item, ...item,
score: item.score.map((item) => ({ ...item, index })) score: item.score.map((item) => ({ ...item, index }))
}; };
}) as SearchDataResponseItemType[], }) as SearchDataResponseItemType[],
tokenLen: 0 tokenLen: 0
}; };
} catch (error) {
addLog.error('multiQueryRecall error', error);
return {
fullTextRecallResults: [],
tokenLen: 0
};
}
}; };
const multiQueryRecall = async ({ const multiQueryRecall = async ({
embeddingLimit, embeddingLimit,
......
packages/web/common/system/utils.ts
...@@ -6,10 +6,6 @@ export const getUserFingerprint = async () => { ...@@ -6,10 +6,6 @@ export const getUserFingerprint = async () => {
console.log(result.visitorId); console.log(result.visitorId);
}; };
export const hasHttps = () => {
return window.location.protocol === 'https:';
};
export const subRoute = process.env.NEXT_PUBLIC_BASE_URL; export const subRoute = process.env.NEXT_PUBLIC_BASE_URL;
export const getWebReqUrl = (url: string = '') => { export const getWebReqUrl = (url: string = '') => {
......
packages/web/hooks/useCopyData.tsx
import { useTranslation } from 'next-i18next'; import { useTranslation } from 'next-i18next';
import { useToast } from './useToast'; import { useToast } from './useToast';
import { useCallback } from 'react'; import { useCallback } from 'react';
import { hasHttps } from '../common/system/utils';
import { isProduction } from '@fastgpt/global/common/system/constants';
import MyModal from '../components/common/MyModal'; import MyModal from '../components/common/MyModal';
import React from 'react'; import React from 'react';
import { Box, ModalBody } from '@chakra-ui/react'; import { Box, ModalBody } from '@chakra-ui/react';
...@@ -26,7 +24,7 @@ export const useCopyData = () => { ...@@ -26,7 +24,7 @@ export const useCopyData = () => {
data = data.trim(); data = data.trim();
try { try {
if ((hasHttps() || !isProduction) && navigator.clipboard) { if (navigator.clipboard && window.isSecureContext) {
await navigator.clipboard.writeText(data); await navigator.clipboard.writeText(data);
if (title) { if (title) {
toast({ toast({
...@@ -36,13 +34,35 @@ export const useCopyData = () => { ...@@ -36,13 +34,35 @@ export const useCopyData = () => {
}); });
} }
} else { } else {
throw new Error(''); let textArea = document.createElement('textarea');
textArea.value = data;
// 使text area不在viewport，同时设置不可见
textArea.style.position = 'absolute';
// @ts-ignore
textArea.style.opacity = 0;
textArea.style.left = '-999999px';
textArea.style.top = '-999999px';
document.body.appendChild(textArea);
textArea.focus();
textArea.select();
await new Promise((res, rej) => {
document.execCommand('copy') ? res('') : rej();
textArea.remove();
}).then(() => {
if (title) {
toast({
title,
status: 'success',
duration
});
}
});
} }
} catch (error) { } catch (error) {
setCopyContent(data); setCopyContent(data);
} }
}, },
[t, toast] [setCopyContent, t, toast]
); );
return { return {
......
packages/web/i18n/en/common.json
...@@ -941,6 +941,7 @@ ...@@ -941,6 +941,7 @@
"pay_corporate_payment": "Payment to the public", "pay_corporate_payment": "Payment to the public",
"pay_money": "Amount payable", "pay_money": "Amount payable",
"pay_success": "Payment successfully", "pay_success": "Payment successfully",
"pay_year_tip": "Pay 10 months, enjoy 1 year!",
"permission.Collaborator": "Collaborator", "permission.Collaborator": "Collaborator",
"permission.Default permission": "Default Permission", "permission.Default permission": "Default Permission",
"permission.Manage": "Manage", "permission.Manage": "Manage",
...@@ -1144,7 +1145,7 @@ ...@@ -1144,7 +1145,7 @@
"support.wallet.subscription.Next plan": "Future Package", "support.wallet.subscription.Next plan": "Future Package",
"support.wallet.subscription.Stand plan level": "Subscription Package", "support.wallet.subscription.Stand plan level": "Subscription Package",
"support.wallet.subscription.Sub plan": "Subscription Package", "support.wallet.subscription.Sub plan": "Subscription Package",
"support.wallet.subscription.Sub plan tip": "Free to use {{title}} or upgrade to a higher package", "support.wallet.subscription.Sub plan tip": "Free to use [{{title}}] or upgrade to a higher package",
"support.wallet.subscription.Team plan and usage": "Package and Usage", "support.wallet.subscription.Team plan and usage": "Package and Usage",
"support.wallet.subscription.Training weight": "Training Priority: {{weight}}", "support.wallet.subscription.Training weight": "Training Priority: {{weight}}",
"support.wallet.subscription.Update extra ai points": "Extra AI Points", "support.wallet.subscription.Update extra ai points": "Extra AI Points",
...@@ -1159,7 +1160,6 @@ ...@@ -1159,7 +1160,6 @@
"support.wallet.subscription.function.Points": "{{amount}} AI Points", "support.wallet.subscription.function.Points": "{{amount}} AI Points",
"support.wallet.subscription.mode.Month": "Month", "support.wallet.subscription.mode.Month": "Month",
"support.wallet.subscription.mode.Period": "Subscription Period", "support.wallet.subscription.mode.Period": "Subscription Period",
"support.wallet.subscription.mode.Ten Year": "Pay 10 months, imagine 1 year!",
"support.wallet.subscription.mode.Year": "Year", "support.wallet.subscription.mode.Year": "Year",
"support.wallet.subscription.mode.Year sale": "Two Months Free", "support.wallet.subscription.mode.Year sale": "Two Months Free",
"support.wallet.subscription.point": "Points", "support.wallet.subscription.point": "Points",
......
packages/web/i18n/zh-CN/common.json
...@@ -940,6 +940,7 @@ ...@@ -940,6 +940,7 @@
"pay_corporate_payment": "对公支付", "pay_corporate_payment": "对公支付",
"pay_money": "应付金额", "pay_money": "应付金额",
"pay_success": "支付成功", "pay_success": "支付成功",
"pay_year_tip": "支付 10 个月，畅享 1 年！",
"permission.Collaborator": "协作者", "permission.Collaborator": "协作者",
"permission.Default permission": "默认权限", "permission.Default permission": "默认权限",
"permission.Manage": "管理", "permission.Manage": "管理",
...@@ -1143,7 +1144,7 @@ ...@@ -1143,7 +1144,7 @@
"support.wallet.subscription.Next plan": "未来套餐", "support.wallet.subscription.Next plan": "未来套餐",
"support.wallet.subscription.Stand plan level": "订阅套餐", "support.wallet.subscription.Stand plan level": "订阅套餐",
"support.wallet.subscription.Sub plan": "订阅套餐", "support.wallet.subscription.Sub plan": "订阅套餐",
"support.wallet.subscription.Sub plan tip": "免费使用 {{title}} 或升级更高的套餐", "support.wallet.subscription.Sub plan tip": "免费使用【{{title}}】或升级更高的套餐",
"support.wallet.subscription.Team plan and usage": "套餐与用量", "support.wallet.subscription.Team plan and usage": "套餐与用量",
"support.wallet.subscription.Training weight": "训练优先级：{{weight}}", "support.wallet.subscription.Training weight": "训练优先级：{{weight}}",
"support.wallet.subscription.Update extra ai points": "额外 AI 积分", "support.wallet.subscription.Update extra ai points": "额外 AI 积分",
...@@ -1158,7 +1159,6 @@ ...@@ -1158,7 +1159,6 @@
"support.wallet.subscription.function.Points": "{{amount}} AI 积分", "support.wallet.subscription.function.Points": "{{amount}} AI 积分",
"support.wallet.subscription.mode.Month": "按月", "support.wallet.subscription.mode.Month": "按月",
"support.wallet.subscription.mode.Period": "订阅周期", "support.wallet.subscription.mode.Period": "订阅周期",
"support.wallet.subscription.mode.Ten Year": "支付10个月，畅想1年！",
"support.wallet.subscription.mode.Year": "按年", "support.wallet.subscription.mode.Year": "按年",
"support.wallet.subscription.mode.Year sale": "赠送两个月", "support.wallet.subscription.mode.Year sale": "赠送两个月",
"support.wallet.subscription.point": "积分", "support.wallet.subscription.point": "积分",
......
packages/web/i18n/zh-Hant/common.json
...@@ -940,6 +940,7 @@ ...@@ -940,6 +940,7 @@
"pay_corporate_payment": "對公支付", "pay_corporate_payment": "對公支付",
"pay_money": "應付金額", "pay_money": "應付金額",
"pay_success": "支付成功", "pay_success": "支付成功",
"pay_year_tip": "支付 10 個月，暢享 1 年！",
"permission.Collaborator": "協作者", "permission.Collaborator": "協作者",
"permission.Default permission": "預設權限", "permission.Default permission": "預設權限",
"permission.Manage": "管理", "permission.Manage": "管理",
...@@ -1143,7 +1144,7 @@ ...@@ -1143,7 +1144,7 @@
"support.wallet.subscription.Next plan": "未來方案", "support.wallet.subscription.Next plan": "未來方案",
"support.wallet.subscription.Stand plan level": "訂閱方案", "support.wallet.subscription.Stand plan level": "訂閱方案",
"support.wallet.subscription.Sub plan": "訂閱方案", "support.wallet.subscription.Sub plan": "訂閱方案",
"support.wallet.subscription.Sub plan tip": "免費使用 {{title}} 或升級更進階的方案", "support.wallet.subscription.Sub plan tip": "免費使用【{{title}}】或升級更進階的方案",
"support.wallet.subscription.Team plan and usage": "方案與使用量", "support.wallet.subscription.Team plan and usage": "方案與使用量",
"support.wallet.subscription.Training weight": "訓練優先權：{{weight}}", "support.wallet.subscription.Training weight": "訓練優先權：{{weight}}",
"support.wallet.subscription.Update extra ai points": "額外 AI 點數", "support.wallet.subscription.Update extra ai points": "額外 AI 點數",
...@@ -1158,7 +1159,6 @@ ...@@ -1158,7 +1159,6 @@
"support.wallet.subscription.function.Points": "{{amount}} AI 點數", "support.wallet.subscription.function.Points": "{{amount}} AI 點數",
"support.wallet.subscription.mode.Month": "按月", "support.wallet.subscription.mode.Month": "按月",
"support.wallet.subscription.mode.Period": "訂閱週期", "support.wallet.subscription.mode.Period": "訂閱週期",
"support.wallet.subscription.mode.Ten Year": "支付10個月，暢想1年！",
"support.wallet.subscription.mode.Year": "按年", "support.wallet.subscription.mode.Year": "按年",
"support.wallet.subscription.mode.Year sale": "贈送兩個月", "support.wallet.subscription.mode.Year sale": "贈送兩個月",
"support.wallet.subscription.point": "點數", "support.wallet.subscription.point": "點數",
......
projects/app/src/global/core/chat/api.d.ts
...@@ -60,6 +60,11 @@ export type InitChatResponse = { ...@@ -60,6 +60,11 @@ export type InitChatResponse = {
export type GetHistoriesProps = OutLinkChatAuthProps & { export type GetHistoriesProps = OutLinkChatAuthProps & {
appId?: string; appId?: string;
source?: `${ChatSourceEnum}`; source?: `${ChatSourceEnum}`;
startCreateTime?: string;
endCreateTime?: string;
startUpdateTime?: string;
endUpdateTime?: string;
}; };
export type UpdateHistoryProps = OutLinkChatAuthProps & { export type UpdateHistoryProps = OutLinkChatAuthProps & {
......
projects/app/src/pageComponents/price/Standard.tsx
...@@ -89,7 +89,7 @@ const Standard = ({ ...@@ -89,7 +89,7 @@ const Standard = ({
mb={2} mb={2}
mr={'-2'} mr={'-2'}
> >
{t('common:support.wallet.subscription.mode.Ten Year')} {t('common:pay_year_tip')}
</Box> </Box>
<RowTabs <RowTabs
list={[ list={[
...@@ -110,7 +110,7 @@ const Standard = ({ ...@@ -110,7 +110,7 @@ const Standard = ({
onChange={(e) => setSelectSubMode(e as `${SubModeEnum}`)} onChange={(e) => setSelectSubMode(e as `${SubModeEnum}`)}
/> />
</Box> </Box>
<MyIcon name={'price/pricearrow'} mt={'10px'} ml={'3px'} /> <MyIcon name={'price/pricearrow'} mt={'10px'} ml={'6px'} />
</Flex> </Flex>
{/* card */} {/* card */}
......
projects/app/src/pages/api/core/chat/getHistories.ts
...@@ -20,7 +20,18 @@ async function handler( ...@@ -20,7 +20,18 @@ async function handler(
req: ApiRequestProps<getHistoriesBody, getHistoriesQuery>, req: ApiRequestProps<getHistoriesBody, getHistoriesQuery>,
_res: ApiResponseType<any> _res: ApiResponseType<any>
): Promise<PaginationResponse<getHistoriesResponse>> { ): Promise<PaginationResponse<getHistoriesResponse>> {
const { appId, shareId, outLinkUid, teamId, teamToken, source } = req.body; const {
appId,
shareId,
outLinkUid,
teamId,
teamToken,
source,
startCreateTime,
endCreateTime,
startUpdateTime,
endUpdateTime
} = req.body;
const { offset, pageSize } = parsePaginationRequest(req); const { offset, pageSize } = parsePaginationRequest(req);
const match = await (async () => { const match = await (async () => {
...@@ -49,7 +60,7 @@ async function handler( ...@@ -49,7 +60,7 @@ async function handler(
return { return {
tmbId, tmbId,
appId, appId,
source ...(source && { source })
}; };
} }
})(); })();
...@@ -61,13 +72,29 @@ async function handler( ...@@ -61,13 +72,29 @@ async function handler(
}; };
} }
const timeMatch: Record<string, any> = {};
if (startCreateTime || endCreateTime) {
timeMatch.createTime = {
...(startCreateTime && { $gte: new Date(startCreateTime) }),
...(endCreateTime && { $lte: new Date(endCreateTime) })
};
}
if (startUpdateTime || endUpdateTime) {
timeMatch.updateTime = {
...(startUpdateTime && { $gte: new Date(startUpdateTime) }),
...(endUpdateTime && { $lte: new Date(endUpdateTime) })
};
}
const mergeMatch = { ...match, ...timeMatch };
const [data, total] = await Promise.all([ const [data, total] = await Promise.all([
await MongoChat.find(match, 'chatId title top customTitle appId updateTime') await MongoChat.find(mergeMatch, 'chatId title top customTitle appId updateTime')
.sort({ top: -1, updateTime: -1 }) .sort({ top: -1, updateTime: -1 })
.skip(offset) .skip(offset)
.limit(pageSize) .limit(pageSize)
.lean(), .lean(),
MongoChat.countDocuments(match) MongoChat.countDocuments(mergeMatch)
]); ]);
return { return {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment