Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit c5664c7e by Archer Committed by GitHub
Options

feat: vision model (#489) 

* mongo init

* perf: mongo connect

* perf: tts

perf: whisper and tts

peref: tts whisper permission

log

reabase (#488)

* perf: modal

* i18n

* perf: schema lean

* feat: vision model format

* perf: tts loading

* perf: static data

* perf: tts

* feat: image

* perf: image

* perf: upload image and title

* perf: image size

* doc

* perf: color

* doc

* speaking can not select file

* doc
parent 70f33732
Showing with 407 additions and 94 deletions
docSite/content/docs/development/configuration.md
......@@ -36,6 +36,7 @@ weight: 520
"quoteMaxToken": 2000, // 最大引用内容长度
"maxTemperature": 1.2, // 最大温度值
"censor": false, // 是否开启敏感词过滤(商业版)
"vision": false, // 支持图片输入
"defaultSystemChatPrompt": ""
},
{
......@@ -47,6 +48,7 @@ weight: 520
"quoteMaxToken": 8000,
"maxTemperature": 1.2,
"censor": false,
"vision": false,
"defaultSystemChatPrompt": ""
},
{
......@@ -58,6 +60,19 @@ weight: 520
"quoteMaxToken": 4000,
"maxTemperature": 1.2,
"censor": false,
"vision": false,
"defaultSystemChatPrompt": ""
},
{
"model": "gpt-4-vision-preview",
"name": "GPT4-Vision",
"maxContext": 128000,
"maxResponse": 4000,
"price": 0,
"quoteMaxToken": 100000,
"maxTemperature": 1.2,
"censor": false,
"vision": true,
"defaultSystemChatPrompt": ""
}
],
......
docSite/content/docs/installation/upgrading/461.md 0 → 100644
---
title: 'V4.6.1'
description: 'FastGPT V4.6 .1'
icon: 'upgrade'
draft: false
toc: true
weight: 835
---
## V4.6.1 功能介绍
1. 新增 - GPT4-v 模型支持
2. 新增 - whisper 语音输入
3. 优化 - TTS 流传输
4. 优化 - TTS 缓存
packages/global/common/string/tools.ts
......@@ -24,7 +24,7 @@ export const simpleText = (text: string) => {
};
/*
replace {{variable}} to value
replace {{variable}} to value
*/
export function replaceVariable(text: string, obj: Record<string, string | number>) {
for (const key in obj) {
......
packages/global/core/ai/model.d.ts
......@@ -9,6 +9,7 @@ export type ChatModelItemType = LLMModelItemType & {
quoteMaxToken: number;
maxTemperature: number;
censor?: boolean;
vision?: boolean;
defaultSystemChatPrompt?: string;
};
......
packages/global/core/ai/model.ts
......@@ -17,6 +17,7 @@ export const defaultChatModels: ChatModelItemType[] = [
quoteMaxToken: 2000,
maxTemperature: 1.2,
censor: false,
vision: false,
defaultSystemChatPrompt: ''
},
{
......@@ -28,6 +29,7 @@ export const defaultChatModels: ChatModelItemType[] = [
quoteMaxToken: 8000,
maxTemperature: 1.2,
censor: false,
vision: false,
defaultSystemChatPrompt: ''
},
{
......@@ -39,6 +41,19 @@ export const defaultChatModels: ChatModelItemType[] = [
quoteMaxToken: 4000,
maxTemperature: 1.2,
censor: false,
vision: false,
defaultSystemChatPrompt: ''
},
{
model: 'gpt-4-vision-preview',
name: 'GPT4-Vision',
maxContext: 128000,
maxResponse: 4000,
price: 0,
quoteMaxToken: 100000,
maxTemperature: 1.2,
censor: false,
vision: true,
defaultSystemChatPrompt: ''
}
];
......
packages/global/core/ai/type.d.ts
......@@ -5,12 +5,14 @@ import type {
ChatCompletionMessageParam,
ChatCompletionContentPart
} from 'openai/resources';
export type ChatCompletionContentPart = ChatCompletionContentPart;
export type ChatCompletionCreateParams = ChatCompletionCreateParams;
export type ChatMessageItemType = Omit<ChatCompletionMessageParam> & {
export type ChatMessageItemType = Omit<ChatCompletionMessageParam, 'name'> & {
name?: any;
dataId?: string;
content: any;
};
} & any;
export type ChatCompletion = ChatCompletion;
export type StreamChatType = Stream<ChatCompletionChunk>;
......
packages/global/core/chat/constants.ts
......@@ -54,3 +54,6 @@ export const ChatSourceMap = {
export const HUMAN_ICON = `/icon/human.svg`;
export const LOGO_ICON = `/icon/logo.svg`;
export const IMG_BLOCK_KEY = 'img-block';
export const FILE_BLOCK_KEY = 'file-block';
packages/global/core/chat/utils.ts 0 → 100644
import { IMG_BLOCK_KEY, FILE_BLOCK_KEY } from './constants';
export function chatContentReplaceBlock(content: string = '') {
const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
return content.replace(regex, '').trim();
}
packages/service/common/buffer/tts/schema.ts
......@@ -33,3 +33,4 @@ try {
export const MongoTTSBuffer: Model<TTSBufferSchemaType> =
models[collectionName] || model(collectionName, TTSBufferSchema);
MongoTTSBuffer.syncIndexes();
packages/service/common/file/image/controller.ts
......@@ -5,12 +5,26 @@ export function getMongoImgUrl(id: string) {
return `${imageBaseUrl}${id}`;
}
export async function uploadMongoImg({ base64Img, userId }: { base64Img: string; userId: string }) {
export const maxImgSize = 1024 * 1024 * 12;
export async function uploadMongoImg({
base64Img,
teamId,
expiredTime
}: {
base64Img: string;
teamId: string;
expiredTime?: Date;
}) {
if (base64Img.length > maxImgSize) {
return Promise.reject('Image too large');
}
const base64Data = base64Img.split(',')[1];
const { _id } = await MongoImage.create({
userId,
binary: Buffer.from(base64Data, 'base64')
teamId,
binary: Buffer.from(base64Data, 'base64'),
expiredTime
});
return getMongoImgUrl(String(_id));
......
packages/service/common/file/image/schema.ts
import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
import { connectionMongo, type Model } from '../../mongo';
const { Schema, model, models } = connectionMongo;
const ImageSchema = new Schema({
userId: {
teamId: {
type: Schema.Types.ObjectId,
ref: 'user',
required: true
ref: TeamCollectionName
},
binary: {
type: Buffer
},
expiredTime: {
type: Date
}
});
export const MongoImage: Model<{ userId: string; binary: Buffer }> =
try {
ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 });
} catch (error) {
console.log(error);
}
export const MongoImage: Model<{ teamId: string; binary: Buffer }> =
models['image'] || model('image', ImageSchema);
MongoImage.syncIndexes();
packages/service/core/app/schema.ts
......@@ -67,3 +67,5 @@ try {
export const MongoApp: Model<AppType> =
models[appCollectionName] || model(appCollectionName, AppSchema);
MongoApp.syncIndexes();
packages/service/core/chat/chatItemSchema.ts
......@@ -83,3 +83,5 @@ try {
export const MongoChatItem: Model<ChatItemType> =
models['chatItem'] || model('chatItem', ChatItemSchema);
MongoChatItem.syncIndexes();
packages/service/core/chat/chatSchema.ts
......@@ -92,7 +92,7 @@ const ChatSchema = new Schema({
});
try {
ChatSchema.index({ userId: 1 });
ChatSchema.index({ tmbId: 1 });
ChatSchema.index({ updateTime: -1 });
ChatSchema.index({ appId: 1 });
} catch (error) {
......@@ -101,3 +101,4 @@ try {
export const MongoChat: Model<ChatType> =
models[chatCollectionName] || model(chatCollectionName, ChatSchema);
MongoChat.syncIndexes();
packages/service/core/chat/utils.ts
import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { ChatRoleEnum, IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants';
import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt';
import type { ChatCompletionContentPart } from '@fastgpt/global/core/ai/type.d';
/* slice chat context by tokens */
export function ChatContextFilter({
......@@ -51,3 +52,101 @@ export function ChatContextFilter({
return [...systemPrompts, ...chats];
}
/**
string to vision model. Follow the markdown code block rule for interception:
@rule:
```img-block
{src:""}
{src:""}
```
```file-block
{name:"",src:""},
{name:"",src:""}
```
@example:
What’s in this image?
```img-block
{src:"https://1.png"}
```
@return
[
{ type: 'text', text: 'What’s in this image?' },
{
type: 'image_url',
image_url: {
url: 'https://1.png'
}
}
]
*/
export function formatStr2ChatContent(str: string) {
const content: ChatCompletionContentPart[] = [];
let lastIndex = 0;
const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
let match;
while ((match = regex.exec(str)) !== null) {
// add previous text
if (match.index > lastIndex) {
const text = str.substring(lastIndex, match.index).trim();
if (text) {
content.push({ type: 'text', text });
}
}
const blockType = match[1].trim();
if (blockType === IMG_BLOCK_KEY) {
const blockContentLines = match[2].trim().split('\n');
const jsonLines = blockContentLines.map((item) => {
try {
return JSON.parse(item) as { src: string };
} catch (error) {
return { src: '' };
}
});
for (const item of jsonLines) {
if (!item.src) throw new Error("image block's content error");
}
content.push(
...jsonLines.map((item) => ({
type: 'image_url' as any,
image_url: {
url: item.src
}
}))
);
}
lastIndex = regex.lastIndex;
}
// add remaining text
if (lastIndex < str.length) {
const remainingText = str.substring(lastIndex).trim();
if (remainingText) {
content.push({ type: 'text', text: remainingText });
}
}
// Continuous text type content, if type=text, merge them
for (let i = 0; i < content.length - 1; i++) {
const currentContent = content[i];
const nextContent = content[i + 1];
if (currentContent.type === 'text' && nextContent.type === 'text') {
currentContent.text += nextContent.text;
content.splice(i + 1, 1);
i--;
}
}
if (content.length === 1 && content[0].type === 'text') {
return content[0].text;
}
return content ? content : null;
}
packages/service/core/dataset/controller.ts
......@@ -22,9 +22,9 @@ export async function findDatasetIdTreeByTopDatasetId(
}
export async function getCollectionWithDataset(collectionId: string) {
const data = (
await MongoDatasetCollection.findById(collectionId).populate('datasetId')
)?.toJSON() as CollectionWithDatasetType;
const data = (await MongoDatasetCollection.findById(collectionId)
.populate('datasetId')
.lean()) as CollectionWithDatasetType;
if (!data) {
return Promise.reject('Collection is not exist');
}
......
packages/service/core/dataset/data/schema.ts
......@@ -76,3 +76,4 @@ try {
export const MongoDatasetData: Model<DatasetDataSchemaType> =
models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema);
MongoDatasetData.syncIndexes();
packages/service/core/dataset/schema.ts
......@@ -82,3 +82,4 @@ try {
export const MongoDataset: Model<DatasetSchemaType> =
models[DatasetCollectionName] || model(DatasetCollectionName, DatasetSchema);
MongoDataset.syncIndexes();
packages/service/core/dataset/training/schema.ts
......@@ -104,3 +104,5 @@ try {
export const MongoDatasetTraining: Model<DatasetTrainingSchemaType> =
models[DatasetTrainingCollectionName] || model(DatasetTrainingCollectionName, TrainingDataSchema);
MongoDatasetTraining.syncIndexes();
packages/service/core/plugin/schema.ts
......@@ -46,10 +46,11 @@ const PluginSchema = new Schema({
});
try {
PluginSchema.index({ userId: 1 });
PluginSchema.index({ tmbId: 1 });
} catch (error) {
console.log(error);
}
export const MongoPlugin: Model<PluginItemSchema> =
models[ModuleCollectionName] || model(ModuleCollectionName, PluginSchema);
MongoPlugin.syncIndexes();
packages/service/support/activity/promotion/schema.ts
......@@ -31,3 +31,4 @@ const PromotionRecordSchema = new Schema({
export const MongoPromotionRecord: Model<PromotionRecordType> =
models['promotionRecord'] || model('promotionRecord', PromotionRecordSchema);
MongoPromotionRecord.syncIndexes();
packages/service/support/openapi/schema.ts
......@@ -70,3 +70,4 @@ const OpenApiSchema = new Schema(
export const MongoOpenApi: Model<OpenApiSchema> =
models['openapi'] || model('openapi', OpenApiSchema);
MongoOpenApi.syncIndexes();
packages/service/support/outLink/schema.ts
......@@ -71,3 +71,5 @@ const OutLinkSchema = new Schema({
export const MongoOutLink: Model<SchemaType> =
models['outlinks'] || model('outlinks', OutLinkSchema);
MongoOutLink.syncIndexes();
packages/service/support/permission/auth/app.ts
......@@ -22,12 +22,12 @@ export async function authApp({
}
> {
const result = await parseHeaderCert(props);
const { userId, teamId, tmbId } = result;
const { teamId, tmbId } = result;
const { role } = await getTeamInfoByTmbId({ tmbId });
const { app, isOwner, canWrite } = await (async () => {
// get app
const app = (await MongoApp.findOne({ _id: appId, teamId }))?.toJSON();
const app = await MongoApp.findOne({ _id: appId, teamId }).lean();
if (!app) {
return Promise.reject(AppErrEnum.unAuthApp);
}
......
packages/service/support/permission/auth/chat.ts
......@@ -24,9 +24,9 @@ export async function authChat({
const { chat, isOwner, canWrite } = await (async () => {
// get chat
const chat = (
await MongoChat.findOne({ chatId, teamId }).populate('appId')
)?.toJSON() as ChatWithAppSchema;
const chat = (await MongoChat.findOne({ chatId, teamId })
.populate('appId')
.lean()) as ChatWithAppSchema;
if (!chat) {
return Promise.reject('Chat is not exists');
......
packages/service/support/permission/auth/dataset.ts
......@@ -31,7 +31,7 @@ export async function authDataset({
const { role } = await getTeamInfoByTmbId({ tmbId });
const { dataset, isOwner, canWrite } = await (async () => {
const dataset = (await MongoDataset.findOne({ _id: datasetId, teamId }))?.toObject();
const dataset = await MongoDataset.findOne({ _id: datasetId, teamId }).lean();
if (!dataset) {
return Promise.reject(DatasetErrEnum.unAuthDataset);
......
packages/service/support/user/schema.ts
......@@ -64,3 +64,4 @@ const UserSchema = new Schema({
export const MongoUser: Model<UserModelSchema> =
models[userCollectionName] || model(userCollectionName, UserSchema);
MongoUser.syncIndexes();
packages/service/support/wallet/bill/schema.ts
......@@ -59,3 +59,4 @@ try {
}
export const MongoBill: Model<BillType> = models['bill'] || model('bill', BillSchema);
MongoBill.syncIndexes();
projects/app/data/config.json
......@@ -15,6 +15,7 @@
"quoteMaxToken": 2000,
"maxTemperature": 1.2,
"censor": false,
"vision": false,
"defaultSystemChatPrompt": ""
},
{
......@@ -26,6 +27,7 @@
"quoteMaxToken": 8000,
"maxTemperature": 1.2,
"censor": false,
"vision": false,
"defaultSystemChatPrompt": ""
},
{
......@@ -37,6 +39,19 @@
"quoteMaxToken": 4000,
"maxTemperature": 1.2,
"censor": false,
"vision": false,
"defaultSystemChatPrompt": ""
},
{
"model": "gpt-4-vision-preview",
"name": "GPT4-Vision",
"maxContext": 128000,
"maxResponse": 4000,
"price": 0,
"quoteMaxToken": 100000,
"maxTemperature": 1.2,
"censor": false,
"vision": true,
"defaultSystemChatPrompt": ""
}
],
......
projects/app/public/locales/en/common.json
......@@ -191,6 +191,7 @@
"Update Success": "Update Success",
"Update Successful": "Update Successful",
"Update Time": "Update Time",
"Upload File Failed": "Upload File Failed",
"Username": "UserName",
"error": {
"unKnow": "There was an accident"
......
projects/app/public/locales/zh/common.json
......@@ -191,6 +191,7 @@
"Update Success": "更新成功",
"Update Successful": "更新成功",
"Update Time": "更新时间",
"Upload File Failed": "上传文件失败",
"Username": "用户名",
"error": {
"unKnow": "出现了点意外~"
......
projects/app/src/components/ChatBox/index.tsx
......@@ -17,7 +17,17 @@ import { useToast } from '@/web/common/hooks/useToast';
import { useAudioPlay } from '@/web/common/utils/voice';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useCopyData } from '@/web/common/hooks/useCopyData';
import { Box, Card, Flex, Input, Button, useTheme, BoxProps, FlexProps } from '@chakra-ui/react';
import {
Box,
Card,
Flex,
Input,
Button,
useTheme,
BoxProps,
FlexProps,
Image
} from '@chakra-ui/react';
import { feConfigs } from '@/web/common/system/staticData';
import { eventBus } from '@/web/common/utils/eventbus';
import { adaptChat2GptMessages } from '@fastgpt/global/core/chat/adapt';
......@@ -90,6 +100,7 @@ type Props = {
appAvatar?: string;
userAvatar?: string;
userGuideModule?: ModuleItemType;
showFileSelector?: boolean;
active?: boolean; // can use
onUpdateVariable?: (e: Record<string, any>) => void;
onStartChat?: (e: StartChatFnProps) => Promise<{
......@@ -109,6 +120,7 @@ const ChatBox = (
appAvatar,
userAvatar,
userGuideModule,
showFileSelector,
active = true,
onUpdateVariable,
onStartChat,
......@@ -795,6 +807,7 @@ const ChatBox = (
isChatting={isChatting}
TextareaDom={TextareaDom}
resetInputVal={resetInputVal}
showFileSelector={showFileSelector}
/>
) : null}
......@@ -1109,11 +1122,11 @@ function ChatController({
{...controlIconStyle}
mr={1}
name={'core/chat/stopSpeech'}
_hover={{ color: '#E74694' }}
color={'#E74694'}
onClick={() => cancelAudio()}
/>
</MyTooltip>
{/* <MyIcon name={'loading'} w={'16px'} /> */}
<Image src="/icon/speaking.gif" w={'23px'} alt={''} />
</Flex>
) : (
<MyTooltip label={t('core.app.TTS')}>
......
projects/app/src/components/Markdown/chat/Image.tsx
import { Box, Flex } from '@chakra-ui/react';
import MdImage from '../img/Image';
import { useMemo } from 'react';
const ImageBlock = ({ images }: { images: string }) => {
const formatData = useMemo(
() =>
images.split('\n').map((item) => {
try {
return JSON.parse(item) as { src: string };
} catch (error) {
return { src: '' };
}
}),
[images]
);
return (
<Flex w={'100%'} wrap={'wrap'}>
{JSON.parse(images).map((src: string) => {
<Flex alignItems={'center'} wrap={'wrap'} gap={4}>
{formatData.map(({ src }) => {
return (
<Box key={src} mr={2} mb={2} rounded={'md'} flex={'0 0 auto'} w={'100px'} h={'100px'}>
<Box key={src} rounded={'md'} flex={'0 0 auto'} w={'120px'}>
<MdImage src={src} />
</Box>
);
......
projects/app/src/components/Markdown/img/Image.tsx
......@@ -27,6 +27,7 @@ const MdImage = ({ src }: { src?: string }) => {
borderRadius={'md'}
src={src}
alt={''}
maxH={'150px'}
fallbackSrc={'/imgs/errImg.png'}
fallbackStrategy={'onError'}
cursor={succeed ? 'pointer' : 'default'}
......@@ -45,7 +46,6 @@ const MdImage = ({ src }: { src?: string }) => {
<Modal isOpen={isOpen} onClose={onClose}>
<ModalOverlay />
<ModalContent m={'auto'}>
<ModalCloseButton />
<Image
src={src}
alt={''}
......@@ -55,6 +55,7 @@ const MdImage = ({ src }: { src?: string }) => {
objectFit={'contain'}
/>
</ModalContent>
<ModalCloseButton bg={'myWhite.500'} zIndex={999999} />
</Modal>
</Skeleton>
);
......
projects/app/src/components/core/module/Flow/ChatTest.tsx
......@@ -16,6 +16,7 @@ import MyTooltip from '@/components/MyTooltip';
import { useUserStore } from '@/web/support/user/useUserStore';
import ChatBox, { type ComponentRef, type StartChatFnProps } from '@/components/ChatBox';
import { getGuideModule } from '@/global/core/app/modules/utils';
import { checkChatSupportSelectFileByModules } from '@/web/core/chat/utils';
export type ChatTestComponentRef = {
resetChatTest: () => void;
......@@ -115,6 +116,7 @@ const ChatTest = (
userAvatar={userInfo?.avatar}
showMarkIcon
userGuideModule={getGuideModule(modules)}
showFileSelector={checkChatSupportSelectFileByModules(modules)}
onStartChat={startChat}
onDelMessage={() => {}}
/>
......
projects/app/src/components/support/user/team/UpdateInviteModal/index.tsx
......@@ -52,19 +52,19 @@ const UpdateInviteModal = () => {
return (
<MyModal
isOpen={inviteList.length > 0}
isOpen={inviteList && inviteList.length > 0}
title={
<>
<Box>{t('user.team.Processing invitations')}</Box>
<Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'}>
{t('user.team.Processing invitations Tips', { amount: inviteList.length })}
{t('user.team.Processing invitations Tips', { amount: inviteList?.length })}
</Box>
</>
}
maxW={['90vw', '500px']}
>
<ModalBody>
{inviteList.map((item) => (
{inviteList?.map((item) => (
<Flex
key={item.teamId}
alignItems={'center'}
......
projects/app/src/pages/_app.tsx
......@@ -95,7 +95,7 @@ function App({ Component, pageProps }: AppProps) {
<title>{feConfigs?.systemTitle || process.env.SYSTEM_NAME || 'GPT'}</title>
<meta
name="description"
content="FastGPT is a knowledge-based question answering system built on the LLM. It offers out-of-the-box data processing and model invocation capabilities. Moreover, it allows for workflow orchestration through Flow visualization, thereby enabling complex question and answer scenarios!"
content="FastGPT 是一个大模型应用编排系统，提供开箱即用的数据处理、模型调用等能力，可以快速的构建知识库并通过 Flow 可视化进行工作流编排，实现复杂的知识库场景！"
/>
<meta
name="viewport"
......
projects/app/src/pages/api/common/file/uploadImage.ts
......@@ -4,17 +4,18 @@ import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { uploadMongoImg } from '@fastgpt/service/common/file/image/controller';
type Props = { base64Img: string };
type Props = { base64Img: string; expiredTime?: Date };
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { userId } = await authCert({ req, authToken: true });
const { base64Img } = req.body as Props;
const { teamId } = await authCert({ req, authToken: true });
const { base64Img, expiredTime } = req.body as Props;
const data = await uploadMongoImg({
userId,
base64Img
teamId,
base64Img,
expiredTime
});
jsonRes(res, { data });
......
projects/app/src/pages/api/core/dataset/allDataset.ts
......@@ -17,10 +17,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const datasets = await MongoDataset.find({
...mongoRPermission({ teamId, tmbId, role }),
type: 'dataset'
});
}).lean();
const data = datasets.map((item) => ({
...item.toJSON(),
...item,
vectorModel: getVectorModel(item.vectorModel),
agentModel: getQAModel(item.agentModel),
canWrite: String(item.tmbId) === tmbId,
......
projects/app/src/pages/api/core/dataset/list.ts
......@@ -22,11 +22,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
...mongoRPermission({ teamId, tmbId, role }),
...(parentId !== undefined && { parentId: parentId || null }),
...(type && { type })
}).sort({ updateTime: -1 });
})
.sort({ updateTime: -1 })
.lean();
const data = await Promise.all(
datasets.map(async (item) => ({
...item.toJSON(),
...item,
vectorModel: getVectorModel(item.vectorModel),
agentModel: getQAModel(item.agentModel),
canWrite,
......
projects/app/src/pages/app/detail/components/BasicEdit/index.tsx
......@@ -59,6 +59,7 @@ import { useAppStore } from '@/web/core/app/store/useAppStore';
import PermissionIconText from '@/components/support/permission/IconText';
import QGSwitch from '../QGSwitch';
import TTSSelect from '../TTSSelect';
import { checkChatSupportSelectFileByModules } from '@/web/core/chat/utils';
const VariableEditModal = dynamic(() => import('@/components/core/module/VariableEditModal'));
const InfoModal = dynamic(() => import('../InfoModal'));
......@@ -676,6 +677,7 @@ const ChatTest = ({ appId }: { appId: string }) => {
userAvatar={userInfo?.avatar}
showMarkIcon
userGuideModule={getGuideModule(modules)}
showFileSelector={checkChatSupportSelectFileByModules(modules)}
onStartChat={startChat}
onDelMessage={() => {}}
/>
......
projects/app/src/pages/app/detail/components/TTSSelect.tsx
import MyIcon from '@/components/Icon';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { Box, Button, Flex, ModalBody, useDisclosure } from '@chakra-ui/react';
import { Box, Button, Flex, ModalBody, useDisclosure, Image } from '@chakra-ui/react';
import React, { useCallback, useMemo } from 'react';
import { useTranslation } from 'next-i18next';
import MySelect from '@/components/Select';
......@@ -130,9 +130,9 @@ const TTSSelect = ({
<Flex mt={10} justifyContent={'end'}>
{audioPlaying ? (
<Flex>
<MyIcon name={'core/chat/speaking'} w={'16px'} />
<Image src="/icon/speaking.gif" w={'24px'} alt={''} />
<Button
ml={3}
ml={2}
variant={'gray'}
isLoading={audioLoading}
leftIcon={<MyIcon name={'core/chat/stopSpeech'} w={'16px'} />}
......
projects/app/src/pages/chat/components/ChatHeader.tsx
......@@ -7,6 +7,7 @@ import Avatar from '@/components/Avatar';
import ToolMenu from './ToolMenu';
import type { ChatItemType } from '@fastgpt/global/core/chat/type';
import { useRouter } from 'next/router';
import { chatContentReplaceBlock } from '@fastgpt/global/core/chat/utils';
const ChatHeader = ({
history,
......@@ -27,7 +28,10 @@ const ChatHeader = ({
const theme = useTheme();
const { isPc } = useSystemStore();
const title = useMemo(
() => history[history.length - 2]?.value?.slice(0, 8) || appName || '新对话',
() =>
chatContentReplaceBlock(history[history.length - 2]?.value)?.slice(0, 8) ||
appName ||
'新对话',
[appName, history]
);
......
projects/app/src/pages/chat/index.tsx
......@@ -32,6 +32,8 @@ import { getErrText } from '@fastgpt/global/common/error/utils';
import { useUserStore } from '@/web/support/user/useUserStore';
import { serviceSideProps } from '@/web/common/utils/i18n';
import { useAppStore } from '@/web/core/app/store/useAppStore';
import { checkChatSupportSelectFileByChatModels } from '@/web/core/chat/utils';
import { chatContentReplaceBlock } from '@fastgpt/global/core/chat/utils';
const Chat = ({ appId, chatId }: { appId: string; chatId: string }) => {
const router = useRouter();
......@@ -78,7 +80,10 @@ const Chat = ({ appId, chatId }: { appId: string; chatId: string }) => {
abortSignal: controller
});
const newTitle = prompts[0].content?.slice(0, 20) || '新对话';
const newTitle =
chatContentReplaceBlock(prompts[0].content).slice(0, 20) ||
prompts[1]?.value?.slice(0, 20) ||
'新对话';
// update history
if (completionChatId !== chatId) {
......@@ -363,6 +368,7 @@ const Chat = ({ appId, chatId }: { appId: string; chatId: string }) => {
appAvatar={chatData.app.avatar}
userAvatar={userInfo?.avatar}
userGuideModule={chatData.app?.userGuideModule}
showFileSelector={checkChatSupportSelectFileByChatModels(chatData.app.chatModels)}
feedbackType={'user'}
onUpdateVariable={(e) => {}}
onStartChat={startChat}
......
projects/app/src/pages/chat/share.tsx
......@@ -20,6 +20,7 @@ import PageContainer from '@/components/PageContainer';
import ChatHeader from './components/ChatHeader';
import ChatHistorySlider from './components/ChatHistorySlider';
import { serviceSideProps } from '@/web/common/utils/i18n';
import { checkChatSupportSelectFileByChatModels } from '@/web/core/chat/utils';
const OutLink = ({
shareId,
......@@ -254,6 +255,9 @@ const OutLink = ({
appAvatar={shareChatData.app.avatar}
userAvatar={shareChatData.userAvatar}
userGuideModule={shareChatData.app?.userGuideModule}
showFileSelector={checkChatSupportSelectFileByChatModels(
shareChatData.app.chatModels
)}
feedbackType={'user'}
onUpdateVariable={(e) => {
setShareChatData((state) => ({
......
projects/app/src/service/events/generateQA.ts
......@@ -29,16 +29,16 @@ export async function generateQA(): Promise<any> {
error = false
} = await (async () => {
try {
const data = (
await MongoDatasetTraining.findOneAndUpdate(
{
mode: TrainingModeEnum.qa,
lockTime: { $lte: new Date(Date.now() - 10 * 60 * 1000) }
},
{
lockTime: new Date()
}
).select({
const data = await MongoDatasetTraining.findOneAndUpdate(
{
mode: TrainingModeEnum.qa,
lockTime: { $lte: new Date(Date.now() - 10 * 60 * 1000) }
},
{
lockTime: new Date()
}
)
.select({
_id: 1,
userId: 1,
teamId: 1,
......@@ -50,7 +50,7 @@ export async function generateQA(): Promise<any> {
billId: 1,
prompt: 1
})
)?.toJSON();
.lean();
// task preemption
if (!data) {
......
projects/app/src/service/events/generateVector.ts
......@@ -24,16 +24,16 @@ export async function generateVector(): Promise<any> {
error = false
} = await (async () => {
try {
const data = (
await MongoDatasetTraining.findOneAndUpdate(
{
mode: TrainingModeEnum.chunk,
lockTime: { $lte: new Date(Date.now() - 1 * 60 * 1000) }
},
{
lockTime: new Date()
}
).select({
const data = await MongoDatasetTraining.findOneAndUpdate(
{
mode: TrainingModeEnum.chunk,
lockTime: { $lte: new Date(Date.now() - 1 * 60 * 1000) }
},
{
lockTime: new Date()
}
)
.select({
_id: 1,
userId: 1,
teamId: 1,
......@@ -46,7 +46,7 @@ export async function generateVector(): Promise<any> {
model: 1,
billId: 1
})
)?.toJSON();
.lean();
// task preemption
if (!data) {
......
projects/app/src/service/moduleDispatch/chat/oneapi.ts
......@@ -21,6 +21,7 @@ import type { ModuleDispatchProps } from '@/types/core/chat/type';
import { responseWrite, responseWriteController } from '@fastgpt/service/common/response';
import { getChatModel, ModelTypeEnum } from '@/service/core/ai/model';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { formatStr2ChatContent } from '@fastgpt/service/core/chat/utils';
export type ChatProps = ModuleDispatchProps<
AIChatProps & {
......@@ -106,6 +107,21 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
temperature = Math.max(temperature, 0.01);
const ai = getAIApi(user.openaiAccount, 480000);
const concatMessages = [
...(modelConstantsData.defaultSystemChatPrompt
? [
{
role: ChatCompletionRequestMessageRoleEnum.System,
content: modelConstantsData.defaultSystemChatPrompt
}
]
: []),
...messages.map((item) => ({
...item,
content: modelConstantsData.vision ? formatStr2ChatContent(item.content) : item.content
}))
];
const response = await ai.chat.completions.create(
{
model,
......@@ -113,17 +129,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
max_tokens,
stream,
seed: temperature < 0.3 ? 1 : undefined,
messages: [
...(modelConstantsData.defaultSystemChatPrompt
? [
{
role: ChatCompletionRequestMessageRoleEnum.System,
content: modelConstantsData.defaultSystemChatPrompt
}
]
: []),
...messages
]
messages: concatMessages
},
{
headers: {
......
projects/app/src/service/utils/chat/saveChat.ts
......@@ -4,6 +4,7 @@ import { ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import { addLog } from '@fastgpt/service/common/mongo/controller';
import { chatContentReplaceBlock } from '@fastgpt/global/core/chat/utils';
type Props = {
chatId: string;
......@@ -51,12 +52,17 @@ export async function saveChat({
)
];
const title =
chatContentReplaceBlock(content[0].value).slice(0, 20) ||
content[1]?.value?.slice(0, 20) ||
'Chat';
if (chatHistory) {
promise.push(
MongoChat.updateOne(
{ chatId },
{
title: content[0].value.slice(0, 20),
title,
updateTime: new Date()
}
)
......@@ -69,7 +75,7 @@ export async function saveChat({
tmbId,
appId,
variables,
title: content[0].value.slice(0, 20),
title,
source,
shareId
})
......
projects/app/src/web/common/file/api.ts
import { GET, POST, PUT, DELETE } from '@/web/common/api/request';
import { AxiosProgressEvent } from 'axios';
export const postUploadImg = (base64Img: string) =>
POST<string>('/common/file/uploadImage', { base64Img });
export const postUploadImg = (base64Img: string, expiredTime?: Date) =>
POST<string>('/common/file/uploadImage', { base64Img, expiredTime });
export const postUploadFiles = (
data: FormData,
......
projects/app/src/web/common/file/controller.ts
......@@ -37,12 +37,14 @@ export const compressImgAndUpload = ({
file,
maxW = 200,
maxH = 200,
maxSize = 1024 * 100
maxSize = 1024 * 100, // 100kb
expiredTime
}: {
file: File;
maxW?: number;
maxH?: number;
maxSize?: number;
expiredTime?: Date;
}) =>
new Promise<string>((resolve, reject) => {
const reader = new FileReader();
......@@ -87,7 +89,7 @@ export const compressImgAndUpload = ({
const src = await (async () => {
try {
const src = await postUploadImg(compressedDataUrl);
const src = await postUploadImg(compressedDataUrl, expiredTime);
return src;
} catch (error) {
return compressedDataUrl;
......
projects/app/src/web/common/file/hooks/useSelectFile.tsx
......@@ -3,9 +3,13 @@ import { Box } from '@chakra-ui/react';
import { useToast } from '@/web/common/hooks/useToast';
import { useTranslation } from 'next-i18next';
export const useSelectFile = (props?: { fileType?: string; multiple?: boolean }) => {
export const useSelectFile = (props?: {
fileType?: string;
multiple?: boolean;
maxCount?: number;
}) => {
const { t } = useTranslation();
const { fileType = '*', multiple = false } = props || {};
const { fileType = '*', multiple = false, maxCount = 10 } = props || {};
const { toast } = useToast();
const SelectFileDom = useRef<HTMLInputElement>(null);
......@@ -19,7 +23,7 @@ export const useSelectFile = (props?: { fileType?: string; multiple?: boolean })
multiple={multiple}
onChange={(e) => {
if (!e.target.files || e.target.files?.length === 0) return;
if (e.target.files.length > 10) {
if (e.target.files.length > maxCount) {
return toast({
status: 'warning',
title: t('file.Select a maximum of 10 files')
......@@ -30,7 +34,7 @@ export const useSelectFile = (props?: { fileType?: string; multiple?: boolean })
/>
</Box>
),
[fileType, multiple, t, toast]
[fileType, maxCount, multiple, t, toast]
);
const onOpen = useCallback(() => {
......
projects/app/src/web/common/utils/voice.ts
import { useState, useCallback, useEffect, useMemo } from 'react';
import { useState, useCallback, useEffect, useMemo, useRef } from 'react';
import { useToast } from '@/web/common/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { AppTTSConfigType } from '@/types/app';
......@@ -14,6 +14,7 @@ export const useAudioPlay = (props?: { ttsConfig?: AppTTSConfigType }) => {
const [audio, setAudio] = useState<HTMLAudioElement>();
const [audioLoading, setAudioLoading] = useState(false);
const [audioPlaying, setAudioPlaying] = useState(false);
const audioController = useRef(new AbortController());
// Check whether the voice is supported
const hasAudio = useMemo(() => {
......@@ -49,12 +50,15 @@ export const useAudioPlay = (props?: { ttsConfig?: AppTTSConfigType }) => {
return resolve({ buffer });
}
audioController.current = new AbortController();
/* request tts */
const response = await fetch('/api/core/chat/item/getSpeech', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
signal: audioController.current.signal,
body: JSON.stringify({
chatItemId,
ttsConfig,
......@@ -120,6 +124,7 @@ export const useAudioPlay = (props?: { ttsConfig?: AppTTSConfigType }) => {
audio.src = '';
}
window.speechSynthesis?.cancel();
audioController.current?.abort();
setAudioPlaying(false);
}, [audio]);
......
projects/app/src/web/core/chat/storeShareChat.ts
......@@ -7,6 +7,7 @@ import type {
} from '@fastgpt/global/support/outLink/api.d';
import type { ChatSiteItemType } from '@fastgpt/global/core/chat/type.d';
import { HUMAN_ICON } from '@fastgpt/global/core/chat/constants';
import { chatContentReplaceBlock } from '@fastgpt/global/core/chat/utils';
type State = {
shareChatData: ShareChatType;
......@@ -64,6 +65,10 @@ export const useShareChatStore = create<State>()(
shareChatHistory: [],
saveChatResponse({ chatId, prompts, variables, shareId }) {
const chatHistory = get().shareChatHistory.find((item) => item.chatId === chatId);
const newTitle =
chatContentReplaceBlock(prompts[prompts.length - 2]?.value).slice(0, 20) ||
prompts[prompts.length - 1]?.value?.slice(0, 20) ||
'Chat';
const historyList = (() => {
if (chatHistory) {
......@@ -71,7 +76,7 @@ export const useShareChatStore = create<State>()(
item.chatId === chatId
? {
...item,
title: prompts[prompts.length - 2]?.value,
title: newTitle,
updateTime: new Date(),
chats: chatHistory.chats.concat(prompts).slice(-30),
variables
......@@ -82,7 +87,7 @@ export const useShareChatStore = create<State>()(
return get().shareChatHistory.concat({
chatId,
shareId,
title: prompts[prompts.length - 2]?.value,
title: newTitle,
updateTime: new Date(),
chats: prompts,
variables
......
projects/app/src/web/core/chat/utils.ts 0 → 100644
import { chatModelList } from '@/web/common/system/staticData';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
import { ModuleItemType } from '@fastgpt/global/core/module/type.d';
export function checkChatSupportSelectFileByChatModels(models: string[] = []) {
for (const model of models) {
const modelData = chatModelList.find((item) => item.model === model || item.name === model);
if (modelData?.vision) {
return true;
}
}
return false;
}
export function checkChatSupportSelectFileByModules(modules: ModuleItemType[] = []) {
const chatModules = modules.filter((item) => item.flowType === FlowNodeTypeEnum.chatNode);
const models: string[] = chatModules.map(
(item) => item.inputs.find((item) => item.key === 'model')?.value || ''
);
return checkChatSupportSelectFileByChatModels(models);
}
projects/app/src/web/styles/theme.ts
......@@ -83,7 +83,7 @@ const Button = defineStyleConfig({
_hover: {
color: 'myBlue.600',
bg: 'myWhite.400',
boxShadow: '0 0 5px rgba(0,0,0,0.2)'
boxShadow: '0 0 5px rgba(0,0,0,0.1)'
},
_active: {
color: 'myBlue.700'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment