Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 2fcf4216 by Archer Committed by archer
Options

add model test log (#4272) 

* sync collection

* remove lock

* add model test log

* update ui

* update log

* fix: channel test

* preview chunk ui

* test model ux

* test model log

* perf: dataset selector

* fix: system plugin auth

* update nextjs
parent a680b565
Showing with 150 additions and 93 deletions
docSite/content/zh-cn/docs/development/upgrading/492.md
...@@ -25,7 +25,11 @@ weight: 799 ...@@ -25,7 +25,11 @@ weight: 799
2. 邀请链接交互。 2. 邀请链接交互。
3. 无 SSL 证书时复制失败，会提示弹窗用于手动复制。 3. 无 SSL 证书时复制失败，会提示弹窗用于手动复制。
4. FastGPT 未内置 ai proxy 渠道时，也能正常展示其名称。 4. FastGPT 未内置 ai proxy 渠道时，也能正常展示其名称。
5. 升级 nextjs 版本至 14.2.25。
## 🐛 修复 ## 🐛 修复
1. 飞书和语雀知识库无法同步。 1. 飞书和语雀知识库无法同步。
\ No newline at end of file 2. 渠道测试时，如果配置了模型自定义请求地址，会走自定义请求地址，而不是渠道请求地址。
3. 语音识别模型测试未启用的模型时，无法正常测试。
4. 管理员配置系统插件时，如果插件包含其他系统应用，无法正常鉴权。
\ No newline at end of file
packages/global/core/plugin/type.d.ts
...@@ -41,6 +41,8 @@ export type PluginTemplateType = PluginRuntimeType & { ...@@ -41,6 +41,8 @@ export type PluginTemplateType = PluginRuntimeType & {
export type PluginRuntimeType = { export type PluginRuntimeType = {
id: string; id: string;
teamId?: string; teamId?: string;
tmbId?: string;
name: string; name: string;
avatar: string; avatar: string;
showStatus?: boolean; showStatus?: boolean;
......
packages/global/package.json
...@@ -10,7 +10,7 @@ ...@@ -10,7 +10,7 @@
"js-yaml": "^4.1.0", "js-yaml": "^4.1.0",
"jschardet": "3.1.1", "jschardet": "3.1.1",
"nanoid": "^5.1.3", "nanoid": "^5.1.3",
"next": "14.2.24", "next": "14.2.25",
"openai": "4.61.0", "openai": "4.61.0",
"openapi-types": "^12.1.3", "openapi-types": "^12.1.3",
"json5": "^2.2.3", "json5": "^2.2.3",
......
packages/service/core/ai/audio/transcriptions.ts
...@@ -3,21 +3,25 @@ import { getAxiosConfig } from '../config'; ...@@ -3,21 +3,25 @@ import { getAxiosConfig } from '../config';
import axios from 'axios'; import axios from 'axios';
import FormData from 'form-data'; import FormData from 'form-data';
import { getSTTModel } from '../model'; import { getSTTModel } from '../model';
import { STTModelType } from '@fastgpt/global/core/ai/model.d';
export const aiTranscriptions = async ({ export const aiTranscriptions = async ({
model, model: modelData,
fileStream, fileStream,
headers headers
}: { }: {
model: string; model: STTModelType;
fileStream: fs.ReadStream; fileStream: fs.ReadStream;
headers?: Record<string, string>; headers?: Record<string, string>;
}) => { }) => {
if (!modelData) {
return Promise.reject('no model');
}
const data = new FormData(); const data = new FormData();
data.append('model', model); data.append('model', modelData.model);
data.append('file', fileStream); data.append('file', fileStream);
const modelData = getSTTModel(model);
const aiAxiosConfig = getAxiosConfig(); const aiAxiosConfig = getAxiosConfig();
const { data: result } = await axios<{ text: string }>({ const { data: result } = await axios<{ text: string }>({
......
packages/service/core/app/plugin/controller.ts
...@@ -37,11 +37,12 @@ export async function splitCombinePluginId(id: string) { ...@@ -37,11 +37,12 @@ export async function splitCombinePluginId(id: string) {
return { source, pluginId: id }; return { source, pluginId: id };
} }
type ChildAppType = SystemPluginTemplateItemType & { teamId?: string }; type ChildAppType = SystemPluginTemplateItemType & { teamId?: string; tmbId?: string };
const getSystemPluginTemplateById = async ( const getSystemPluginTemplateById = async (
pluginId: string, pluginId: string,
versionId?: string versionId?: string
): Promise<SystemPluginTemplateItemType> => { ): Promise<ChildAppType> => {
const item = getSystemPluginTemplates().find((plugin) => plugin.id === pluginId); const item = getSystemPluginTemplates().find((plugin) => plugin.id === pluginId);
if (!item) return Promise.reject(PluginErrEnum.unAuth); if (!item) return Promise.reject(PluginErrEnum.unAuth);
...@@ -67,12 +68,17 @@ const getSystemPluginTemplateById = async ( ...@@ -67,12 +68,17 @@ const getSystemPluginTemplateById = async (
: await getAppLatestVersion(plugin.associatedPluginId, app); : await getAppLatestVersion(plugin.associatedPluginId, app);
if (!version.versionId) return Promise.reject('App version not found'); if (!version.versionId) return Promise.reject('App version not found');
plugin.workflow = { return {
nodes: version.nodes, ...plugin,
edges: version.edges, workflow: {
chatConfig: version.chatConfig nodes: version.nodes,
edges: version.edges,
chatConfig: version.chatConfig
},
version: versionId || String(version.versionId),
teamId: String(app.teamId),
tmbId: String(app.tmbId)
}; };
plugin.version = versionId || String(version.versionId);
} }
return plugin; return plugin;
}; };
...@@ -168,6 +174,7 @@ export async function getChildAppRuntimeById( ...@@ -168,6 +174,7 @@ export async function getChildAppRuntimeById(
return { return {
id: String(item._id), id: String(item._id),
teamId: String(item.teamId), teamId: String(item.teamId),
tmbId: String(item.tmbId),
name: item.name, name: item.name,
avatar: item.avatar, avatar: item.avatar,
intro: item.intro, intro: item.intro,
...@@ -187,6 +194,7 @@ export async function getChildAppRuntimeById( ...@@ -187,6 +194,7 @@ export async function getChildAppRuntimeById(
pluginOrder: 0 pluginOrder: 0
}; };
} else { } else {
// System
return getSystemPluginTemplateById(pluginId, versionId); return getSystemPluginTemplateById(pluginId, versionId);
} }
})(); })();
...@@ -194,6 +202,7 @@ export async function getChildAppRuntimeById( ...@@ -194,6 +202,7 @@ export async function getChildAppRuntimeById(
return { return {
id: app.id, id: app.id,
teamId: app.teamId, teamId: app.teamId,
tmbId: app.tmbId,
name: app.name, name: app.name,
avatar: app.avatar, avatar: app.avatar,
showStatus: app.showStatus, showStatus: app.showStatus,
......
packages/service/core/workflow/dispatch/plugin/run.ts
...@@ -88,9 +88,9 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi ...@@ -88,9 +88,9 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
: {}), : {}),
runningAppInfo: { runningAppInfo: {
id: String(plugin.id), id: String(plugin.id),
// 如果是系统插件，则使用当前团队的 teamId 和 tmbId // 如果系统插件有 teamId 和 tmbId，则使用系统插件的 teamId 和 tmbId（管理员指定了插件作为系统插件）
teamId: plugin.teamId || runningAppInfo.teamId, teamId: plugin.teamId || runningAppInfo.teamId,
tmbId: pluginData?.tmbId || runningAppInfo.tmbId tmbId: plugin.tmbId || runningAppInfo.tmbId
}, },
variables: runtimeVariables, variables: runtimeVariables,
query: getPluginRunUserQuery({ query: getPluginRunUserQuery({
......
packages/service/package.json
...@@ -26,7 +26,7 @@ ...@@ -26,7 +26,7 @@
"mammoth": "^1.6.0", "mammoth": "^1.6.0",
"mongoose": "^8.10.1", "mongoose": "^8.10.1",
"multer": "1.4.5-lts.1", "multer": "1.4.5-lts.1",
"next": "14.2.24", "next": "14.2.25",
"nextjs-cors": "^2.2.0", "nextjs-cors": "^2.2.0",
"node-cron": "^3.0.3", "node-cron": "^3.0.3",
"node-xlsx": "^0.24.0", "node-xlsx": "^0.24.0",
......
packages/web/i18n/en/account_model.json
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
"channel_status_unknown": "unknown", "channel_status_unknown": "unknown",
"channel_type": "Manufacturer", "channel_type": "Manufacturer",
"clear_model": "Clear the model", "clear_model": "Clear the model",
"confirm_delete_channel": "Confirm the deletion of the [{{name}}] channel?",
"copy_model_id_success": "Copyed model id", "copy_model_id_success": "Copyed model id",
"create_channel": "Added channels", "create_channel": "Added channels",
"default_url": "Default address", "default_url": "Default address",
......
packages/web/i18n/en/dataset.json
...@@ -80,7 +80,7 @@ ...@@ -80,7 +80,7 @@
"permission.des.write": "Ability to add and change knowledge base content", "permission.des.write": "Ability to add and change knowledge base content",
"preview_chunk": "Preview chunks", "preview_chunk": "Preview chunks",
"preview_chunk_empty": "Unable to read the contents of the file", "preview_chunk_empty": "Unable to read the contents of the file",
"preview_chunk_intro": "Display up to 10 pieces", "preview_chunk_intro": "A total of {{total}} blocks, up to 10",
"preview_chunk_not_selected": "Click on the file on the left to preview", "preview_chunk_not_selected": "Click on the file on the left to preview",
"rebuild_embedding_start_tip": "Index model switching task has started", "rebuild_embedding_start_tip": "Index model switching task has started",
"rebuilding_index_count": "Number of indexes being rebuilt: {{count}}", "rebuilding_index_count": "Number of indexes being rebuilt: {{count}}",
......
packages/web/i18n/zh-CN/account_model.json
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
"channel_status_unknown": "未知", "channel_status_unknown": "未知",
"channel_type": "厂商", "channel_type": "厂商",
"clear_model": "清空模型", "clear_model": "清空模型",
"confirm_delete_channel": "确认删除 【{{name}}】渠道？",
"copy_model_id_success": "已复制模型id", "copy_model_id_success": "已复制模型id",
"create_channel": "新增渠道", "create_channel": "新增渠道",
"default_url": "默认地址", "default_url": "默认地址",
......
packages/web/i18n/zh-CN/dataset.json
...@@ -80,7 +80,7 @@ ...@@ -80,7 +80,7 @@
"permission.des.write": "可增加和变更知识库内容", "permission.des.write": "可增加和变更知识库内容",
"preview_chunk": "分块预览", "preview_chunk": "分块预览",
"preview_chunk_empty": "无法读取该文件内容", "preview_chunk_empty": "无法读取该文件内容",
"preview_chunk_intro": "最多展示 10 个分块", "preview_chunk_intro": "共 {{total}} 个分块，最多展示 10 个",
"preview_chunk_not_selected": "点击左侧文件后进行预览", "preview_chunk_not_selected": "点击左侧文件后进行预览",
"rebuild_embedding_start_tip": "切换索引模型任务已开始", "rebuild_embedding_start_tip": "切换索引模型任务已开始",
"rebuilding_index_count": "重建中索引数量：{{count}}", "rebuilding_index_count": "重建中索引数量：{{count}}",
......
packages/web/i18n/zh-Hant/account_model.json
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
"channel_status_unknown": "未知", "channel_status_unknown": "未知",
"channel_type": "廠商", "channel_type": "廠商",
"clear_model": "清空模型", "clear_model": "清空模型",
"confirm_delete_channel": "確認刪除 【{{name}}】渠道？",
"copy_model_id_success": "已復制模型id", "copy_model_id_success": "已復制模型id",
"create_channel": "新增渠道", "create_channel": "新增渠道",
"default_url": "默認地址", "default_url": "默認地址",
......
packages/web/i18n/zh-Hant/dataset.json
...@@ -80,7 +80,7 @@ ...@@ -80,7 +80,7 @@
"permission.des.write": "可新增和變更資料集內容", "permission.des.write": "可新增和變更資料集內容",
"preview_chunk": "分塊預覽", "preview_chunk": "分塊預覽",
"preview_chunk_empty": "無法讀取該文件內容", "preview_chunk_empty": "無法讀取該文件內容",
"preview_chunk_intro": "最多展示 10 個分塊", "preview_chunk_intro": "共 {{total}} 個分塊，最多展示 10 個",
"preview_chunk_not_selected": "點擊左側文件後進行預覽", "preview_chunk_not_selected": "點擊左側文件後進行預覽",
"rebuild_embedding_start_tip": "切換索引模型任務已開始", "rebuild_embedding_start_tip": "切換索引模型任務已開始",
"rebuilding_index_count": "重建中索引數量：{{count}}", "rebuilding_index_count": "重建中索引數量：{{count}}",
......
projects/app/package.json
...@@ -42,7 +42,7 @@ ...@@ -42,7 +42,7 @@
"lodash": "^4.17.21", "lodash": "^4.17.21",
"mermaid": "^10.2.3", "mermaid": "^10.2.3",
"nanoid": "^5.1.3", "nanoid": "^5.1.3",
"next": "14.2.24", "next": "14.2.25",
"next-i18next": "15.4.2", "next-i18next": "15.4.2",
"nprogress": "^0.2.0", "nprogress": "^0.2.0",
"qrcode": "^1.5.4", "qrcode": "^1.5.4",
......
projects/app/src/components/core/app/DatasetSelectModal.tsx
...@@ -69,31 +69,37 @@ export const DatasetSelectModal = ({ ...@@ -69,31 +69,37 @@ export const DatasetSelectModal = ({
{selectedDatasets.map((item) => {selectedDatasets.map((item) =>
(() => { (() => {
return ( return (
<Card <MyTooltip label={item.name}>
key={item.datasetId} <Card
p={3} key={item.datasetId}
border={theme.borders.base} p={3}
boxShadow={'sm'} border={'base'}
bg={'primary.200'} boxShadow={'sm'}
> bg={'primary.200'}
<Flex alignItems={'center'} h={'38px'}> >
<Avatar src={item.avatar} w={['1.25rem', '1.75rem']}></Avatar> <Flex alignItems={'center'} h={'38px'}>
<Box flex={'1 0 0'} w={0} className="textEllipsis" mx={3}> <Avatar
{item.name} src={item.avatar}
</Box> w={['1.25rem', '1.75rem']}
<MyIcon borderRadius={'sm'}
name={'delete'} ></Avatar>
w={'14px'} <Box flex={'1 0 0'} w={0} className="textEllipsis" mx={3} fontSize={'sm'}>
cursor={'pointer'} {item.name}
_hover={{ color: 'red.500' }} </Box>
onClick={() => { <MyIcon
setSelectedDatasets((state) => name={'delete'}
state.filter((dataset) => dataset.datasetId !== item.datasetId) w={'14px'}
); cursor={'pointer'}
}} _hover={{ color: 'red.500' }}
/> onClick={() => {
</Flex> setSelectedDatasets((state) =>
</Card> state.filter((dataset) => dataset.datasetId !== item.datasetId)
);
}}
/>
</Flex>
</Card>
</MyTooltip>
); );
})() })()
)} )}
...@@ -117,7 +123,7 @@ export const DatasetSelectModal = ({ ...@@ -117,7 +123,7 @@ export const DatasetSelectModal = ({
label={ label={
item.type === DatasetTypeEnum.folder item.type === DatasetTypeEnum.folder
? t('common:dataset.Select Folder') ? t('common:dataset.Select Folder')
: t('common:dataset.Select Dataset') : item.name
} }
> >
<Card <Card
...@@ -152,14 +158,18 @@ export const DatasetSelectModal = ({ ...@@ -152,14 +158,18 @@ export const DatasetSelectModal = ({
}} }}
> >
<Flex alignItems={'center'} h={'38px'}> <Flex alignItems={'center'} h={'38px'}>
<Avatar src={item.avatar} w={['24px', '28px']}></Avatar> <Avatar
src={item.avatar}
w={['1.25rem', '1.75rem']}
borderRadius={'sm'}
></Avatar>
<Box <Box
flex={'1 0 0'} flex={'1 0 0'}
w={0} w={0}
className="textEllipsis" className="textEllipsis"
ml={3} ml={3}
fontSize={'md'}
color={'myGray.900'} color={'myGray.900'}
fontSize={'sm'}
> >
{item.name} {item.name}
</Box> </Box>
......
projects/app/src/components/core/chat/components/AIResponseBox.tsx
...@@ -268,12 +268,10 @@ const RenderUserFormInteractive = React.memo(function RenderFormInput({ ...@@ -268,12 +268,10 @@ const RenderUserFormInteractive = React.memo(function RenderFormInput({
{interactive.params.description && <Markdown source={interactive.params.description} />} {interactive.params.description && <Markdown source={interactive.params.description} />}
{interactive.params.inputForm?.map((input) => ( {interactive.params.inputForm?.map((input) => (
<Box key={input.label}> <Box key={input.label}>
<Flex mb={1} alignItems={'center'} w={'full'}> <FormLabel mb={1} required={input.required} whiteSpace={'pre-wrap'}>
<FormLabel required={input.required} w={'full'} whiteSpace={'pre-wrap'}> {input.label}
{input.label} {input.description && <QuestionTip ml={1} label={input.description} />}
{input.description && <QuestionTip ml={1} label={input.description} />} </FormLabel>
</FormLabel>
</Flex>
{input.type === FlowNodeInputTypeEnum.input && ( {input.type === FlowNodeInputTypeEnum.input && (
<MyTextarea <MyTextarea
isDisabled={interactive.params.submitted} isDisabled={interactive.params.submitted}
......
projects/app/src/components/core/chat/components/WholeResponseModal.tsx
...@@ -250,22 +250,24 @@ export const WholeResponseContent = ({ ...@@ -250,22 +250,24 @@ export const WholeResponseContent = ({
value={activeModule?.similarity} value={activeModule?.similarity}
/> />
<Row label={t('common:core.chat.response.module limit')} value={activeModule?.limit} /> <Row label={t('common:core.chat.response.module limit')} value={activeModule?.limit} />
<Row {activeModule?.searchUsingReRank !== undefined && (
label={t('common:core.chat.response.search using reRank')} <Row
rawDom={ label={t('common:core.chat.response.search using reRank')}
<Box border={'base'} borderRadius={'md'} p={2}> rawDom={
{activeModule?.searchUsingReRank ? ( <Box border={'base'} borderRadius={'md'} p={2}>
activeModule?.rerankModel ? ( {activeModule?.searchUsingReRank ? (
<Box>{`${activeModule.rerankModel}: ${activeModule.rerankWeight}`}</Box> activeModule?.rerankModel ? (
<Box>{`${activeModule.rerankModel}: ${activeModule.rerankWeight}`}</Box>
) : (
'True'
)
) : ( ) : (
'True' `False`
) )}
) : ( </Box>
`False` }
)} />
</Box> )}
}
/>
{activeModule.queryExtensionResult && ( {activeModule.queryExtensionResult && (
<> <>
<Row <Row
......
projects/app/src/pageComponents/account/model/Channel/index.tsx
...@@ -38,6 +38,7 @@ import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; ...@@ -38,6 +38,7 @@ import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput'; import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput';
import { getModelProvider } from '@fastgpt/global/core/ai/provider'; import { getModelProvider } from '@fastgpt/global/core/ai/provider';
import MyIcon from '@fastgpt/web/components/common/Icon'; import MyIcon from '@fastgpt/web/components/common/Icon';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
const EditChannelModal = dynamic(() => import('./EditChannelModal'), { ssr: false }); const EditChannelModal = dynamic(() => import('./EditChannelModal'), { ssr: false });
const ModelTest = dynamic(() => import('./ModelTest'), { ssr: false }); const ModelTest = dynamic(() => import('./ModelTest'), { ssr: false });
...@@ -77,6 +78,9 @@ const ChannelTable = ({ Tab }: { Tab: React.ReactNode }) => { ...@@ -77,6 +78,9 @@ const ChannelTable = ({ Tab }: { Tab: React.ReactNode }) => {
} }
); );
const { openConfirm, ConfirmModal } = useConfirm({
type: 'delete'
});
const { runAsync: onDeleteChannel, loading: loadingDeleteChannel } = useRequest2(deleteChannel, { const { runAsync: onDeleteChannel, loading: loadingDeleteChannel } = useRequest2(deleteChannel, {
manual: true, manual: true,
onSuccess: () => { onSuccess: () => {
...@@ -212,7 +216,14 @@ const ChannelTable = ({ Tab }: { Tab: React.ReactNode }) => { ...@@ -212,7 +216,14 @@ const ChannelTable = ({ Tab }: { Tab: React.ReactNode }) => {
type: 'danger', type: 'danger',
icon: 'delete', icon: 'delete',
label: t('common:common.Delete'), label: t('common:common.Delete'),
onClick: () => onDeleteChannel(item.id) onClick: () =>
openConfirm(
() => onDeleteChannel(item.id),
undefined,
t('account_model:confirm_delete_channel', {
name: item.name
})
)()
} }
] ]
} }
...@@ -238,6 +249,7 @@ const ChannelTable = ({ Tab }: { Tab: React.ReactNode }) => { ...@@ -238,6 +249,7 @@ const ChannelTable = ({ Tab }: { Tab: React.ReactNode }) => {
{!!modelTestData && ( {!!modelTestData && (
<ModelTest {...modelTestData} onClose={() => setTestModelData(undefined)} /> <ModelTest {...modelTestData} onClose={() => setTestModelData(undefined)} />
)} )}
<ConfirmModal />
</> </>
); );
}; };
......
projects/app/src/pageComponents/account/model/Log/index.tsx
...@@ -197,7 +197,7 @@ const ChannelLog = ({ Tab }: { Tab: React.ReactNode }) => { ...@@ -197,7 +197,7 @@ const ChannelLog = ({ Tab }: { Tab: React.ReactNode }) => {
/> />
</Box> </Box>
</HStack> </HStack>
<HStack flex={'0 0 200px'}> <HStack>
<FormLabel>{t('account_model:channel_name')}</FormLabel> <FormLabel>{t('account_model:channel_name')}</FormLabel>
<Box flex={'1 0 0'}> <Box flex={'1 0 0'}>
<MySelect<string> <MySelect<string>
...@@ -210,7 +210,7 @@ const ChannelLog = ({ Tab }: { Tab: React.ReactNode }) => { ...@@ -210,7 +210,7 @@ const ChannelLog = ({ Tab }: { Tab: React.ReactNode }) => {
/> />
</Box> </Box>
</HStack> </HStack>
<HStack flex={'0 0 200px'}> <HStack>
<FormLabel>{t('account_model:model_name')}</FormLabel> <FormLabel>{t('account_model:model_name')}</FormLabel>
<Box flex={'1 0 0'}> <Box flex={'1 0 0'}>
<MySelect<string> <MySelect<string>
......
projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx
...@@ -34,9 +34,9 @@ const PreviewData = () => { ...@@ -34,9 +34,9 @@ const PreviewData = () => {
const [previewFile, setPreviewFile] = useState<ImportSourceItemType>(); const [previewFile, setPreviewFile] = useState<ImportSourceItemType>();
const { data = [], loading: isLoading } = useRequest2( const { data = { chunks: [], total: 0 }, loading: isLoading } = useRequest2(
async () => { async () => {
if (!previewFile) return; if (!previewFile) return { chunks: [], total: 0 };
if (importSource === ImportDataSourceEnum.fileCustom) { if (importSource === ImportDataSourceEnum.fileCustom) {
const chunkSplitter = processParamsForm.getValues('chunkSplitter'); const chunkSplitter = processParamsForm.getValues('chunkSplitter');
const { chunks } = splitText2Chunks({ const { chunks } = splitText2Chunks({
...@@ -46,10 +46,13 @@ const PreviewData = () => { ...@@ -46,10 +46,13 @@ const PreviewData = () => {
overlapRatio: chunkOverlapRatio, overlapRatio: chunkOverlapRatio,
customReg: chunkSplitter ? [chunkSplitter] : [] customReg: chunkSplitter ? [chunkSplitter] : []
}); });
return chunks.map((chunk) => ({ return {
q: chunk, chunks: chunks.map((chunk) => ({
a: '' q: chunk,
})); a: ''
})),
total: chunks.length
};
} }
return getPreviewChunks({ return getPreviewChunks({
...@@ -81,7 +84,7 @@ const PreviewData = () => { ...@@ -81,7 +84,7 @@ const PreviewData = () => {
manual: false, manual: false,
onSuccess(result) { onSuccess(result) {
if (!previewFile) return; if (!previewFile) return;
if (!result || result.length === 0) { if (!result || result.total === 0) {
toast({ toast({
title: t('dataset:preview_chunk_empty'), title: t('dataset:preview_chunk_empty'),
status: 'error' status: 'error'
...@@ -130,14 +133,14 @@ const PreviewData = () => { ...@@ -130,14 +133,14 @@ const PreviewData = () => {
<Flex py={4} px={5} borderBottom={'base'} justifyContent={'space-between'}> <Flex py={4} px={5} borderBottom={'base'} justifyContent={'space-between'}>
<FormLabel fontSize={'md'}>{t('dataset:preview_chunk')}</FormLabel> <FormLabel fontSize={'md'}>{t('dataset:preview_chunk')}</FormLabel>
<Box fontSize={'xs'} color={'myGray.500'}> <Box fontSize={'xs'} color={'myGray.500'}>
{t('dataset:preview_chunk_intro')} {t('dataset:preview_chunk_intro', { total: data.total })}
</Box> </Box>
</Flex> </Flex>
<MyBox isLoading={isLoading} flex={'1 0 0'} h={0}> <MyBox isLoading={isLoading} flex={'1 0 0'} h={0}>
<Box h={'100%'} overflowY={'auto'} px={5} py={3}> <Box h={'100%'} overflowY={'auto'} px={5} py={3}>
{previewFile ? ( {previewFile ? (
<> <>
{data.map((item, index) => ( {data.chunks.map((item, index) => (
<Box <Box
key={index} key={index}
fontSize={'sm'} fontSize={'sm'}
......
projects/app/src/pages/api/core/ai/model/test.ts
...@@ -35,11 +35,17 @@ async function handler( ...@@ -35,11 +35,17 @@ async function handler(
if (!modelData) return Promise.reject('Model not found'); if (!modelData) return Promise.reject('Model not found');
if (channelId) {
delete modelData.requestUrl;
delete modelData.requestAuth;
}
const headers: Record<string, string> = channelId const headers: Record<string, string> = channelId
? { ? {
'Aiproxy-Channel': String(channelId) 'Aiproxy-Channel': String(channelId)
} }
: {}; : {};
addLog.debug(`Test model`, modelData);
if (modelData.type === 'llm') { if (modelData.type === 'llm') {
return testLLMModel(modelData, headers); return testLLMModel(modelData, headers);
...@@ -63,10 +69,6 @@ async function handler( ...@@ -63,10 +69,6 @@ async function handler(
export default NextAPI(handler); export default NextAPI(handler);
const testLLMModel = async (model: LLMModelItemType, headers: Record<string, string>) => { const testLLMModel = async (model: LLMModelItemType, headers: Record<string, string>) => {
const ai = getAIApi({
timeout: 10000
});
const requestBody = llmCompletionsBodyFormat( const requestBody = llmCompletionsBodyFormat(
{ {
model: model.model, model: model.model,
...@@ -75,6 +77,7 @@ const testLLMModel = async (model: LLMModelItemType, headers: Record<string, str ...@@ -75,6 +77,7 @@ const testLLMModel = async (model: LLMModelItemType, headers: Record<string, str
}, },
model model
); );
const { response, isStreamResponse } = await createChatCompletion({ const { response, isStreamResponse } = await createChatCompletion({
body: requestBody, body: requestBody,
options: { options: {
...@@ -144,7 +147,7 @@ const testTTSModel = async (model: TTSModelType, headers: Record<string, string> ...@@ -144,7 +147,7 @@ const testTTSModel = async (model: TTSModelType, headers: Record<string, string>
const testSTTModel = async (model: STTModelType, headers: Record<string, string>) => { const testSTTModel = async (model: STTModelType, headers: Record<string, string>) => {
const path = isProduction ? '/app/data/test.mp3' : 'data/test.mp3'; const path = isProduction ? '/app/data/test.mp3' : 'data/test.mp3';
const { text } = await aiTranscriptions({ const { text } = await aiTranscriptions({
model: model.model, model,
fileStream: fs.createReadStream(path), fileStream: fs.createReadStream(path),
headers headers
}); });
......
projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
...@@ -43,9 +43,12 @@ export type PostPreviewFilesChunksProps = { ...@@ -43,9 +43,12 @@ export type PostPreviewFilesChunksProps = {
externalFileId?: string; externalFileId?: string;
}; };
export type PreviewChunksResponse = { export type PreviewChunksResponse = {
q: string; chunks: {
a: string; q: string;
}[]; a: string;
}[];
total: number;
};
async function handler( async function handler(
req: ApiRequestProps<PostPreviewFilesChunksProps> req: ApiRequestProps<PostPreviewFilesChunksProps>
...@@ -123,13 +126,17 @@ async function handler( ...@@ -123,13 +126,17 @@ async function handler(
customPdfParse customPdfParse
}); });
return rawText2Chunks({ const chunks = rawText2Chunks({
rawText, rawText,
chunkSize, chunkSize,
maxSize: getLLMMaxChunkSize(getLLMModel(dataset.agentModel)), maxSize: getLLMMaxChunkSize(getLLMModel(dataset.agentModel)),
overlapRatio, overlapRatio,
customReg: chunkSplitter ? [chunkSplitter] : [], customReg: chunkSplitter ? [chunkSplitter] : [],
isQAImport: isQAImport isQAImport: isQAImport
}).slice(0, 10); });
return {
chunks: chunks.slice(0, 10),
total: chunks.length
};
} }
export default NextAPI(handler); export default NextAPI(handler);
projects/app/src/pages/api/v1/audio/transcriptions.ts
...@@ -66,7 +66,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) { ...@@ -66,7 +66,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
// } // }
const result = await aiTranscriptions({ const result = await aiTranscriptions({
model: getDefaultSTTModel().model, model: getDefaultSTTModel(),
fileStream: fs.createReadStream(file.path) fileStream: fs.createReadStream(file.path)
}); });
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment